eval_new_task_prompt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e062dd85-17e6-41df-8dd7-7784934f455c",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import sys\n",
    "import time\n",
    "import warnings\n",
    "import argparse\n",
    "import functools\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "\n",
    "import tensorflow.compat.v1 as tf\n",
    "import tensorflow_datasets as tfds\n",
    "\n",
    "import t5\n",
    "import t5.models\n",
    "import seqio\n",
    "\n",
    "from pycocoevalcap.bleu.bleu import Bleu\n",
    "from pycocoevalcap.meteor.meteor import Meteor\n",
    "from pycocoevalcap.rouge.rouge import Rouge\n",
    "from statistics import mean\n",
    "import pandas as pd\n",
    "\n",
    "tpu_name = \"node-3\"\n",
    "input_len = 384\n",
    "target_len = 512\n",
    "lr = 0.00002\n",
    "shot = 'all'\n",
    "support = 'all'\n",
    "MODEL_SIZE = '3B'\n",
    "eval_checkpoint = 1020544\n",
    "\n",
    "BASE_DIR = \"gs://t5_fewshot_mqg\" #@param { type: \"string\" }\n",
    "if not BASE_DIR or BASE_DIR == \"gs://\":\n",
    "    raise ValueError(\"You must enter a BASE_DIR.\")\n",
    "DATA_DIR = os.path.join(BASE_DIR, \"data/t5_experiment_1/\")\n",
    "MODELS_DIR = os.path.join(BASE_DIR, \"models/t5_experiment_1/\")\n",
    "ON_CLOUD = True\n",
    "\n",
    "# Public GCS path for T5 pre-trained model checkpoints\n",
    "BASE_PRETRAINED_DIR = \"gs://t5-data/pretrained_models\"\n",
    "PRETRAINED_DIR = os.path.join(BASE_PRETRAINED_DIR, MODEL_SIZE)\n",
    "MODEL_DIR = os.path.join(MODELS_DIR, MODEL_SIZE, f\"{shot}_SHOT\", support)\n",
    "\n",
    "model_parallelism, train_batch_size, keep_checkpoint_max = {\n",
    "    \"small\": (1, 256,  10),\n",
    "    \"base\":  (2, 128,  10),\n",
    "    \"large\": (8,  64,  10),\n",
    "    \"3B\":    (8,  16,  10),\n",
    "    \"11B\":   (8,   4,  10)}[MODEL_SIZE]\n",
    "\n",
    "# Directory of Fewshot HotpotQG data on GCS.\n",
    "fhp_tsv_path = {support_type: {\n",
    "    \"test\": os.path.join(DATA_DIR, f\"test.{support_type}.tsv\")\n",
    "} for support_type in ['supp', 'all']}\n",
    "\n",
    "if ON_CLOUD:\n",
    "    print(\"Setting up GCS access...\")\n",
    "# Use legacy GCS authentication method.\n",
    "os.environ['USE_AUTH_EPHEM'] = '0'\n",
    "import tensorflow_gcs_config\n",
    "\n",
    "# from google.colab import auth\n",
    "# auth.authenticate_user()\n",
    "\n",
    "# Set credentials for GCS reading/writing from Colab and TPU.\n",
    "TPU_TOPOLOGY = \"v3-8\"\n",
    "try:\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=tpu_name)  # TPU detection\n",
    "    TPU_ADDRESS = tpu.get_master()\n",
    "    print('Running on TPU:', TPU_ADDRESS)\n",
    "except ValueError:\n",
    "    raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n",
    "tf.enable_eager_execution()\n",
    "tf.config.experimental_connect_to_host(TPU_ADDRESS)\n",
    "# tensorflow_gcs_config.configure_gcs_from_colab_auth()\n",
    "\n",
    "tf.disable_v2_behavior()\n",
    "\n",
    "# Improve logging.\n",
    "from contextlib import contextmanager\n",
    "import logging as py_logging\n",
    "\n",
    "if ON_CLOUD:\n",
    "    tf.get_logger().propagate = False\n",
    "py_logging.root.setLevel('INFO')\n",
    "\n",
    "def tf_verbosity_level(level):\n",
    "    og_level = tf.logging.get_verbosity()\n",
    "    tf.logging.set_verbosity(level)\n",
    "    yield\n",
    "    tf.logging.set_verbosity(og_level)\n",
    "\n",
    "def fewshot_qg_preprocessor(ds):\n",
    "    def normalize_text(text):\n",
    "        \"\"\"Remove quotes from a TensorFlow string.\"\"\"\n",
    "        #text = tf.strings.lower(text)\n",
    "        text = tf.strings.regex_replace(text,\"'(.*)'\", r\"\\1\")\n",
    "        text = tf.strings.regex_replace(text,'\"(.*)\"', r\"\\1\")\n",
    "        text = tf.strings.regex_replace(text,\"\\\"\", \"\")\n",
    "        text = tf.strings.regex_replace(text,\"'\", \"\")\n",
    "        return text\n",
    "\n",
    "    def to_inputs_and_targets(ex):\n",
    "        \"\"\"Map {\"question\": ..., \"answer\": ...}->{\"inputs\": ..., \"targets\": ...}.\"\"\"\n",
    "        return {\n",
    "            \"inputs\":\n",
    "                tf.strings.join(\n",
    "                    [normalize_text(ex[\"question\"])]),\n",
    "            \"targets\": normalize_text(ex[\"answer\"])\n",
    "        }\n",
    "    return ds.map(to_inputs_and_targets, \n",
    "                    num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
    "\n",
    "DEFAULT_OUTPUT_FEATURES = {\n",
    "    \"inputs\":\n",
    "        seqio.Feature(\n",
    "            vocabulary=t5.data.get_default_vocabulary(), add_eos=True),\n",
    "    \"targets\":\n",
    "        seqio.Feature(\n",
    "            vocabulary=t5.data.get_default_vocabulary(), add_eos=True)\n",
    "}\n",
    "\n",
    "b4m, mm, rm = Bleu(4), Meteor(), Rouge()\n",
    "\n",
    "bleu4 = lambda targets, predictions: {'bleu4': 100 * mean([b4m.compute_score({i:[targets[i]]}, {i:[predictions[i]]})[0][3] for i in range(len(targets))])}\n",
    "bleu3 = lambda targets, predictions: {'bleu3': 100 * mean([b4m.compute_score({i:[targets[i]]}, {i:[predictions[i]]})[0][2] for i in range(len(targets))])}\n",
    "bleu2 = lambda targets, predictions: {'bleu2': 100 * mean([b4m.compute_score({i:[targets[i]]}, {i:[predictions[i]]})[0][1] for i in range(len(targets))])}\n",
    "bleu1 = lambda targets, predictions: {'bleu1': 100 * mean([b4m.compute_score({i:[targets[i]]}, {i:[predictions[i]]})[0][0] for i in range(len(targets))])}\n",
    "\n",
    "rouge = lambda targets, predictions: {'rougeL': 100 * mean([rm.compute_score({i:[targets[i]]}, {i:[predictions[i]]})[0] for i in range(len(targets))])}\n",
    "\n",
    "def meteor(targets, predictions):\n",
    "    from pycocoevalcap.meteor.meteor import Meteor\n",
    "    return {'meteor': 100 * mean([mm.compute_score({i:[targets[i]]}, {i:[predictions[i]]})[0] for i in range(len(targets))])}\n",
    "\n",
    "def remove_tags_and_strip(string, **unused_kwargs): \n",
    "    return re.sub('<extra_id_[0-9]{1,2}>', '', string).strip()\n",
    "\n",
    "tf.io.gfile.makedirs(MODEL_DIR)\n",
    "# The models from our paper are based on the Mesh Tensorflow Transformer.\n",
    "model = t5.models.MtfModel(\n",
    "    model_dir=MODEL_DIR,\n",
    "    tpu=TPU_ADDRESS,\n",
    "    tpu_topology=TPU_TOPOLOGY,\n",
    "    model_parallelism=model_parallelism,\n",
    "    batch_size=train_batch_size,\n",
    "    sequence_length={\"inputs\": input_len, \"targets\": target_len},\n",
    "    learning_rate_schedule=lr,\n",
    "    save_checkpoints_steps=1000,\n",
    "    keep_checkpoint_max=keep_checkpoint_max if ON_CLOUD else None,\n",
    "    iterations_per_loop=100,\n",
    ")\n",
    "\n",
    "import sys\n",
    "from absl import app\n",
    "\n",
    "# Addresses `UnrecognizedFlagError: Unknown command line flag 'f'`\n",
    "sys.argv = sys.argv[:1]\n",
    "\n",
    "# `app.run` calls `sys.exit`\n",
    "try:\n",
    "    app.run(lambda argv: None)\n",
    "except:\n",
    "    pass\n",
    "\n",
    "# Use a larger batch size for evaluation, which requires less memory.\n",
    "model.batch_size = train_batch_size*4\n",
    "\n",
    "# model.eval(\n",
    "#     mixture_or_task_name=\"fewshot_mqg_rationale\",\n",
    "#     checkpoint_steps=eval_checkpoint,\n",
    "#     compute_sequence_length=False,\n",
    "#     split='validation'\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "e8c3a622-1342-45c5-a043-d76f6e5ab212",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import string \n",
    "\n",
    "# Storing the sets of punctuation in variable result \n",
    "result = string.punctuation \n",
    "\n",
    "test_file = fhp_tsv_path['all']['test']\n",
    "\n",
    "with tf.io.gfile.GFile(test_file) as f:\n",
    "    tsv_reader = csv.reader(f, delimiter='\\t')\n",
    "    test_data = [{\"k\": k, \"p1\":p1, \"p2\":p2, \"a\": a, \"gold_qtype\": qtype, \"gold_q\": gold_q} for k, (p1,p2,a,qtype,gold_q) in enumerate(tsv_reader)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "fce6e364-fd88-4566-b490-1452439f9623",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1536\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "ids = [i for i, e in enumerate(test_data) if not ((e['a'] in e['p1']) or (e['a'] in e['p2'])) and not e['a'] in ('yes', 'no')]\n",
    "for idx in ids:\n",
    "    example = test_data[idx]\n",
    "    print(example['gold_qtype'])\n",
    "    print(example['a'])\n",
    "    print(example['p1'])\n",
    "    print(example['p2'])\n",
    "    print()\n",
    "assert ids == []\n",
    "ids = [i for i, e in enumerate(test_data) if ((e['a'] in e['p1']) and (e['a'] in e['p2']))]\n",
    "print(len(ids))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af55b96c-5246-4898-a390-3267ffd226a0",
   "metadata": {},
   "source": [
    "# REMOVE THIS V BEFORE BIG RUN!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "d9012885-df4a-4c03-877b-4542dce50bbc",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "predict_data = test_data[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "ced01c2c-ade3-4bfa-ab62-4b830b2002d6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import t5.data.mixtures\n",
    "\n",
    "sentinel_token = [f'<extra_id_{num}>' for num in range(100)]\n",
    "\n",
    "# Write out the supplied test examples to text files\n",
    "predict_inputs_path = {task_num: os.path.join(MODEL_DIR, f\"predict_inputs_{task_num}.txt\") for task_num in range(1, 14)}\n",
    "predict_outputs_path = {task_num: os.path.join(MODEL_DIR, f\"predict_outputs_{task_num}.txt\") for task_num in range(1, 14)}\n",
    "\n",
    "# For each task, example number -> list of input/output file line numbers (0 indexed)\n",
    "# Dict of dict -> list of ints\n",
    "predict_e2l = {task_num: None for task_num in range(1,14)}\n",
    "\n",
    "# For each task, input/output file line number -> example number (0 indexed)\n",
    "# Dict of dict -> int \n",
    "predict_l2e = {task_num: None for task_num in range(1,14)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "dd678a35-da40-4ebf-b612-3b888a2d38e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "task_input_prompts = {\n",
    "    1: lambda i: f\"Context 1: {i['p1']} Context 2: {i['p2']} Answer: {i['a']} Common entities found: {sentinel_token[0]} Question type: {sentinel_token[1]}\",\n",
    "    2: lambda i: f\"Context 1: {i['p1']} Context 2: {i['p2']} Answer: {i['a']} Common entities found: {sentinel_token[0]} Bridge entity: {sentinel_token[1]}\",\n",
    "    3: lambda i: f\"Answer: {i['a']} is {sentinel_token[0]} in context: {i['p']}\",\n",
    "    4: lambda i: f\"Entities: {i['a']} and {i['b']} are {sentinel_token[0]}.\",\n",
    "    5: lambda i: f\"Context: {i['p']} Bridge entity: {i['b']} Answer: {i['a']} Assertion: {sentinel_token[0]}\",\n",
    "    6: lambda i: f\"Context: {i['p']} Bridge entity: {i['b']} Assertion: {sentinel_token[0]}\",\n",
    "    7: lambda i: f\"Bridge entity: {i['b']} Assertion 1: {i['s1']} Assertion 2: {i['s2']} Answer: {i['a']} Combined: {sentinel_token[0]}\",\n",
    "    8: lambda i: f\"Removing bridge entity: {i['b']} from: {i['c']} Answer: {i['a']} We get: {sentinel_token[0]}\",\n",
    "    9: lambda i: f\"Contract answer entity {i['a']} from: {i['c-b']} We get: {sentinel_token[0]}\",\n",
    "    10: lambda i: f\"Combined assertion with answer: {i['c-b']} Combined assertion without answer: {i['c-a']} Answer: {i['a']} Question: {sentinel_token[0]}\",\n",
    "    11: lambda i: f\"Context 1: {i['p1']} Context 2: {i['p2']} Answer: {i['a']} Assertion from Context 1: {sentinel_token[0]} Assertion from Context 2: {sentinel_token[1]}\",\n",
    "    12: lambda i: f\"Assertion 1: {i['s1']} Assertion 2: {i['s2']} Answer: {i['a']} Combine, compare and think: {sentinel_token[0]}\",\n",
    "    13: lambda i: f\"Combined assertion: {i['c']} Answer: {i['a']} Question: {sentinel_token[0]}\",\n",
    "}\n",
    "\n",
    "class SimpleCounter:\n",
    "    def __init__(self, count=0):\n",
    "        self.count = count\n",
    "    \n",
    "    def increment(self):\n",
    "        self.count += 1\n",
    "    \n",
    "    def decrement(self):\n",
    "        self.count -= 1\n",
    "    \n",
    "    def get_count(self):\n",
    "        return self.count\n",
    "    \n",
    "    def set_count(self, count):\n",
    "        self.count = count\n",
    "\n",
    "# Manually apply preprocessing:\n",
    "def write_task_input_file(task_num, task_data=predict_data, \n",
    "                          predict_inputs_path=predict_inputs_path,\n",
    "                          predict_outputs_path=predict_outputs_path,\n",
    "                          predict_e2l=predict_e2l,\n",
    "                          predict_l2e=predict_l2e):\n",
    "        \n",
    "    # dictionary from example number to file line numbers \n",
    "    e2l = defaultdict(list)\n",
    "    \n",
    "    # dictionary from file line number to example number\n",
    "    l2e = dict()\n",
    "    \n",
    "    # make a counter that keeps track of\n",
    "    # how many lines have been written a file\n",
    "    file_lines_written = SimpleCounter(0)\n",
    "    \n",
    "\n",
    "    \n",
    "    with tf.io.gfile.GFile(predict_inputs_path[task_num], \"w\") as f:\n",
    "        \n",
    "        def write_example_line(input_example, e2l=e2l, l2e=l2e, file_obj=f,\n",
    "                               task_num=task_num, file_lines_written=file_lines_written,\n",
    "                               task_input_prompts=task_input_prompts):\n",
    "            input_example_num = input_example['k']\n",
    "            e2l[input_example_num].append(file_lines_written.get_count())\n",
    "            l2e[file_lines_written.get_count()] = input_example_num\n",
    "            file_obj.write(task_input_prompts[task_num](input_example) + \"\\n\")\n",
    "            file_lines_written.increment()\n",
    "            \n",
    "        for k, i in enumerate(task_data):\n",
    "            if task_num == 1:\n",
    "                write_example_line(i)\n",
    "            elif task_num in [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]:\n",
    "                if i['qtype'] in ['bridge', 'confused']:\n",
    "                    if task_num in [2, 3, 4, 5, 6, 7, 8, 9, 10]:\n",
    "                        if task_num in [2, 4, 7, 8, 9, 10]:\n",
    "                            write_example_line(i)\n",
    "\n",
    "                        elif task_num == 3:\n",
    "                            i['p'] = i['p1']\n",
    "                            write_example_line(i)\n",
    "                            i['p'] = i['p2']\n",
    "                            write_example_line(i)\n",
    "                            i.pop('p', None)\n",
    "\n",
    "                        elif task_num == 5:\n",
    "                            if not i['b_same_as_a'] or not(i['a_in_p1'] and i['a_in_p2']):\n",
    "                                if i['a_in_p1']:\n",
    "                                    i['p'] = i['p1']\n",
    "                                    task_data[k]['5_p1'] = True\n",
    "                                    task_data[k]['5_p2'] = False\n",
    "                                    write_example_line(i)\n",
    "                                    i.pop('p', None)\n",
    "                                elif i['a_in_p2']:\n",
    "                                    i['p'] = i['p2']\n",
    "                                    task_data[k]['5_p2'] = True\n",
    "                                    task_data[k]['5_p1'] = False\n",
    "                                    write_example_line(i)\n",
    "                                    i.pop('p', None)\n",
    "                            else:\n",
    "                                task_data[k]['5_p1'] = False\n",
    "                                task_data[k]['5_p2'] = False\n",
    "                                \n",
    "                        elif task_num == 6:\n",
    "                            if i['b_same_as_a'] or (i['a_in_p1'] and i['a_in_p2']):\n",
    "                                i['p'] = i['p1']\n",
    "                                task_data[k]['6_p1'] = file_lines_written.get_count()\n",
    "                                write_example_line(i)\n",
    "                                i.pop('p', None)\n",
    "                                \n",
    "                                i['p'] = i['p2']\n",
    "                                task_data[k]['6_p2'] = file_lines_written.get_count()\n",
    "                                write_example_line(i)\n",
    "                                i.pop('p', None)\n",
    "                            else:\n",
    "                                if (i['a_in_p1']) and (not i['a_in_p2']):\n",
    "                                    i['p'] = i['p2']\n",
    "                                    task_data[k]['6_p2'] = file_lines_written.get_count()\n",
    "                                    write_example_line(i)\n",
    "                                    i.pop('p', None)\n",
    "                                    \n",
    "                                if (not i['a_in_p1']) and i['a_in_p2']: \n",
    "                                    i['p'] = i['p1']\n",
    "                                    task_data[k]['6_p1'] = file_lines_written.get_count()\n",
    "                                    write_example_line(i)\n",
    "                                    i.pop('p', None)\n",
    "                            \n",
    "                            for key in ['6_p1', '6_p2']:\n",
    "                                if key not in task_data[k].keys():\n",
    "                                    task_data[k][key] = None\n",
    "\n",
    "                if i['qtype'] in ['comparison', 'confused']:\n",
    "                    if task_num in [11, 12, 13]:\n",
    "                        write_example_line(i)\n",
    "    \n",
    "    predict_e2l[task_num] = e2l\n",
    "    predict_l2e[task_num] = l2e\n",
    "                        \n",
    "# TODO for output parsing:\n",
    "# add failsafes for tasks 1, 2, 3, 4, 7, 8, 9, 12 \n",
    "# only verification possible: 5, 6, 10 \n",
    "# only minimal failsafe/verify possible: 11, 13\n",
    "def verify(**kwargs):\n",
    "    return True\n",
    "def failsafe(**kwargs):\n",
    "    return \"bleh\", \"bleh\"\n",
    "\n",
    "# Manually apply postprocessing and enrich the task_data with outputs from the task output\n",
    "def parse_task_output_file(task_num, task_data, predict_e2l, predict_l2e):\n",
    "    e2l = predict_e2l[task_num]\n",
    "    l2e = predict_l2e[task_num]\n",
    "    \n",
    "    prediction_files = sorted(tf.io.gfile.glob(predict_outputs_path[task_num] + \"*\"))\n",
    "    print(f\"\\nPredictions for task {task_num} using checkpoint %s were read!\\n\" % prediction_files[-1].split(\"-\")[-1])\n",
    "    \n",
    "    with tf.io.gfile.GFile(prediction_files[-1]) as f:\n",
    "        predict_raw = [line.strip() for line in f.readlines()]\n",
    "    \n",
    "    predictions = []\n",
    "    for j, line in enumerate(predict_raw):  \n",
    "        \n",
    "        if task_num in [1,2,11]: # Task 1, 2, 11 have two sentinel tokens\n",
    "            result = re.search(r\"<extra_id_0>(.*)<extra_id_1>(.*)\", line)\n",
    "            result1 = re.search(r\"<extra_id_0>(.*)\", line)\n",
    "            result2 = re.search(r\"(.*)<extra_id_1>(.*)\", line)\n",
    "            if result:\n",
    "                out1, out2 = result.group(1).strip(), result.group(2).strip()\n",
    "                if not verify(out_1=out1, out2=out2, task_num=task_num, \n",
    "                              example=task_data[l2e[j]], line_num=j, \n",
    "                              everything=line, after_sentinel_0=None, after_sentinel_1=None):\n",
    "                    out1, out2 = failsafe(task_num=task_num, example=task_data[l2e[j]], \n",
    "                                          line_num=j, everything=line,\n",
    "                                          after_sentinel_0=result1, after_sentinel_1=result2)\n",
    "            else:\n",
    "                out1, out2 = failsafe(task_num=task_num, example=task_data[l2e[j]], \n",
    "                                        line_num=j, everything=line, \n",
    "                                        after_sentinel_0=result1, after_sentinel_1=result2)\n",
    "        else: # Rest of the tasks only produce one output\n",
    "            result = re.search(r\"<extra_id_0>(.*)\", line)\n",
    "            if result:\n",
    "                out1, out2 = result.group(1).strip(), None\n",
    "                if not verify(out_1=out1, out2=out2, task_num=task_num, \n",
    "                              example=task_data[l2e[j]], line_num=j, \n",
    "                              everything=line, after_sentinel_0=None, after_sentinel_1=None):\n",
    "                    out1, out2 = failsafe(task_num=task_num, example=task_data[l2e[j]], \n",
    "                                          line_num=j, everything=line,\n",
    "                                          after_sentinel_0=None, after_sentinel_1=None)\n",
    "                    \n",
    "            else:\n",
    "                out1, out2 = failsafe(task_num=task_num, example=task_data[l2e[j]], \n",
    "                                      line_num=j, everything=line,\n",
    "                                      after_sentinel_0=None, after_sentinel_1=None)\n",
    "        \n",
    "        predictions.append((out1, out2))\n",
    "    \n",
    "    for k, example in enumerate(task_data):\n",
    "        line_nums = e2l.get(k)\n",
    "        if line_nums != None:\n",
    "            if task_num == 1:\n",
    "                line_num = line_nums[0]\n",
    "                if example['a'] in ('yes', 'no'):\n",
    "                    task_data[k]['qtype'] = 'comparison'\n",
    "                elif 'bridge' in predictions[line_num][1]:\n",
    "                    task_data[k]['qtype'] = 'bridge'\n",
    "                elif 'comparison' in predictions[line_num][1]:\n",
    "                    task_data[k]['qtype'] = 'comparison'\n",
    "                else:\n",
    "                    task_data[k]['qtype'] = 'confused'\n",
    "\n",
    "            elif task_num == 2:\n",
    "                line_num = line_nums[0]\n",
    "                bridge_tentative = re.sub('<extra_id_[0-9]{1,2}>', '', (predictions[line_num][1])).strip()\n",
    "                if bridge_tentative != '':\n",
    "                    task_data[k]['b'] = bridge_tentative\n",
    "                else:\n",
    "                    _, task_data[k]['b'] = failsafe(task_num=task_num,\n",
    "                                                    example=example,\n",
    "                                                    line_num=line_num)\n",
    "\n",
    "            elif task_num == 3:\n",
    "                for idx in [0, 1]:\n",
    "                    line_num = line_nums[idx]\n",
    "                    p = predictions[line_num][0]\n",
    "                    if 'present' in p or 'absent' in p:\n",
    "                        if 'present' in p:\n",
    "                            task_data[k][f'a_in_p{idx+1}'] = True\n",
    "                        else:\n",
    "                            task_data[k][f'a_in_p{idx+1}'] = False\n",
    "                    else:\n",
    "                        task_data[k][f'a_in_p{idx+1}'], _ = failsafe(task_num=task_num, \n",
    "                                                                     example=example, \n",
    "                                                                     line_num=line_num)\n",
    "                # failsafe\n",
    "                if not (task_data[k][f'a_in_p1'] == True or task_data[k][f'a_in_p1'] == True):\n",
    "                    task_data[k][f'a_in_p1'] = task_data[k]['a'] in task_data[k]['p1'] \n",
    "                    task_data[k][f'a_in_p2'] = task_data[k]['a'] in task_data[k]['p2'] \n",
    "                # if still fail then force assign\n",
    "                # TODO: can improve with fuzzy matching\n",
    "                if not (task_data[k][f'a_in_p1'] == True or task_data[k][f'a_in_p1'] == True):\n",
    "                    task_data[k][f'a_in_p1'] = True \n",
    "                    task_data[k][f'a_in_p2'] = False\n",
    "            elif task_num == 4:\n",
    "                idx = 0\n",
    "                line_num = line_nums[idx]\n",
    "                p = predictions[line_num][0]\n",
    "                if 'similar' in p or 'dissimilar' in p:\n",
    "                    if 'dissimilar' in p:\n",
    "                        task_data[k][f'b_same_as_a'] = False\n",
    "                    else:\n",
    "                        task_data[k][f'b_same_as_a'] = True\n",
    "                else:\n",
    "                    task_data[k][f'b_same_as_a'] = failsafe(task_num=task_num,\n",
    "                                                            example=example,\n",
    "                                                            line_num=line_num)\n",
    "\n",
    "            elif task_num == 5:\n",
    "                idx = 0\n",
    "                line_num = line_nums[idx]\n",
    "                if task_data[k]['5_p1']:\n",
    "                    task_data[k]['s1'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['p1']\n",
    "                elif task_data[k]['5_p2']:\n",
    "                    task_data[k]['s2'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['p2']\n",
    "\n",
    "            elif task_num == 6:\n",
    "                for line_num in line_nums:\n",
    "                    if '6_p1' in task_data[k].keys():\n",
    "                        if task_data[k]['6_p1'] == line_num:\n",
    "                            # task 5 gets priority to write s1, alternate assertion produced by 6th task is saved in key 's1_'\n",
    "                            if 's1' not in task_data[k].keys():\n",
    "                                task_data[k]['s1'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['p1']\n",
    "                            else:\n",
    "                                task_data[k]['s1_'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['p1']\n",
    "                    if '6_p2' in task_data[k].keys():\n",
    "                        if task_data[k]['6_p2'] == line_num:\n",
    "                            # task 5 gets priority to write s2, alternate assertion produced by 6th task is saved in key 's2_'\n",
    "                            if 's2' not in task_data[k].keys():\n",
    "                                task_data[k]['s2'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['p2']\n",
    "                            else:\n",
    "                                task_data[k]['s2_'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['p2']\n",
    "            elif task_num == 7:\n",
    "                idx = 0\n",
    "                line_num = line_nums[idx]\n",
    "                task_data[k]['c'] = predictions[line_num][0] if predictions[line_num][0] != '' else f\"{task_data[k]['s1']}. {task_data[k]['s2']}\"\n",
    "                    \n",
    "            elif task_num == 8:\n",
    "                idx = 0\n",
    "                line_num = line_nums[idx]\n",
    "                task_data[k]['c-b'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['c'].replace(task_data[k]['b'], '')\n",
    "                \n",
    "            elif task_num == 9:\n",
    "                idx = 0\n",
    "                line_num = line_nums[idx]\n",
    "                task_data[k]['c-a'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['c-b'].replace(task_data[k]['a'], 'certain entity')\n",
    "                \n",
    "            elif task_num == 10:\n",
    "                idx = 0\n",
    "                line_num = line_nums[idx]\n",
    "                task_data[k]['q'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['c-a'].replace('certain', 'what') + ' ?'\n",
    "                \n",
    "            elif task_num == 11:\n",
    "                idx = 0\n",
    "                line_num = line_nums[idx]\n",
    "                task_data[k]['s1_c'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['p1']\n",
    "                task_data[k]['s2_c'] = predictions[line_num][1] if predictions[line_num][1] != '' else task_data[k]['p2']\n",
    "                \n",
    "            elif task_num == 12:\n",
    "                idx = 0\n",
    "                line_num = line_nums[idx]\n",
    "                task_data[k]['c_c'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['s1_c'] + ' ' + task_data[k]['s1_c']\n",
    "                \n",
    "            elif task_num == 13:\n",
    "                idx = 0\n",
    "                line_num = line_nums[idx]\n",
    "                task_data[k]['q_c'] = predictions[line_num][0] if predictions[line_num][0] != '' else task_data[k]['s1_c'] + ' ' + task_data[k]['s1_c']\n",
    "                \n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "935030ac-b190-4043-9e67-341f8e6f3362",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0617 06:22:26.446726 140076299392832 resource_reader.py:50] system_path_file_exists:gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n",
      "E0617 06:22:26.449726 140076299392832 resource_reader.py:55] Path not found: gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': 'gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': graph_options {\n",
      "  rewrite_options {\n",
      "    disable_meta_optimizer: true\n",
      "  }\n",
      "}\n",
      "cluster_def {\n",
      "  job {\n",
      "    name: \"worker\"\n",
      "    tasks {\n",
      "      key: 0\n",
      "      value: \"10.69.237.130:8470\"\n",
      "    }\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({'worker': ['10.69.237.130:8470']}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': 'grpc://10.69.237.130:8470', '_evaluation_master': 'grpc://10.69.237.130:8470', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1, experimental_allow_per_host_v2_parallel_get_next=False, experimental_feed_hook=None), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu.tpu_cluster_resolver.TPUClusterResolver object at 0x7f650e7fe9d0>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "INFO:tensorflow:Querying Tensorflow master (grpc://10.69.237.130:8470) for TPU system metadata.\n",
      "INFO:tensorflow:Initializing TPU system (master: grpc://10.69.237.130:8470) to fetch topology for model parallelism. This might take a while.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-17 06:22:26.872203: W tensorflow/core/distributed_runtime/rpc/grpc_session.cc:374] GrpcSession::ListDevices will initialize the session with an empty graph and other defaults because the session has not yet been created.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Found TPU system:\n",
      "INFO:tensorflow:*** Num TPU Cores: 8\n",
      "INFO:tensorflow:*** Num TPU Workers: 1\n",
      "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 6203174284852103735)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 3141874392558440078)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, -6959113169627862454)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, -3153061807625208789)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 4212197900597066358)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 5149973259308940275)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 1934008111145682056)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 4237587582203713584)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, -7967013709227532154)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, -4177898833670162469)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, -2424661986629548601)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:num_cores_per_replica: 1\n",
      "INFO:tensorflow:computation_shape: [1, 1, 1, 1]\n",
      "INFO:tensorflow:num_replicas: 8\n",
      "INFO:tensorflow:device_assignment.topology.device_coordinates: [[[0 0 0 0]\n",
      "  [0 0 0 1]\n",
      "  [1 0 0 0]\n",
      "  [1 0 0 1]\n",
      "  [0 1 0 0]\n",
      "  [0 1 0 1]\n",
      "  [1 1 0 0]\n",
      "  [1 1 0 1]]]\n",
      "INFO:tensorflow:device_assignment.core_assignment: [[[0 0 0 0]]\n",
      "\n",
      " [[0 0 0 1]]\n",
      "\n",
      " [[1 0 0 0]]\n",
      "\n",
      " [[1 0 0 1]]\n",
      "\n",
      " [[0 1 0 0]]\n",
      "\n",
      " [[0 1 0 1]]\n",
      "\n",
      " [[1 1 0 0]]\n",
      "\n",
      " [[1 1 0 1]]]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_shape=[8] physical_shape=[2, 2, 2]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_to_physical = [(0, 0, 0), (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 1, 0), (1, 1, 1), (1, 0, 0), (1, 0, 1)]\n",
      "WARNING:tensorflow:SimdMeshImpl ignoring devices ['', '', '', '', '', '', '', '']\n",
      "INFO:tensorflow:SimdMeshImpl init: Shape[model=8] LayoutRules{('d_ff', 'model'), ('vocab', 'model'), ('batch', 'batch'), ('ensemble', 'ensemble'), ('experts', 'batch'), ('heads', 'model')}\n",
      "INFO:tensorflow:Device Assignment: <tensorflow.python.tpu.device_assignment.DeviceAssignment object at 0x7f659a5c4e90>\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Create pnum_tensor\n",
      "INFO:tensorflow:Casting <dtype: 'int32'> to float32 for allreduce\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 32899072     slice_size 4112384      Shape[vocab=32128, d_model=1024]                            \n",
      "INFO:tensorflow:Variable stacked/decoder/block_005/layer_000/layer_norm/scale         size 59392        slice_size 59392        Shape[stacked=58, d_model=1024]                             \n",
      "INFO:tensorflow:    decoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/final_layer_norm/scale\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 2048         slice_size 256          Shape[stacked=2, heads=32, buckets=32]                      \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/layer_norm/scale         size 65536        slice_size 65536        Shape[stacked=64, d_model=1024]                             \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/final_layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:Trainable Variables            count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:All Variables                  count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:Counters:\n",
      "allreduce: 2.46e+10\n",
      " allreduce/[0]: 2.46e+10\n",
      "  allreduce/[0]/einsum_op: 2.46e+10\n",
      "einsum: 7.74e+13\n",
      "einsum_unique: 7.74e+13\n",
      "output: 4.83e+11\n",
      " output/AddOperation: 9.69e+10\n",
      " output/BinaryOpWithBroadcasting: 1.94e+09\n",
      " output/Constant: 4.83e+09\n",
      " output/EinsumOperation: 1.62e+11\n",
      " output/ImportOperation: 1.98e+05\n",
      " output/MinMaxOperation: 8.51e+07\n",
      " output/OneHotOperation: 3.39e+09\n",
      " output/RangeOperation: 6.14e+03\n",
      " output/ReduceOperation: 1.37e+08\n",
      " output/ReshapeOperation: 4.61e+10\n",
      " output/ScalarAddOperation: 1.37e+08\n",
      " output/ScalarMultiplyOperation: 2.27e+09\n",
      " output/ShiftOperation: 1.97e+05\n",
      " output/SlicewiseOperation: 1.35e+11\n",
      " output/StackedVariable: 1e+06\n",
      " output/StopGradient: 2.17e+10\n",
      " output/UnstackOperation: 1e+06\n",
      " output/Variable: 2.85e+09\n",
      " output/WhileLoopOperation: 4.83e+09\n",
      "output_unique: 3.49e+11\n",
      " output_unique/AddOperation: 7.57e+10\n",
      " output_unique/BinaryOpWithBroadcasting: 2.42e+08\n",
      " output_unique/Constant: 4.83e+09\n",
      " output_unique/EinsumOperation: 8.88e+10\n",
      " output_unique/ImportOperation: 2.48e+04\n",
      " output_unique/MinMaxOperation: 1.06e+07\n",
      " output_unique/OneHotOperation: 1.81e+09\n",
      " output_unique/RangeOperation: 768\n",
      " output_unique/ReduceOperation: 1.16e+08\n",
      " output_unique/ReshapeOperation: 3.75e+10\n",
      " output_unique/ScalarAddOperation: 1.72e+07\n",
      " output_unique/ScalarMultiplyOperation: 2.84e+08\n",
      " output_unique/ShiftOperation: 2.46e+04\n",
      " output_unique/SlicewiseOperation: 1.1e+11\n",
      " output_unique/StackedVariable: 1.27e+05\n",
      " output_unique/StopGradient: 2.17e+10\n",
      " output_unique/UnstackOperation: 1.27e+05\n",
      " output_unique/Variable: 2.85e+09\n",
      " output_unique/WhileLoopOperation: 4.83e+09\n",
      "variables: 2.85e+09\n",
      " variables/trainable: 2.85e+09\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/model.ckpt-1020544\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Before copy master to slices.\n",
      "INFO:tensorflow:Done with copy master to slices.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Outfeed finished for iteration (0, 0)\n",
      "INFO:tensorflow:decoded 0: Bridge entity: Shirley Temple Assertion 1: Shirley Temple served as Chief of Protocol. Assertion 2: Kiss and Tell is a 1945 American comedy film starring Shirley Temple. Combined: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Shirley Temple served as Chief of Protocol in Kiss and Tell, a 1945 American comedy film starring Shirley Temple.\n",
      "INFO:tensorflow:decoded 1: Bridge entity: Animorphs Assertion 1: The Hork-Bajir Chronicles is the second companion book to the \"Animorphs\" series. Assertion 2: Animorphs is a science fantasy series of young adult books. Combined: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The Hork-Bajir Chronicles is the second companion book to the \"Animorphs\" series which is a science fantasy series of young adult books.\n",
      "INFO:tensorflow:decoded 2: Bridge entity: Winner Assertion 1: 2014 S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label, YG Entertainment. Assertion 2: Winner is a South Korean boy group. Combined: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> 2014 S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label, YG Entertainment.\n",
      "INFO:tensorflow:decoded 4: Bridge entity: Terry Richardson Assertion 1: Annie Morton has been photographed by Terry Richardson. Assertion 2: Terry Richardson is an American fashion and portrait photographer. Combined: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Annie Morton has been photographed by Terry Richardson who is an American fashion and portrait photographer.\n",
      "INFO:tensorflow:decoded 8: Bridge entity: Alex Ferguson Assertion 1: The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football. Assertion 2: Alex Ferguson is a Scottish former football manager and player who managed Manchester United from 1986 to 2013. Combined: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by Scottish former football manager and player Alex Ferguson.\n",
      "INFO:tensorflow:decoded 16: Bridge entity: Lee Hazlewood Assertion 1: These Boots Are Made for Walkin' is a hit song written by Lee Hazlewood. Assertion 2: Lee Hazlewood was an American country and pop singer, record producer. Combined: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Lee Hazlewood was an American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walkin'.\n",
      "INFO:tensorflow:decoded 32: Bridge entity: Mikhail Gorbachev Assertion 1: The World Summit of Nobel Peace Laureates was initiated by Mikhail Gorbachev. Assertion 2: Mikhail Gorbachev was the eighth and last leader of the Soviet Union. Combined: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Mikhail Gorbachev was the eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:decoded 64: Bridge entity: bleh Assertion 1: Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Assertion 2: Conscription in the United States has been employed by the federal government in four conflicts. Combined: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in four conflicts.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0617 06:24:27.513227 140076299392832 resource_reader.py:50] system_path_file_exists:gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n",
      "E0617 06:24:27.515520 140076299392832 resource_reader.py:55] Path not found: gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predictions for task 7 using checkpoint 1020544 were read!\n",
      "\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': graph_options {\n",
      "  rewrite_options {\n",
      "    disable_meta_optimizer: true\n",
      "  }\n",
      "}\n",
      "cluster_def {\n",
      "  job {\n",
      "    name: \"worker\"\n",
      "    tasks {\n",
      "      key: 0\n",
      "      value: \"10.69.237.130:8470\"\n",
      "    }\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({'worker': ['10.69.237.130:8470']}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': 'grpc://10.69.237.130:8470', '_evaluation_master': 'grpc://10.69.237.130:8470', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1, experimental_allow_per_host_v2_parallel_get_next=False, experimental_feed_hook=None), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu.tpu_cluster_resolver.TPUClusterResolver object at 0x7f650e7fe9d0>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "INFO:tensorflow:Querying Tensorflow master (grpc://10.69.237.130:8470) for TPU system metadata.\n",
      "INFO:tensorflow:Initializing TPU system (master: grpc://10.69.237.130:8470) to fetch topology for model parallelism. This might take a while.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-17 06:24:27.755898: W tensorflow/core/distributed_runtime/rpc/grpc_session.cc:374] GrpcSession::ListDevices will initialize the session with an empty graph and other defaults because the session has not yet been created.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Found TPU system:\n",
      "INFO:tensorflow:*** Num TPU Cores: 8\n",
      "INFO:tensorflow:*** Num TPU Workers: 1\n",
      "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 6203174284852103735)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 3141874392558440078)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, -6959113169627862454)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, -3153061807625208789)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 4212197900597066358)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 5149973259308940275)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 1934008111145682056)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 4237587582203713584)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, -7967013709227532154)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, -4177898833670162469)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, -2424661986629548601)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:num_cores_per_replica: 1\n",
      "INFO:tensorflow:computation_shape: [1, 1, 1, 1]\n",
      "INFO:tensorflow:num_replicas: 8\n",
      "INFO:tensorflow:device_assignment.topology.device_coordinates: [[[0 0 0 0]\n",
      "  [0 0 0 1]\n",
      "  [1 0 0 0]\n",
      "  [1 0 0 1]\n",
      "  [0 1 0 0]\n",
      "  [0 1 0 1]\n",
      "  [1 1 0 0]\n",
      "  [1 1 0 1]]]\n",
      "INFO:tensorflow:device_assignment.core_assignment: [[[0 0 0 0]]\n",
      "\n",
      " [[0 0 0 1]]\n",
      "\n",
      " [[1 0 0 0]]\n",
      "\n",
      " [[1 0 0 1]]\n",
      "\n",
      " [[0 1 0 0]]\n",
      "\n",
      " [[0 1 0 1]]\n",
      "\n",
      " [[1 1 0 0]]\n",
      "\n",
      " [[1 1 0 1]]]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_shape=[8] physical_shape=[2, 2, 2]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_to_physical = [(0, 0, 0), (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 1, 0), (1, 1, 1), (1, 0, 0), (1, 0, 1)]\n",
      "WARNING:tensorflow:SimdMeshImpl ignoring devices ['', '', '', '', '', '', '', '']\n",
      "INFO:tensorflow:SimdMeshImpl init: Shape[model=8] LayoutRules{('d_ff', 'model'), ('vocab', 'model'), ('batch', 'batch'), ('ensemble', 'ensemble'), ('experts', 'batch'), ('heads', 'model')}\n",
      "INFO:tensorflow:Device Assignment: <tensorflow.python.tpu.device_assignment.DeviceAssignment object at 0x7f64d17d3050>\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Create pnum_tensor\n",
      "INFO:tensorflow:Casting <dtype: 'int32'> to float32 for allreduce\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 32899072     slice_size 4112384      Shape[vocab=32128, d_model=1024]                            \n",
      "INFO:tensorflow:Variable stacked/decoder/block_005/layer_000/layer_norm/scale         size 59392        slice_size 59392        Shape[stacked=58, d_model=1024]                             \n",
      "INFO:tensorflow:    decoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/final_layer_norm/scale\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 2048         slice_size 256          Shape[stacked=2, heads=32, buckets=32]                      \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/layer_norm/scale         size 65536        slice_size 65536        Shape[stacked=64, d_model=1024]                             \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/final_layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:Trainable Variables            count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:All Variables                  count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:Counters:\n",
      "allreduce: 2.46e+10\n",
      " allreduce/[0]: 2.46e+10\n",
      "  allreduce/[0]/einsum_op: 2.46e+10\n",
      "einsum: 7.74e+13\n",
      "einsum_unique: 7.74e+13\n",
      "output: 4.83e+11\n",
      " output/AddOperation: 9.69e+10\n",
      " output/BinaryOpWithBroadcasting: 1.94e+09\n",
      " output/Constant: 4.83e+09\n",
      " output/EinsumOperation: 1.62e+11\n",
      " output/ImportOperation: 1.98e+05\n",
      " output/MinMaxOperation: 8.51e+07\n",
      " output/OneHotOperation: 3.39e+09\n",
      " output/RangeOperation: 6.14e+03\n",
      " output/ReduceOperation: 1.37e+08\n",
      " output/ReshapeOperation: 4.61e+10\n",
      " output/ScalarAddOperation: 1.37e+08\n",
      " output/ScalarMultiplyOperation: 2.27e+09\n",
      " output/ShiftOperation: 1.97e+05\n",
      " output/SlicewiseOperation: 1.35e+11\n",
      " output/StackedVariable: 1e+06\n",
      " output/StopGradient: 2.17e+10\n",
      " output/UnstackOperation: 1e+06\n",
      " output/Variable: 2.85e+09\n",
      " output/WhileLoopOperation: 4.83e+09\n",
      "output_unique: 3.49e+11\n",
      " output_unique/AddOperation: 7.57e+10\n",
      " output_unique/BinaryOpWithBroadcasting: 2.42e+08\n",
      " output_unique/Constant: 4.83e+09\n",
      " output_unique/EinsumOperation: 8.88e+10\n",
      " output_unique/ImportOperation: 2.48e+04\n",
      " output_unique/MinMaxOperation: 1.06e+07\n",
      " output_unique/OneHotOperation: 1.81e+09\n",
      " output_unique/RangeOperation: 768\n",
      " output_unique/ReduceOperation: 1.16e+08\n",
      " output_unique/ReshapeOperation: 3.75e+10\n",
      " output_unique/ScalarAddOperation: 1.72e+07\n",
      " output_unique/ScalarMultiplyOperation: 2.84e+08\n",
      " output_unique/ShiftOperation: 2.46e+04\n",
      " output_unique/SlicewiseOperation: 1.1e+11\n",
      " output_unique/StackedVariable: 1.27e+05\n",
      " output_unique/StopGradient: 2.17e+10\n",
      " output_unique/UnstackOperation: 1.27e+05\n",
      " output_unique/Variable: 2.85e+09\n",
      " output_unique/WhileLoopOperation: 4.83e+09\n",
      "variables: 2.85e+09\n",
      " variables/trainable: 2.85e+09\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/model.ckpt-1020544\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Before copy master to slices.\n",
      "INFO:tensorflow:Done with copy master to slices.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Outfeed finished for iteration (0, 0)\n",
      "INFO:tensorflow:decoded 0: Removing bridge entity: Shirley Temple from: Shirley Temple served as Chief of Protocol in Kiss and Tell, a 1945 American comedy film starring Shirley Temple. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The Chief of Protocol served in Kiss and Tell, a 1945 American comedy film starring Shirley Temple.\n",
      "INFO:tensorflow:decoded 1: Removing bridge entity: Animorphs from: The Hork-Bajir Chronicles is the second companion book to the \"Animorphs\" series which is a science fantasy series of young adult books. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The Hork-Bajir Chronicles is the second companion book to the series which is a science fantasy series of young adult books.\n",
      "INFO:tensorflow:decoded 2: Removing bridge entity: Winner from: 2014 S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label, YG Entertainment. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> 2014 S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label, YG Entertainment.\n",
      "INFO:tensorflow:decoded 4: Removing bridge entity: Terry Richardson from: Annie Morton has been photographed by Terry Richardson who is an American fashion and portrait photographer. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Annie Morton has been photographed by an American fashion and portrait photographer.\n",
      "INFO:tensorflow:decoded 8: Removing bridge entity: Alex Ferguson from: The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by Scottish former football manager and player Alex Ferguson. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by Scottish former football manager and player.\n",
      "INFO:tensorflow:decoded 16: Removing bridge entity: Lee Hazlewood from: Lee Hazlewood was an American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walkin'. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walkin'.\n",
      "INFO:tensorflow:decoded 32: Removing bridge entity: Mikhail Gorbachev from: Mikhail Gorbachev was the eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:decoded 64: Removing bridge entity: bleh from: Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in four conflicts. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in four conflicts.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0617 06:26:28.742683 140076299392832 resource_reader.py:50] system_path_file_exists:gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n",
      "E0617 06:26:28.744710 140076299392832 resource_reader.py:55] Path not found: gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predictions for task 8 using checkpoint 1020544 were read!\n",
      "\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': graph_options {\n",
      "  rewrite_options {\n",
      "    disable_meta_optimizer: true\n",
      "  }\n",
      "}\n",
      "cluster_def {\n",
      "  job {\n",
      "    name: \"worker\"\n",
      "    tasks {\n",
      "      key: 0\n",
      "      value: \"10.69.237.130:8470\"\n",
      "    }\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({'worker': ['10.69.237.130:8470']}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': 'grpc://10.69.237.130:8470', '_evaluation_master': 'grpc://10.69.237.130:8470', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1, experimental_allow_per_host_v2_parallel_get_next=False, experimental_feed_hook=None), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu.tpu_cluster_resolver.TPUClusterResolver object at 0x7f650e7fe9d0>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "INFO:tensorflow:Querying Tensorflow master (grpc://10.69.237.130:8470) for TPU system metadata.\n",
      "INFO:tensorflow:Initializing TPU system (master: grpc://10.69.237.130:8470) to fetch topology for model parallelism. This might take a while.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-17 06:26:28.981498: W tensorflow/core/distributed_runtime/rpc/grpc_session.cc:374] GrpcSession::ListDevices will initialize the session with an empty graph and other defaults because the session has not yet been created.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Found TPU system:\n",
      "INFO:tensorflow:*** Num TPU Cores: 8\n",
      "INFO:tensorflow:*** Num TPU Workers: 1\n",
      "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 6203174284852103735)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 3141874392558440078)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, -6959113169627862454)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, -3153061807625208789)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 4212197900597066358)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 5149973259308940275)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 1934008111145682056)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 4237587582203713584)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, -7967013709227532154)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, -4177898833670162469)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, -2424661986629548601)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:num_cores_per_replica: 1\n",
      "INFO:tensorflow:computation_shape: [1, 1, 1, 1]\n",
      "INFO:tensorflow:num_replicas: 8\n",
      "INFO:tensorflow:device_assignment.topology.device_coordinates: [[[0 0 0 0]\n",
      "  [0 0 0 1]\n",
      "  [1 0 0 0]\n",
      "  [1 0 0 1]\n",
      "  [0 1 0 0]\n",
      "  [0 1 0 1]\n",
      "  [1 1 0 0]\n",
      "  [1 1 0 1]]]\n",
      "INFO:tensorflow:device_assignment.core_assignment: [[[0 0 0 0]]\n",
      "\n",
      " [[0 0 0 1]]\n",
      "\n",
      " [[1 0 0 0]]\n",
      "\n",
      " [[1 0 0 1]]\n",
      "\n",
      " [[0 1 0 0]]\n",
      "\n",
      " [[0 1 0 1]]\n",
      "\n",
      " [[1 1 0 0]]\n",
      "\n",
      " [[1 1 0 1]]]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_shape=[8] physical_shape=[2, 2, 2]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_to_physical = [(0, 0, 0), (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 1, 0), (1, 1, 1), (1, 0, 0), (1, 0, 1)]\n",
      "WARNING:tensorflow:SimdMeshImpl ignoring devices ['', '', '', '', '', '', '', '']\n",
      "INFO:tensorflow:SimdMeshImpl init: Shape[model=8] LayoutRules{('d_ff', 'model'), ('vocab', 'model'), ('batch', 'batch'), ('ensemble', 'ensemble'), ('experts', 'batch'), ('heads', 'model')}\n",
      "INFO:tensorflow:Device Assignment: <tensorflow.python.tpu.device_assignment.DeviceAssignment object at 0x7f6595682e10>\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Create pnum_tensor\n",
      "INFO:tensorflow:Casting <dtype: 'int32'> to float32 for allreduce\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 32899072     slice_size 4112384      Shape[vocab=32128, d_model=1024]                            \n",
      "INFO:tensorflow:Variable stacked/decoder/block_005/layer_000/layer_norm/scale         size 59392        slice_size 59392        Shape[stacked=58, d_model=1024]                             \n",
      "INFO:tensorflow:    decoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/final_layer_norm/scale\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 2048         slice_size 256          Shape[stacked=2, heads=32, buckets=32]                      \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/layer_norm/scale         size 65536        slice_size 65536        Shape[stacked=64, d_model=1024]                             \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/final_layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:Trainable Variables            count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:All Variables                  count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:Counters:\n",
      "allreduce: 2.46e+10\n",
      " allreduce/[0]: 2.46e+10\n",
      "  allreduce/[0]/einsum_op: 2.46e+10\n",
      "einsum: 7.74e+13\n",
      "einsum_unique: 7.74e+13\n",
      "output: 4.83e+11\n",
      " output/AddOperation: 9.69e+10\n",
      " output/BinaryOpWithBroadcasting: 1.94e+09\n",
      " output/Constant: 4.83e+09\n",
      " output/EinsumOperation: 1.62e+11\n",
      " output/ImportOperation: 1.98e+05\n",
      " output/MinMaxOperation: 8.51e+07\n",
      " output/OneHotOperation: 3.39e+09\n",
      " output/RangeOperation: 6.14e+03\n",
      " output/ReduceOperation: 1.37e+08\n",
      " output/ReshapeOperation: 4.61e+10\n",
      " output/ScalarAddOperation: 1.37e+08\n",
      " output/ScalarMultiplyOperation: 2.27e+09\n",
      " output/ShiftOperation: 1.97e+05\n",
      " output/SlicewiseOperation: 1.35e+11\n",
      " output/StackedVariable: 1e+06\n",
      " output/StopGradient: 2.17e+10\n",
      " output/UnstackOperation: 1e+06\n",
      " output/Variable: 2.85e+09\n",
      " output/WhileLoopOperation: 4.83e+09\n",
      "output_unique: 3.49e+11\n",
      " output_unique/AddOperation: 7.57e+10\n",
      " output_unique/BinaryOpWithBroadcasting: 2.42e+08\n",
      " output_unique/Constant: 4.83e+09\n",
      " output_unique/EinsumOperation: 8.88e+10\n",
      " output_unique/ImportOperation: 2.48e+04\n",
      " output_unique/MinMaxOperation: 1.06e+07\n",
      " output_unique/OneHotOperation: 1.81e+09\n",
      " output_unique/RangeOperation: 768\n",
      " output_unique/ReduceOperation: 1.16e+08\n",
      " output_unique/ReshapeOperation: 3.75e+10\n",
      " output_unique/ScalarAddOperation: 1.72e+07\n",
      " output_unique/ScalarMultiplyOperation: 2.84e+08\n",
      " output_unique/ShiftOperation: 2.46e+04\n",
      " output_unique/SlicewiseOperation: 1.1e+11\n",
      " output_unique/StackedVariable: 1.27e+05\n",
      " output_unique/StopGradient: 2.17e+10\n",
      " output_unique/UnstackOperation: 1.27e+05\n",
      " output_unique/Variable: 2.85e+09\n",
      " output_unique/WhileLoopOperation: 4.83e+09\n",
      "variables: 2.85e+09\n",
      " variables/trainable: 2.85e+09\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/model.ckpt-1020544\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Before copy master to slices.\n",
      "INFO:tensorflow:Done with copy master to slices.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Outfeed finished for iteration (0, 0)\n",
      "INFO:tensorflow:decoded 0: Contract answer entity Chief of Protocol from: The Chief of Protocol served in Kiss and Tell, a 1945 American comedy film starring Shirley Temple. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> A certain person served in Kiss and Tell, a 1945 American comedy film starring Shirley Temple.\n",
      "INFO:tensorflow:decoded 1: Contract answer entity Animorphs from: The Hork-Bajir Chronicles is the second companion book to the series which is a science fantasy series of young adult books. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The Hork-Bajir Chronicles is the second companion book to the series which is a science fantasy series of young adult books.\n",
      "INFO:tensorflow:decoded 2: Contract answer entity YG Entertainment from: 2014 S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label, YG Entertainment. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label.\n",
      "INFO:tensorflow:decoded 4: Contract answer entity Terry Richardson from: Annie Morton has been photographed by an American fashion and portrait photographer. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Annie Morton has been photographed by an American fashion and portrait photographer.\n",
      "INFO:tensorflow:decoded 8: Contract answer entity from 1986 to 2013 from: The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by Scottish former football manager and player. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by Scottish former football manager and player.\n",
      "INFO:tensorflow:decoded 16: Contract answer entity Barton Lee Hazlewood from: The American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walkin'. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walking.\n",
      "INFO:tensorflow:decoded 32: Contract answer entity Organizations could come together to address global issues from: The eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:decoded 64: Contract answer entity Conscription from: Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in four conflicts. We get: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in four conflicts.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0617 06:28:29.648375 140076299392832 resource_reader.py:50] system_path_file_exists:gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n",
      "E0617 06:28:29.650530 140076299392832 resource_reader.py:55] Path not found: gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predictions for task 9 using checkpoint 1020544 were read!\n",
      "\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': graph_options {\n",
      "  rewrite_options {\n",
      "    disable_meta_optimizer: true\n",
      "  }\n",
      "}\n",
      "cluster_def {\n",
      "  job {\n",
      "    name: \"worker\"\n",
      "    tasks {\n",
      "      key: 0\n",
      "      value: \"10.69.237.130:8470\"\n",
      "    }\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({'worker': ['10.69.237.130:8470']}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': 'grpc://10.69.237.130:8470', '_evaluation_master': 'grpc://10.69.237.130:8470', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1, experimental_allow_per_host_v2_parallel_get_next=False, experimental_feed_hook=None), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu.tpu_cluster_resolver.TPUClusterResolver object at 0x7f650e7fe9d0>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "INFO:tensorflow:Querying Tensorflow master (grpc://10.69.237.130:8470) for TPU system metadata.\n",
      "INFO:tensorflow:Initializing TPU system (master: grpc://10.69.237.130:8470) to fetch topology for model parallelism. This might take a while.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-17 06:28:29.911565: W tensorflow/core/distributed_runtime/rpc/grpc_session.cc:374] GrpcSession::ListDevices will initialize the session with an empty graph and other defaults because the session has not yet been created.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Found TPU system:\n",
      "INFO:tensorflow:*** Num TPU Cores: 8\n",
      "INFO:tensorflow:*** Num TPU Workers: 1\n",
      "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 6203174284852103735)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 3141874392558440078)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, -6959113169627862454)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, -3153061807625208789)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 4212197900597066358)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 5149973259308940275)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 1934008111145682056)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 4237587582203713584)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, -7967013709227532154)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, -4177898833670162469)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, -2424661986629548601)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:num_cores_per_replica: 1\n",
      "INFO:tensorflow:computation_shape: [1, 1, 1, 1]\n",
      "INFO:tensorflow:num_replicas: 8\n",
      "INFO:tensorflow:device_assignment.topology.device_coordinates: [[[0 0 0 0]\n",
      "  [0 0 0 1]\n",
      "  [1 0 0 0]\n",
      "  [1 0 0 1]\n",
      "  [0 1 0 0]\n",
      "  [0 1 0 1]\n",
      "  [1 1 0 0]\n",
      "  [1 1 0 1]]]\n",
      "INFO:tensorflow:device_assignment.core_assignment: [[[0 0 0 0]]\n",
      "\n",
      " [[0 0 0 1]]\n",
      "\n",
      " [[1 0 0 0]]\n",
      "\n",
      " [[1 0 0 1]]\n",
      "\n",
      " [[0 1 0 0]]\n",
      "\n",
      " [[0 1 0 1]]\n",
      "\n",
      " [[1 1 0 0]]\n",
      "\n",
      " [[1 1 0 1]]]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_shape=[8] physical_shape=[2, 2, 2]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_to_physical = [(0, 0, 0), (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 1, 0), (1, 1, 1), (1, 0, 0), (1, 0, 1)]\n",
      "WARNING:tensorflow:SimdMeshImpl ignoring devices ['', '', '', '', '', '', '', '']\n",
      "INFO:tensorflow:SimdMeshImpl init: Shape[model=8] LayoutRules{('d_ff', 'model'), ('vocab', 'model'), ('batch', 'batch'), ('ensemble', 'ensemble'), ('experts', 'batch'), ('heads', 'model')}\n",
      "INFO:tensorflow:Device Assignment: <tensorflow.python.tpu.device_assignment.DeviceAssignment object at 0x7f64d32a1050>\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Create pnum_tensor\n",
      "INFO:tensorflow:Casting <dtype: 'int32'> to float32 for allreduce\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 32899072     slice_size 4112384      Shape[vocab=32128, d_model=1024]                            \n",
      "INFO:tensorflow:Variable stacked/decoder/block_005/layer_000/layer_norm/scale         size 59392        slice_size 59392        Shape[stacked=58, d_model=1024]                             \n",
      "INFO:tensorflow:    decoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/final_layer_norm/scale\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 2048         slice_size 256          Shape[stacked=2, heads=32, buckets=32]                      \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/layer_norm/scale         size 65536        slice_size 65536        Shape[stacked=64, d_model=1024]                             \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/final_layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:Trainable Variables            count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:All Variables                  count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:Counters:\n",
      "allreduce: 2.46e+10\n",
      " allreduce/[0]: 2.46e+10\n",
      "  allreduce/[0]/einsum_op: 2.46e+10\n",
      "einsum: 7.74e+13\n",
      "einsum_unique: 7.74e+13\n",
      "output: 4.83e+11\n",
      " output/AddOperation: 9.69e+10\n",
      " output/BinaryOpWithBroadcasting: 1.94e+09\n",
      " output/Constant: 4.83e+09\n",
      " output/EinsumOperation: 1.62e+11\n",
      " output/ImportOperation: 1.98e+05\n",
      " output/MinMaxOperation: 8.51e+07\n",
      " output/OneHotOperation: 3.39e+09\n",
      " output/RangeOperation: 6.14e+03\n",
      " output/ReduceOperation: 1.37e+08\n",
      " output/ReshapeOperation: 4.61e+10\n",
      " output/ScalarAddOperation: 1.37e+08\n",
      " output/ScalarMultiplyOperation: 2.27e+09\n",
      " output/ShiftOperation: 1.97e+05\n",
      " output/SlicewiseOperation: 1.35e+11\n",
      " output/StackedVariable: 1e+06\n",
      " output/StopGradient: 2.17e+10\n",
      " output/UnstackOperation: 1e+06\n",
      " output/Variable: 2.85e+09\n",
      " output/WhileLoopOperation: 4.83e+09\n",
      "output_unique: 3.49e+11\n",
      " output_unique/AddOperation: 7.57e+10\n",
      " output_unique/BinaryOpWithBroadcasting: 2.42e+08\n",
      " output_unique/Constant: 4.83e+09\n",
      " output_unique/EinsumOperation: 8.88e+10\n",
      " output_unique/ImportOperation: 2.48e+04\n",
      " output_unique/MinMaxOperation: 1.06e+07\n",
      " output_unique/OneHotOperation: 1.81e+09\n",
      " output_unique/RangeOperation: 768\n",
      " output_unique/ReduceOperation: 1.16e+08\n",
      " output_unique/ReshapeOperation: 3.75e+10\n",
      " output_unique/ScalarAddOperation: 1.72e+07\n",
      " output_unique/ScalarMultiplyOperation: 2.84e+08\n",
      " output_unique/ShiftOperation: 2.46e+04\n",
      " output_unique/SlicewiseOperation: 1.1e+11\n",
      " output_unique/StackedVariable: 1.27e+05\n",
      " output_unique/StopGradient: 2.17e+10\n",
      " output_unique/UnstackOperation: 1.27e+05\n",
      " output_unique/Variable: 2.85e+09\n",
      " output_unique/WhileLoopOperation: 4.83e+09\n",
      "variables: 2.85e+09\n",
      " variables/trainable: 2.85e+09\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/model.ckpt-1020544\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Before copy master to slices.\n",
      "INFO:tensorflow:Done with copy master to slices.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Outfeed finished for iteration (0, 0)\n",
      "INFO:tensorflow:decoded 0: Turn: A certain person served in Kiss and Tell, a 1945 American comedy film starring Shirley Temple. into question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Who served in Kiss and Tell, a 1945 American comedy film starring Shirley Temple?\n",
      "INFO:tensorflow:decoded 1: Turn: The Hork-Bajir Chronicles is the second companion book to the series which is a science fantasy series of young adult books. into question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The Hork-Bajir Chronicles is the second companion book to a series which is a science fantasy series of young adult books?\n",
      "INFO:tensorflow:decoded 2: Turn: S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label. into question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> S/S is the debut album of South Korean group WINNER released on what date?\n",
      "INFO:tensorflow:decoded 4: Turn: Annie Morton has been photographed by an American fashion and portrait photographer. into question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Who is the photographer who photographed Annie Morton?\n",
      "INFO:tensorflow:decoded 8: Turn: The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by Scottish former football manager and player. into question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by which Scottish former football manager and player?\n",
      "INFO:tensorflow:decoded 16: Turn: The American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walking. into question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Who is the American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walking?\n",
      "INFO:tensorflow:decoded 32: Turn: The eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates. into question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Who was the eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates?\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:decoded 64: Turn: Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in four conflicts. into question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in how many conflicts?\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0617 06:30:31.224392 140076299392832 resource_reader.py:50] system_path_file_exists:gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n",
      "E0617 06:30:31.226639 140076299392832 resource_reader.py:55] Path not found: gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predictions for task 10 using checkpoint 1020544 were read!\n",
      "\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': graph_options {\n",
      "  rewrite_options {\n",
      "    disable_meta_optimizer: true\n",
      "  }\n",
      "}\n",
      "cluster_def {\n",
      "  job {\n",
      "    name: \"worker\"\n",
      "    tasks {\n",
      "      key: 0\n",
      "      value: \"10.69.237.130:8470\"\n",
      "    }\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({'worker': ['10.69.237.130:8470']}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': 'grpc://10.69.237.130:8470', '_evaluation_master': 'grpc://10.69.237.130:8470', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1, experimental_allow_per_host_v2_parallel_get_next=False, experimental_feed_hook=None), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu.tpu_cluster_resolver.TPUClusterResolver object at 0x7f650e7fe9d0>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "INFO:tensorflow:Querying Tensorflow master (grpc://10.69.237.130:8470) for TPU system metadata.\n",
      "INFO:tensorflow:Initializing TPU system (master: grpc://10.69.237.130:8470) to fetch topology for model parallelism. This might take a while.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-17 06:30:31.467048: W tensorflow/core/distributed_runtime/rpc/grpc_session.cc:374] GrpcSession::ListDevices will initialize the session with an empty graph and other defaults because the session has not yet been created.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Found TPU system:\n",
      "INFO:tensorflow:*** Num TPU Cores: 8\n",
      "INFO:tensorflow:*** Num TPU Workers: 1\n",
      "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 6203174284852103735)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 3141874392558440078)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, -6959113169627862454)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, -3153061807625208789)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 4212197900597066358)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 5149973259308940275)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 1934008111145682056)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 4237587582203713584)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, -7967013709227532154)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, -4177898833670162469)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, -2424661986629548601)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:num_cores_per_replica: 1\n",
      "INFO:tensorflow:computation_shape: [1, 1, 1, 1]\n",
      "INFO:tensorflow:num_replicas: 8\n",
      "INFO:tensorflow:device_assignment.topology.device_coordinates: [[[0 0 0 0]\n",
      "  [0 0 0 1]\n",
      "  [1 0 0 0]\n",
      "  [1 0 0 1]\n",
      "  [0 1 0 0]\n",
      "  [0 1 0 1]\n",
      "  [1 1 0 0]\n",
      "  [1 1 0 1]]]\n",
      "INFO:tensorflow:device_assignment.core_assignment: [[[0 0 0 0]]\n",
      "\n",
      " [[0 0 0 1]]\n",
      "\n",
      " [[1 0 0 0]]\n",
      "\n",
      " [[1 0 0 1]]\n",
      "\n",
      " [[0 1 0 0]]\n",
      "\n",
      " [[0 1 0 1]]\n",
      "\n",
      " [[1 1 0 0]]\n",
      "\n",
      " [[1 1 0 1]]]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_shape=[8] physical_shape=[2, 2, 2]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_to_physical = [(0, 0, 0), (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 1, 0), (1, 1, 1), (1, 0, 0), (1, 0, 1)]\n",
      "WARNING:tensorflow:SimdMeshImpl ignoring devices ['', '', '', '', '', '', '', '']\n",
      "INFO:tensorflow:SimdMeshImpl init: Shape[model=8] LayoutRules{('d_ff', 'model'), ('vocab', 'model'), ('batch', 'batch'), ('ensemble', 'ensemble'), ('experts', 'batch'), ('heads', 'model')}\n",
      "INFO:tensorflow:Device Assignment: <tensorflow.python.tpu.device_assignment.DeviceAssignment object at 0x7f64ecb5de10>\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Create pnum_tensor\n",
      "INFO:tensorflow:Casting <dtype: 'int32'> to float32 for allreduce\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 32899072     slice_size 4112384      Shape[vocab=32128, d_model=1024]                            \n",
      "INFO:tensorflow:Variable stacked/decoder/block_005/layer_000/layer_norm/scale         size 59392        slice_size 59392        Shape[stacked=58, d_model=1024]                             \n",
      "INFO:tensorflow:    decoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/final_layer_norm/scale\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 2048         slice_size 256          Shape[stacked=2, heads=32, buckets=32]                      \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/layer_norm/scale         size 65536        slice_size 65536        Shape[stacked=64, d_model=1024]                             \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/final_layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:Trainable Variables            count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:All Variables                  count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:Counters:\n",
      "allreduce: 2.46e+10\n",
      " allreduce/[0]: 2.46e+10\n",
      "  allreduce/[0]/einsum_op: 2.46e+10\n",
      "einsum: 7.74e+13\n",
      "einsum_unique: 7.74e+13\n",
      "output: 4.83e+11\n",
      " output/AddOperation: 9.69e+10\n",
      " output/BinaryOpWithBroadcasting: 1.94e+09\n",
      " output/Constant: 4.83e+09\n",
      " output/EinsumOperation: 1.62e+11\n",
      " output/ImportOperation: 1.98e+05\n",
      " output/MinMaxOperation: 8.51e+07\n",
      " output/OneHotOperation: 3.39e+09\n",
      " output/RangeOperation: 6.14e+03\n",
      " output/ReduceOperation: 1.37e+08\n",
      " output/ReshapeOperation: 4.61e+10\n",
      " output/ScalarAddOperation: 1.37e+08\n",
      " output/ScalarMultiplyOperation: 2.27e+09\n",
      " output/ShiftOperation: 1.97e+05\n",
      " output/SlicewiseOperation: 1.35e+11\n",
      " output/StackedVariable: 1e+06\n",
      " output/StopGradient: 2.17e+10\n",
      " output/UnstackOperation: 1e+06\n",
      " output/Variable: 2.85e+09\n",
      " output/WhileLoopOperation: 4.83e+09\n",
      "output_unique: 3.49e+11\n",
      " output_unique/AddOperation: 7.57e+10\n",
      " output_unique/BinaryOpWithBroadcasting: 2.42e+08\n",
      " output_unique/Constant: 4.83e+09\n",
      " output_unique/EinsumOperation: 8.88e+10\n",
      " output_unique/ImportOperation: 2.48e+04\n",
      " output_unique/MinMaxOperation: 1.06e+07\n",
      " output_unique/OneHotOperation: 1.81e+09\n",
      " output_unique/RangeOperation: 768\n",
      " output_unique/ReduceOperation: 1.16e+08\n",
      " output_unique/ReshapeOperation: 3.75e+10\n",
      " output_unique/ScalarAddOperation: 1.72e+07\n",
      " output_unique/ScalarMultiplyOperation: 2.84e+08\n",
      " output_unique/ShiftOperation: 2.46e+04\n",
      " output_unique/SlicewiseOperation: 1.1e+11\n",
      " output_unique/StackedVariable: 1.27e+05\n",
      " output_unique/StopGradient: 2.17e+10\n",
      " output_unique/UnstackOperation: 1.27e+05\n",
      " output_unique/Variable: 2.85e+09\n",
      " output_unique/WhileLoopOperation: 4.83e+09\n",
      "variables: 2.85e+09\n",
      " variables/trainable: 2.85e+09\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/model.ckpt-1020544\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Before copy master to slices.\n",
      "INFO:tensorflow:Done with copy master to slices.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Outfeed finished for iteration (0, 0)\n",
      "INFO:tensorflow:decoded 0: Context 1: Scott Derrickson - Scott Derrickson (born July 16, 1966) is an American director, screenwriter and producer. He lives in Los Angeles, California. He is best known for directing horror films such as \"Sinister\", \"The Exorcism of Emily Rose\", and \"Deliver Us From Evil\", as well as the 2016 Marvel Cinematic Universe installment, \"Doctor Strange.\" Context 2: Ed Wood - Edward Davis Wood Jr. (October 10, 1924 – December 10, 1978) was an American filmmaker, actor, writer, producer, and director. Answer: yes Assertion from Context 1: <extra_id_0> Assertion from Context 2: <extra_id_1>\n",
      "INFO:tensorflow:            -> <extra_id_0> Scott Derrickson is an American director, screenwriter and producer. <extra_id_1> Ed Wood was an American filmmaker, actor, writer, producer, and director.\n",
      "INFO:tensorflow:decoded 1: Context 1: Laleli Mosque - The Laleli Mosque (Turkish- \"Laleli Camii, or Tulip Mosque\" ) is an 18th-century Ottoman imperial mosque located in Laleli, Fatih, Istanbul, Turkey. Context 2: Esma Sultan Mansion - The Esma Sultan Mansion (Turkish- \"Esma Sultan Yal ⁇ s ⁇ \" ), a historical yal ⁇  (English- waterside mansion ) located at Bosphorus in Ortaköy neighborhood of Istanbul, Turkey and named after its original owner Esma Sultan, is used today as a cultural center after being redeveloped. Answer: no Assertion from Context 1: <extra_id_0> Assertion from Context 2: <extra_id_1>\n",
      "INFO:tensorflow:            -> <extra_id_0> The Laleli Mosque is an 18th-century Ottoman imperial mosque located in Istanbul, Turkey. <extra_id_1> The Esma Sultan Mansion is a historical yal ⁇ \n",
      "INFO:tensorflow:decoded 2: Context 1: Local H - Local H is an American rock band originally formed by guitarist and vocalist Scott Lucas, bassist Matt Garcia, drummer Joe Daniels, and lead guitarist John Sparkman in Zion, Illinois in 1987. The members all met in high school in 1987 and founded Local H three years later. Context 2: For Against - For Against is a United States post-punk/dream pop band from Lincoln, Nebraska. Despite numerous lineup shuffles and some periods of dormancy, the band has produced material steadily from its 1984 inception to the present. Answer: yes Assertion from Context 1: <extra_id_0> Assertion from Context 2: <extra_id_1>\n",
      "INFO:tensorflow:            -> <extra_id_0> Local H is an American rock band. <extra_id_1> For Against is a post-punk/dream pop band.\n",
      "INFO:tensorflow:decoded 4: Context 1: Giuseppe Verdi - Giuseppe Fortunino Francesco Verdi (] ; 9 or 10 October 1813 – 27 January 1901) was an Italian opera composer. Context 2: Ambroise Thomas - Charles Louis Ambroise Thomas (5 August 1811 – 12 February 1896) was a French composer, best known for his operas \"Mignon\" (1866) and \"Hamlet\" (1868, after Shakespeare) and as Director of the Conservatoire de Paris from 1871 till his death. Answer: yes Assertion from Context 1: <extra_id_0> Assertion from Context 2: <extra_id_1>\n",
      "INFO:tensorflow:            -> <extra_id_0> Giuseppe Verdi was an Italian opera composer. <extra_id_1> Ambroise Thomas was a French composer.\n",
      "INFO:tensorflow:decoded 8: Context 1: In the Realm of the Hackers - In The Realm of the Hackers is a 2003 Australian documentary directed by Kevin Anderson about the prominent hacker community, centered in Melbourne, Australia in the late 1980s until early 1990. The storyline is centered on the Australian teenagers going by the hacker names \"Electron\" and \"Phoenix\", who were members of an elite computer hacking group called 'The Realm' and hacked into some of the most secure computer networks in the world, including those of the US Naval Research Laboratory, Lawrence Livermore National Laboratory, a government lab charged with the security of the US nuclear stockpile, and NASA. The film runs for 55 minutes and was inspired by the book \"Underground\", by Melbourne-based writer and academic Suelette Dreyfus. Context 2: Freakonomics (film) - Freakonomics- The Movie is a 2010 American documentary film based on the 2005 book \"Freakonomics- A Rogue Economist Explores the Hidden Side of Everything\" by economist Steven D. Levitt and writer Stephen J. Dubner. The film premiered at the Tribeca Film Festival in April 2010 with a theatrical release later in the year. The film received a lukewarm reception and a 65% fresh rating at Rottentomatoes. Answer: no Assertion from Context 1: <extra_id_0> Assertion from Context 2: <extra_id_1>\n",
      "INFO:tensorflow:            -> <extra_id_0> In the Realm of the Hackers is an Australian documentary. <extra_id_1> Freakonomics is an American documentary.\n",
      "INFO:tensorflow:decoded 16: Context 1: Ajuga - Ajuga , also known as bugleweed, ground pine, carpet bugle, or just bugle, is a genus of 40 species annual and perennial herbaceous flowering plantsin the mint family Lamiaceae, with most species native to Europe, Asia, and Africa, but also two species in southeastern Australia. They grow to 5–50 cm tall, with opposite leaves. Context 2: Cypress - Cypress is a conifer tree or shrub of northern temperate regions that belongs to the family Cupressaceae. The word \"cypress\" is derived from Old French \"cipres\", which was imported from Latin \"cypressus\", the latinisation of the Greek  ⁇  (\"kyparissos\"). Answer: no Assertion from Context 1: <extra_id_0> Assertion from Context 2: <extra_id_1>\n",
      "INFO:tensorflow:            -> <extra_id_0> Ajuga is a genus of 40 species annual and perennial herbaceous flowering plants. <extra_id_1> Cypress is a conifer tree or shrub of northern temperate climates.\n",
      "INFO:tensorflow:decoded 32: Context 1: Scott Derrickson - Scott Derrickson (born July 16, 1966) is an American director, screenwriter and producer. He lives in Los Angeles, California. He is best known for directing horror films such as \"Sinister\", \"The Exorcism of Emily Rose\", and \"Deliver Us From Evil\", as well as the 2016 Marvel Cinematic Universe installment, \"Doctor Strange.\" Context 2: Ed Wood - Edward Davis Wood Jr. (October 10, 1924 – December 10, 1978) was an American filmmaker, actor, writer, producer, and director. Answer: yes Assertion from Context 1: <extra_id_0> Assertion from Context 2: <extra_id_1>\n",
      "INFO:tensorflow:            -> <extra_id_0> Scott Derrickson is an American director, screenwriter and producer. <extra_id_1> Ed Wood was an American filmmaker, actor, writer, producer, and director.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0617 06:32:31.088376 140076299392832 resource_reader.py:50] system_path_file_exists:gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n",
      "E0617 06:32:31.090622 140076299392832 resource_reader.py:55] Path not found: gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predictions for task 11 using checkpoint 1020544 were read!\n",
      "\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': graph_options {\n",
      "  rewrite_options {\n",
      "    disable_meta_optimizer: true\n",
      "  }\n",
      "}\n",
      "cluster_def {\n",
      "  job {\n",
      "    name: \"worker\"\n",
      "    tasks {\n",
      "      key: 0\n",
      "      value: \"10.69.237.130:8470\"\n",
      "    }\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({'worker': ['10.69.237.130:8470']}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': 'grpc://10.69.237.130:8470', '_evaluation_master': 'grpc://10.69.237.130:8470', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1, experimental_allow_per_host_v2_parallel_get_next=False, experimental_feed_hook=None), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu.tpu_cluster_resolver.TPUClusterResolver object at 0x7f650e7fe9d0>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "INFO:tensorflow:Querying Tensorflow master (grpc://10.69.237.130:8470) for TPU system metadata.\n",
      "INFO:tensorflow:Initializing TPU system (master: grpc://10.69.237.130:8470) to fetch topology for model parallelism. This might take a while.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-17 06:32:31.337023: W tensorflow/core/distributed_runtime/rpc/grpc_session.cc:374] GrpcSession::ListDevices will initialize the session with an empty graph and other defaults because the session has not yet been created.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Found TPU system:\n",
      "INFO:tensorflow:*** Num TPU Cores: 8\n",
      "INFO:tensorflow:*** Num TPU Workers: 1\n",
      "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 6203174284852103735)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 3141874392558440078)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, -6959113169627862454)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, -3153061807625208789)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 4212197900597066358)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 5149973259308940275)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 1934008111145682056)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 4237587582203713584)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, -7967013709227532154)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, -4177898833670162469)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, -2424661986629548601)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:num_cores_per_replica: 1\n",
      "INFO:tensorflow:computation_shape: [1, 1, 1, 1]\n",
      "INFO:tensorflow:num_replicas: 8\n",
      "INFO:tensorflow:device_assignment.topology.device_coordinates: [[[0 0 0 0]\n",
      "  [0 0 0 1]\n",
      "  [1 0 0 0]\n",
      "  [1 0 0 1]\n",
      "  [0 1 0 0]\n",
      "  [0 1 0 1]\n",
      "  [1 1 0 0]\n",
      "  [1 1 0 1]]]\n",
      "INFO:tensorflow:device_assignment.core_assignment: [[[0 0 0 0]]\n",
      "\n",
      " [[0 0 0 1]]\n",
      "\n",
      " [[1 0 0 0]]\n",
      "\n",
      " [[1 0 0 1]]\n",
      "\n",
      " [[0 1 0 0]]\n",
      "\n",
      " [[0 1 0 1]]\n",
      "\n",
      " [[1 1 0 0]]\n",
      "\n",
      " [[1 1 0 1]]]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_shape=[8] physical_shape=[2, 2, 2]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_to_physical = [(0, 0, 0), (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 1, 0), (1, 1, 1), (1, 0, 0), (1, 0, 1)]\n",
      "WARNING:tensorflow:SimdMeshImpl ignoring devices ['', '', '', '', '', '', '', '']\n",
      "INFO:tensorflow:SimdMeshImpl init: Shape[model=8] LayoutRules{('d_ff', 'model'), ('vocab', 'model'), ('batch', 'batch'), ('ensemble', 'ensemble'), ('experts', 'batch'), ('heads', 'model')}\n",
      "INFO:tensorflow:Device Assignment: <tensorflow.python.tpu.device_assignment.DeviceAssignment object at 0x7f64d2f63e50>\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Create pnum_tensor\n",
      "INFO:tensorflow:Casting <dtype: 'int32'> to float32 for allreduce\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 32899072     slice_size 4112384      Shape[vocab=32128, d_model=1024]                            \n",
      "INFO:tensorflow:Variable stacked/decoder/block_005/layer_000/layer_norm/scale         size 59392        slice_size 59392        Shape[stacked=58, d_model=1024]                             \n",
      "INFO:tensorflow:    decoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/final_layer_norm/scale\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 2048         slice_size 256          Shape[stacked=2, heads=32, buckets=32]                      \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/layer_norm/scale         size 65536        slice_size 65536        Shape[stacked=64, d_model=1024]                             \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/final_layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:Trainable Variables            count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:All Variables                  count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:Counters:\n",
      "allreduce: 2.46e+10\n",
      " allreduce/[0]: 2.46e+10\n",
      "  allreduce/[0]/einsum_op: 2.46e+10\n",
      "einsum: 7.74e+13\n",
      "einsum_unique: 7.74e+13\n",
      "output: 4.83e+11\n",
      " output/AddOperation: 9.69e+10\n",
      " output/BinaryOpWithBroadcasting: 1.94e+09\n",
      " output/Constant: 4.83e+09\n",
      " output/EinsumOperation: 1.62e+11\n",
      " output/ImportOperation: 1.98e+05\n",
      " output/MinMaxOperation: 8.51e+07\n",
      " output/OneHotOperation: 3.39e+09\n",
      " output/RangeOperation: 6.14e+03\n",
      " output/ReduceOperation: 1.37e+08\n",
      " output/ReshapeOperation: 4.61e+10\n",
      " output/ScalarAddOperation: 1.37e+08\n",
      " output/ScalarMultiplyOperation: 2.27e+09\n",
      " output/ShiftOperation: 1.97e+05\n",
      " output/SlicewiseOperation: 1.35e+11\n",
      " output/StackedVariable: 1e+06\n",
      " output/StopGradient: 2.17e+10\n",
      " output/UnstackOperation: 1e+06\n",
      " output/Variable: 2.85e+09\n",
      " output/WhileLoopOperation: 4.83e+09\n",
      "output_unique: 3.49e+11\n",
      " output_unique/AddOperation: 7.57e+10\n",
      " output_unique/BinaryOpWithBroadcasting: 2.42e+08\n",
      " output_unique/Constant: 4.83e+09\n",
      " output_unique/EinsumOperation: 8.88e+10\n",
      " output_unique/ImportOperation: 2.48e+04\n",
      " output_unique/MinMaxOperation: 1.06e+07\n",
      " output_unique/OneHotOperation: 1.81e+09\n",
      " output_unique/RangeOperation: 768\n",
      " output_unique/ReduceOperation: 1.16e+08\n",
      " output_unique/ReshapeOperation: 3.75e+10\n",
      " output_unique/ScalarAddOperation: 1.72e+07\n",
      " output_unique/ScalarMultiplyOperation: 2.84e+08\n",
      " output_unique/ShiftOperation: 2.46e+04\n",
      " output_unique/SlicewiseOperation: 1.1e+11\n",
      " output_unique/StackedVariable: 1.27e+05\n",
      " output_unique/StopGradient: 2.17e+10\n",
      " output_unique/UnstackOperation: 1.27e+05\n",
      " output_unique/Variable: 2.85e+09\n",
      " output_unique/WhileLoopOperation: 4.83e+09\n",
      "variables: 2.85e+09\n",
      " variables/trainable: 2.85e+09\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/model.ckpt-1020544\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Before copy master to slices.\n",
      "INFO:tensorflow:Done with copy master to slices.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Outfeed finished for iteration (0, 0)\n",
      "INFO:tensorflow:decoded 0: Assertion 1: Scott Derrickson is an American director, screenwriter and producer. Assertion 2: Ed Wood was an American filmmaker, actor, writer, producer, and director. Combine, compare and think: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Both Scott Derrickson and Ed Wood were American filmmakers, actors, writers, and directors.\n",
      "INFO:tensorflow:decoded 1: Assertion 1: The Laleli Mosque is an 18th-century Ottoman imperial mosque located in Istanbul, Turkey. Assertion 2: The Esma Sultan Mansion is a historical yal ?? Combine, compare and think: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> The Laleli Mosque and the Esma Sultan Mansion are both historical buildings.\n",
      "INFO:tensorflow:decoded 2: Assertion 1: Local H is an American rock band. Assertion 2: For Against is a post-punk/dream pop band. Combine, compare and think: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Local H and For Against are both bands.\n",
      "INFO:tensorflow:decoded 4: Assertion 1: Giuseppe Verdi was an Italian opera composer. Assertion 2: Ambroise Thomas was a French composer. Combine, compare and think: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Ambroise Thomas was a French composer and Giuseppe Verdi was an Italian opera composer.\n",
      "INFO:tensorflow:decoded 8: Assertion 1: In the Realm of the Hackers is an Australian documentary. Assertion 2: Freakonomics is an American documentary. Combine, compare and think: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> In the Realm of the Hackers is an Australian documentary while Freakonomics is an American documentary.\n",
      "INFO:tensorflow:decoded 16: Assertion 1: Ajuga is a genus of 40 species annual and perennial herbaceous flowering plants. Assertion 2: Cypress is a conifer tree or shrub of northern temperate climates. Combine, compare and think: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Ajuga and Cypress are both types of plant.\n",
      "INFO:tensorflow:decoded 32: Assertion 1: Scott Derrickson is an American director, screenwriter and producer. Assertion 2: Ed Wood was an American filmmaker, actor, writer, producer, and director. Combine, compare and think: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Both Scott Derrickson and Ed Wood were American filmmakers, actors, writers, and directors.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0617 06:34:30.692345 140076299392832 resource_reader.py:50] system_path_file_exists:gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n",
      "E0617 06:34:30.694514 140076299392832 resource_reader.py:55] Path not found: gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/operative_config.gin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predictions for task 12 using checkpoint 1020544 were read!\n",
      "\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 1000, '_save_checkpoints_secs': None, '_session_config': graph_options {\n",
      "  rewrite_options {\n",
      "    disable_meta_optimizer: true\n",
      "  }\n",
      "}\n",
      "cluster_def {\n",
      "  job {\n",
      "    name: \"worker\"\n",
      "    tasks {\n",
      "      key: 0\n",
      "      value: \"10.69.237.130:8470\"\n",
      "    }\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_checkpoint_save_graph_def': True, '_service': None, '_cluster_spec': ClusterSpec({'worker': ['10.69.237.130:8470']}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': 'grpc://10.69.237.130:8470', '_evaluation_master': 'grpc://10.69.237.130:8470', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=100, num_shards=None, num_cores_per_replica=1, per_host_input_for_training=4, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None, eval_training_input_configuration=2, experimental_host_call_every_n_steps=1, experimental_allow_per_host_v2_parallel_get_next=False, experimental_feed_hook=None), '_cluster': <tensorflow.python.distribute.cluster_resolver.tpu.tpu_cluster_resolver.TPUClusterResolver object at 0x7f650e7fe9d0>}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "INFO:tensorflow:Querying Tensorflow master (grpc://10.69.237.130:8470) for TPU system metadata.\n",
      "INFO:tensorflow:Initializing TPU system (master: grpc://10.69.237.130:8470) to fetch topology for model parallelism. This might take a while.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-17 06:34:30.917165: W tensorflow/core/distributed_runtime/rpc/grpc_session.cc:374] GrpcSession::ListDevices will initialize the session with an empty graph and other defaults because the session has not yet been created.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Found TPU system:\n",
      "INFO:tensorflow:*** Num TPU Cores: 8\n",
      "INFO:tensorflow:*** Num TPU Workers: 1\n",
      "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, -1, 6203174284852103735)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 17179869184, 3141874392558440078)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 17179869184, -6959113169627862454)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 17179869184, -3153061807625208789)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 17179869184, 4212197900597066358)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 17179869184, 5149973259308940275)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 17179869184, 1934008111145682056)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 17179869184, 4237587582203713584)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 17179869184, -7967013709227532154)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 17179869184, -4177898833670162469)\n",
      "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 17179869184, -2424661986629548601)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:num_cores_per_replica: 1\n",
      "INFO:tensorflow:computation_shape: [1, 1, 1, 1]\n",
      "INFO:tensorflow:num_replicas: 8\n",
      "INFO:tensorflow:device_assignment.topology.device_coordinates: [[[0 0 0 0]\n",
      "  [0 0 0 1]\n",
      "  [1 0 0 0]\n",
      "  [1 0 0 1]\n",
      "  [0 1 0 0]\n",
      "  [0 1 0 1]\n",
      "  [1 1 0 0]\n",
      "  [1 1 0 1]]]\n",
      "INFO:tensorflow:device_assignment.core_assignment: [[[0 0 0 0]]\n",
      "\n",
      " [[0 0 0 1]]\n",
      "\n",
      " [[1 0 0 0]]\n",
      "\n",
      " [[1 0 0 1]]\n",
      "\n",
      " [[0 1 0 0]]\n",
      "\n",
      " [[0 1 0 1]]\n",
      "\n",
      " [[1 1 0 0]]\n",
      "\n",
      " [[1 1 0 1]]]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_shape=[8] physical_shape=[2, 2, 2]\n",
      "INFO:tensorflow:auto_logical_to_physical_tpu logical_to_physical = [(0, 0, 0), (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 1, 0), (1, 1, 1), (1, 0, 0), (1, 0, 1)]\n",
      "WARNING:tensorflow:SimdMeshImpl ignoring devices ['', '', '', '', '', '', '', '']\n",
      "INFO:tensorflow:SimdMeshImpl init: Shape[model=8] LayoutRules{('d_ff', 'model'), ('vocab', 'model'), ('batch', 'batch'), ('ensemble', 'ensemble'), ('experts', 'batch'), ('heads', 'model')}\n",
      "INFO:tensorflow:Device Assignment: <tensorflow.python.tpu.device_assignment.DeviceAssignment object at 0x7f64d3c66e50>\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable encoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "WARNING:tensorflow:Using default tf glorot_uniform_initializer for variable decoder/block_000/layer_000/SelfAttention/relative_attention_bias  The initialzer will guess the input and output dimensions  based on dimension order.\n",
      "INFO:tensorflow:Create pnum_tensor\n",
      "INFO:tensorflow:Casting <dtype: 'int32'> to float32 for allreduce\n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_000/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_001/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_002/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_003/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_004/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_005/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_006/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_007/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_008/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_009/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_010/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_011/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_012/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_013/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_014/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_015/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_016/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_017/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_018/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_019/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_020/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_021/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_022/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/k                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/o                size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/q                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_001/EncDecAttention/v                size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable decoder/block_023/layer_002/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_000/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_001/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_002/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_003/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_004/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_005/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_006/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_007/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_008/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_009/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_010/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_011/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_012/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_013/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_014/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_015/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_016/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_017/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_018/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_019/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_020/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_021/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_022/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/k                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/o                  size 4194304      slice_size 524288       Shape[heads=4096, d_model=1024]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/q                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_000/SelfAttention/v                  size 4194304      slice_size 524288       Shape[d_model=1024, heads=4096]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wi/kernel         size 16777216     slice_size 2097152      Shape[d_model=1024, d_ff=16384]                             \n",
      "INFO:tensorflow:Variable encoder/block_023/layer_001/DenseReluDense/wo/kernel         size 16777216     slice_size 2097152      Shape[d_ff=16384, d_model=1024]                             \n",
      "INFO:tensorflow:Variable shared/embedding                                             size 32899072     slice_size 4112384      Shape[vocab=32128, d_model=1024]                            \n",
      "INFO:tensorflow:Variable stacked/decoder/block_005/layer_000/layer_norm/scale         size 59392        slice_size 59392        Shape[stacked=58, d_model=1024]                             \n",
      "INFO:tensorflow:    decoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_005/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_006/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_007/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_008/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_009/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_010/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_011/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_012/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_013/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_014/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_015/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_016/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_017/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_018/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_019/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_020/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_021/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_022/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_023/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/final_layer_norm/scale\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/SelfAttention/relative_attention_bias size 2048         slice_size 256          Shape[stacked=2, heads=32, buckets=32]                      \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/SelfAttention/relative_attention_bias\n",
      "INFO:tensorflow:Variable stacked/encoder/block_000/layer_000/layer_norm/scale         size 65536        slice_size 65536        Shape[stacked=64, d_model=1024]                             \n",
      "INFO:tensorflow:    encoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_005/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_006/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_007/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_008/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_009/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_010/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_011/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_012/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_013/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_014/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_015/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_016/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_017/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_018/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_019/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_020/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_021/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_022/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/block_023/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    encoder/final_layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_000/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_001/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_002/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_003/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_000/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_001/layer_norm/scale\n",
      "INFO:tensorflow:    decoder/block_004/layer_002/layer_norm/scale\n",
      "INFO:tensorflow:Trainable Variables            count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:All Variables                  count: 388     Total size: 2851598336       Total slice_size: 356559104      \n",
      "INFO:tensorflow:Counters:\n",
      "allreduce: 2.46e+10\n",
      " allreduce/[0]: 2.46e+10\n",
      "  allreduce/[0]/einsum_op: 2.46e+10\n",
      "einsum: 7.74e+13\n",
      "einsum_unique: 7.74e+13\n",
      "output: 4.83e+11\n",
      " output/AddOperation: 9.69e+10\n",
      " output/BinaryOpWithBroadcasting: 1.94e+09\n",
      " output/Constant: 4.83e+09\n",
      " output/EinsumOperation: 1.62e+11\n",
      " output/ImportOperation: 1.98e+05\n",
      " output/MinMaxOperation: 8.51e+07\n",
      " output/OneHotOperation: 3.39e+09\n",
      " output/RangeOperation: 6.14e+03\n",
      " output/ReduceOperation: 1.37e+08\n",
      " output/ReshapeOperation: 4.61e+10\n",
      " output/ScalarAddOperation: 1.37e+08\n",
      " output/ScalarMultiplyOperation: 2.27e+09\n",
      " output/ShiftOperation: 1.97e+05\n",
      " output/SlicewiseOperation: 1.35e+11\n",
      " output/StackedVariable: 1e+06\n",
      " output/StopGradient: 2.17e+10\n",
      " output/UnstackOperation: 1e+06\n",
      " output/Variable: 2.85e+09\n",
      " output/WhileLoopOperation: 4.83e+09\n",
      "output_unique: 3.49e+11\n",
      " output_unique/AddOperation: 7.57e+10\n",
      " output_unique/BinaryOpWithBroadcasting: 2.42e+08\n",
      " output_unique/Constant: 4.83e+09\n",
      " output_unique/EinsumOperation: 8.88e+10\n",
      " output_unique/ImportOperation: 2.48e+04\n",
      " output_unique/MinMaxOperation: 1.06e+07\n",
      " output_unique/OneHotOperation: 1.81e+09\n",
      " output_unique/RangeOperation: 768\n",
      " output_unique/ReduceOperation: 1.16e+08\n",
      " output_unique/ReshapeOperation: 3.75e+10\n",
      " output_unique/ScalarAddOperation: 1.72e+07\n",
      " output_unique/ScalarMultiplyOperation: 2.84e+08\n",
      " output_unique/ShiftOperation: 2.46e+04\n",
      " output_unique/SlicewiseOperation: 1.1e+11\n",
      " output_unique/StackedVariable: 1.27e+05\n",
      " output_unique/StopGradient: 2.17e+10\n",
      " output_unique/UnstackOperation: 1.27e+05\n",
      " output_unique/Variable: 2.85e+09\n",
      " output_unique/WhileLoopOperation: 4.83e+09\n",
      "variables: 2.85e+09\n",
      " variables/trainable: 2.85e+09\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:TPU job name worker\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from gs://t5_fewshot_mqg/models/t5_experiment_1/3B/all_SHOT/all/model.ckpt-1020544\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Starting infeed thread controller.\n",
      "INFO:tensorflow:Starting outfeed thread controller.\n",
      "INFO:tensorflow:Initialized dataset iterators in 0 seconds\n",
      "INFO:tensorflow:Before copy master to slices.\n",
      "INFO:tensorflow:Done with copy master to slices.\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Outfeed finished for iteration (0, 0)\n",
      "INFO:tensorflow:decoded 0: Combined assertion: Both Scott Derrickson and Ed Wood were American filmmakers, actors, writers, and directors. Answer: yes Question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Were Scott Derrickson and Ed Wood both American filmmakers, actors, writers, and directors?\n",
      "INFO:tensorflow:decoded 1: Combined assertion: The Laleli Mosque and the Esma Sultan Mansion are both historical buildings. Answer: no Question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Are the Laleli Mosque and the Esma Sultan Mansion both historical buildings?\n",
      "INFO:tensorflow:decoded 2: Combined assertion: Local H and For Against are both bands. Answer: yes Question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Are Local H and For Against both bands?\n",
      "INFO:tensorflow:decoded 4: Combined assertion: Ambroise Thomas was a French composer and Giuseppe Verdi was an Italian opera composer. Answer: yes Question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Were Ambroise Thomas and Giuseppe Verdi both opera composers?\n",
      "INFO:tensorflow:decoded 8: Combined assertion: In the Realm of the Hackers is an Australian documentary while Freakonomics is an American documentary. Answer: no Question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Are In the Realm of the Hackers and Freakonomics both documentaries?\n",
      "INFO:tensorflow:decoded 16: Combined assertion: Ajuga and Cypress are both types of plant. Answer: no Question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Are Ajuga and Cypress both types of plant?\n",
      "INFO:tensorflow:decoded 32: Combined assertion: Both Scott Derrickson and Ed Wood were American filmmakers, actors, writers, and directors. Answer: yes Question: <extra_id_0>\n",
      "INFO:tensorflow:            -> <extra_id_0> Were Scott Derrickson and Ed Wood both American filmmakers, actors, writers, and directors?\n",
      "INFO:tensorflow:Enqueue next (1) batch(es) of data to infeed.\n",
      "INFO:tensorflow:Dequeue next (1) batch(es) of data from outfeed.\n",
      "INFO:tensorflow:Stop infeed thread controller\n",
      "INFO:tensorflow:Shutting down InfeedController thread.\n",
      "INFO:tensorflow:InfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Infeed thread finished, shutting down.\n",
      "INFO:tensorflow:infeed marked as finished\n",
      "INFO:tensorflow:Stop output thread controller\n",
      "INFO:tensorflow:Shutting down OutfeedController thread.\n",
      "INFO:tensorflow:OutfeedController received shutdown signal, stopping.\n",
      "INFO:tensorflow:Outfeed thread finished, shutting down.\n",
      "INFO:tensorflow:outfeed marked as finished\n",
      "INFO:tensorflow:Shutdown TPU system.\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "\n",
      "Predictions for task 13 using checkpoint 1020544 were read!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model.batch_size = 64\n",
    "\n",
    "for task_num in range(7,14):\n",
    "    write_task_input_file(task_num)\n",
    "    model.predict(\n",
    "      input_file=predict_inputs_path[task_num],\n",
    "      output_file=predict_outputs_path[task_num],\n",
    "      checkpoint_steps=eval_checkpoint,\n",
    "      temperature=0,\n",
    "    )\n",
    "    parse_task_output_file(task_num=task_num, task_data=predict_data, predict_e2l=predict_e2l, predict_l2e=predict_l2e)\n",
    "fin = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "19efdc31-10d3-4853-af6a-df85862dbfd5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Predictions for task 6 using checkpoint 1020544 were read!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "parse_task_output_file(task_num=6, task_data=predict_data, predict_e2l=predict_e2l, predict_l2e=predict_l2e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "919bf260-4dcb-48de-a6d2-1cff2d023f90",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for data in predict_data:\n",
    "    if data['qtype'] in ['confused','bridge']:\n",
    "        for key in sorted(data.keys()):\n",
    "            if key not in ['p1', 'p2', 'gold_q', 'gold_qtype']:\n",
    "                print(key, data[key], )\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "ed172260-5e5d-4b87-a74b-fa9671cfc514",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'k': 0,\n",
       "  'p1': 'Scott Derrickson - Scott Derrickson (born July 16, 1966) is an American director, screenwriter and producer. He lives in Los Angeles, California. He is best known for directing horror films such as \"Sinister\", \"The Exorcism of Emily Rose\", and \"Deliver Us From Evil\", as well as the 2016 Marvel Cinematic Universe installment, \"Doctor Strange.\"',\n",
       "  'p2': 'Ed Wood - Edward Davis Wood Jr. (October 10, 1924 – December 10, 1978) was an American filmmaker, actor, writer, producer, and director.',\n",
       "  'a': 'yes',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Were Scott Derrickson and Ed Wood of the same nationality?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Scott Derrickson is an American director, screenwriter and producer.',\n",
       "  's2_c': 'Ed Wood was an American filmmaker, actor, writer, producer, and director.',\n",
       "  'c_c': 'Both Scott Derrickson and Ed Wood were American filmmakers, actors, writers, and directors.',\n",
       "  'q_c': 'Were Scott Derrickson and Ed Wood both American filmmakers, actors, writers, and directors?'},\n",
       " {'k': 1,\n",
       "  'p1': \"Shirley Temple - Shirley Temple Black (April 23, 1928 – February 10, 2014) was an American actress, singer, dancer, businesswoman, and diplomat who was Hollywood's number one box-office draw as a child actress from 1935 to 1938. As an adult, she was named United States ambassador to Ghana and to Czechoslovakia and also served as Chief of Protocol of the United States.\",\n",
       "  'p2': \"Kiss and Tell (1945 film) - Kiss and Tell is a 1945 American comedy film starring then 17-year-old Shirley Temple as Corliss Archer. In the film, two teenage girls cause their respective parents much concern when they start to become interested in boys. The parents' bickering about which girl is the worse influence causes more problems than it solves.\",\n",
       "  'a': 'Chief of Protocol',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Shirley Temple',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Shirley Temple served as Chief of Protocol.',\n",
       "  '6_p2': 0,\n",
       "  '6_p1': None,\n",
       "  's2': 'Kiss and Tell is a 1945 American comedy film starring Shirley Temple.',\n",
       "  's2_': 'Kiss and Tell is a 1945 American comedy film starring Shirley Temple.',\n",
       "  'c': 'Shirley Temple served as Chief of Protocol in Kiss and Tell, a 1945 American comedy film starring Shirley Temple.',\n",
       "  'c-b': 'The Chief of Protocol served in Kiss and Tell, a 1945 American comedy film starring Shirley Temple.',\n",
       "  'c-a': 'A certain person served in Kiss and Tell, a 1945 American comedy film starring Shirley Temple.',\n",
       "  'q': 'Who served in Kiss and Tell, a 1945 American comedy film starring Shirley Temple?'},\n",
       " {'k': 2,\n",
       "  'p1': 'The Hork-Bajir Chronicles - The Hork-Bajir Chronicles is the second companion book to the \"Animorphs\" series, written by K. A. Applegate. With respect to continuity within the series, it takes place before book #23, \"The Pretender\", although the events told in the story occur between the time of \"The Ellimist Chronicles\" and \"The Andalite Chronicles\". The book is introduced by Tobias, who flies to the valley of the free Hork-Bajir, where Jara Hamee tells him the story of how the Yeerks enslaved the Hork-Bajir, and how Aldrea, an Andalite, and her companion, Dak Hamee, a Hork-Bajir, tried to save their world from the invasion. Jara Hamee\\'s story is narrated from the points of view of Aldrea, Dak Hamee, and Esplin 9466, alternating in similar fashion to the \"Megamorphs\" books.',\n",
       "  'p2': 'Animorphs - Animorphs is a science fantasy series of young adult books written by Katherine Applegate and her husband Michael Grant, writing together under the name K. A. Applegate, and published by Scholastic. It is told in first person, with all six main characters taking turns narrating the books through their own perspectives. Horror, war, dehumanization, sanity, morality, innocence, leadership, freedom and growing up are the core themes of the series.',\n",
       "  'a': 'Animorphs',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Animorphs',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 1,\n",
       "  '6_p2': 2,\n",
       "  's1': 'The Hork-Bajir Chronicles is the second companion book to the \"Animorphs\" series.',\n",
       "  's2': 'Animorphs is a science fantasy series of young adult books.',\n",
       "  's1_': 'The Hork-Bajir Chronicles is the second companion book to the \"Animorphs\" series.',\n",
       "  's2_': 'Animorphs is a science fantasy series of young adult books.',\n",
       "  'c': 'The Hork-Bajir Chronicles is the second companion book to the \"Animorphs\" series which is a science fantasy series of young adult books.',\n",
       "  'c-b': 'The Hork-Bajir Chronicles is the second companion book to the series which is a science fantasy series of young adult books.',\n",
       "  'c-a': 'The Hork-Bajir Chronicles is the second companion book to the series which is a science fantasy series of young adult books.',\n",
       "  'q': 'The Hork-Bajir Chronicles is the second companion book to a series which is a science fantasy series of young adult books?'},\n",
       " {'k': 3,\n",
       "  'p1': 'Laleli Mosque - The Laleli Mosque (Turkish- \"Laleli Camii, or Tulip Mosque\" ) is an 18th-century Ottoman imperial mosque located in Laleli, Fatih, Istanbul, Turkey.',\n",
       "  'p2': 'Esma Sultan Mansion - The Esma Sultan Mansion (Turkish- \"Esma Sultan Yalısı\" ), a historical yalı (English- waterside mansion ) located at Bosphorus in Ortaköy neighborhood of Istanbul, Turkey and named after its original owner Esma Sultan, is used today as a cultural center after being redeveloped.',\n",
       "  'a': 'no',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'The Laleli Mosque is an 18th-century Ottoman imperial mosque located in Istanbul, Turkey.',\n",
       "  's2_c': 'The Esma Sultan Mansion is a historical yal ⁇',\n",
       "  'c_c': 'The Laleli Mosque and the Esma Sultan Mansion are both historical buildings.',\n",
       "  'q_c': 'Are the Laleli Mosque and the Esma Sultan Mansion both historical buildings?'},\n",
       " {'k': 4,\n",
       "  'p1': \"2014 S/S - 2014 S/S is the debut album of South Korean group WINNER. It was released on August 12, 2014 by the group's record label, YG Entertainment. The members were credited for writing the lyrics and composing the majority of the album's songs.\",\n",
       "  'p2': 'Winner (band) - Winner (Hangul- 위너), often stylized as WINNER, is a South Korean boy group formed in 2013 by YG Entertainment and debuted in 2014. It currently consists of four members, Jinwoo, Seunghoon, Mino and Seungyoon. Originally a five-piece group with Taehyun, who later departed from the group in November 2016.',\n",
       "  'a': 'YG Entertainment',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': '2014 S/S is the debut album of a South Korean boy group that was formed by who?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Winner',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': \"2014 S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label, YG Entertainment.\",\n",
       "  '6_p1': 3,\n",
       "  '6_p2': 4,\n",
       "  's2': 'Winner is a South Korean boy group.',\n",
       "  's1_': '2014 S/S is the debut album of South Korean group WINNER.',\n",
       "  's2_': 'Winner is a South Korean boy group.',\n",
       "  'c': \"2014 S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label, YG Entertainment.\",\n",
       "  'c-b': \"2014 S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label, YG Entertainment.\",\n",
       "  'c-a': \"S/S is the debut album of South Korean group WINNER released on August 12, 2014 by the group's record label.\",\n",
       "  'q': 'S/S is the debut album of South Korean group WINNER released on what date?'},\n",
       " {'k': 5,\n",
       "  'p1': 'Eenasul Fateh - Eenasul Fateh (Bengali- ঈনাসুল ফাতেহ ; born 3 April 1959), also known by his stage name Aladin, is a Bangladeshi-British cultural practitioner, magician, live artist and former international management consultant.',\n",
       "  'p2': \"Management consulting - Management consulting is the practice of helping organizations to improve their performance, operating primarily through the analysis of existing organizational problems and the development of plans for improvement. Organizations may draw upon the services of management consultants for a number of reasons, including gaining external (and presumably objective) advice and access to the consultants' specialized expertise.\",\n",
       "  'a': 'Eenasul Fateh',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Who was known by his stage name Aladin and helped organizations improve their performance as a consultant?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Management consulting',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Eenasul Fateh is a former international management consultant.',\n",
       "  '6_p2': 5,\n",
       "  '6_p1': None,\n",
       "  's2': 'Management consulting is the practice of helping organizations to improve their performance.',\n",
       "  's2_': 'Management consulting is the practice of helping organizations to improve their performance.',\n",
       "  'c': 'Eenasul Fateh is a former international management consultant who is involved in the practice of management consulting which is the practice of helping organizations to improve their performance.',\n",
       "  'c-b': 'Eenasul Fateh is a former international management consultant who is involved in the practice of helping organizations to improve their performance.',\n",
       "  'c-a': 'A certain person is a former international management consultant who is involved in the practice of helping organizations to improve their performance.',\n",
       "  'q': 'Who is a former international management consultant who is involved in the practice of helping organizations to improve their performance?'},\n",
       " {'k': 6,\n",
       "  'p1': 'Annie Morton - Annie Morton (born October 8, 1970) is an American model born in Pennsylvania. She has appeared on the covers of \"British Vogue\", \"ID\", \"Marie Claire\", and other magazines. She has been photographed by Helmut Newton; Peter Lindbergh; Annie Leibovitz; Richard Avedon; Juergen Teller; Paul Jasmin, Mary Ellen Mark and Terry Richardson, and modeled for Donna Karan, Givenchy, Guerlain, Chanel, \"Harper\\'s Bazaar\", \"Sports Illustrated\" and Victoria\\'s Secret. A long time vegetarian, an advocate for organic lifestyle choices and natural healthcare. She co-founded Tsi-La Organics, a \"Green Luxury\" company that creates and sells vegan, organic perfume and skin care products.',\n",
       "  'p2': 'Terry Richardson - Terrence \"Uncle Terry\" Richardson (born August 14, 1965) is an American fashion and portrait photographer who has shot advertising campaigns for Marc Jacobs, Aldo, Supreme, Sisley, Tom Ford, and Yves Saint Laurent among others. He has also done work for magazines such as \"Rolling Stone\", \"GQ\", \"Vogue\", \"Vanity Fair\", \"Harper\\'s Bazaar\", \"i-D\", and \"Vice\".',\n",
       "  'a': 'Terry Richardson',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Who is older, Annie Morton or Terry Richardson?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Terry Richardson',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 6,\n",
       "  '6_p2': 7,\n",
       "  's1': 'Annie Morton has been photographed by Terry Richardson.',\n",
       "  's2': 'Terry Richardson is an American fashion and portrait photographer.',\n",
       "  's1_': 'Annie Morton has been photographed by Terry Richardson.',\n",
       "  's2_': 'Terry Richardson is an American fashion and portrait photographer.',\n",
       "  'c': 'Annie Morton has been photographed by Terry Richardson who is an American fashion and portrait photographer.',\n",
       "  'c-b': 'Annie Morton has been photographed by an American fashion and portrait photographer.',\n",
       "  'c-a': 'Annie Morton has been photographed by an American fashion and portrait photographer.',\n",
       "  'q': 'Who is the photographer who photographed Annie Morton?'},\n",
       " {'k': 7,\n",
       "  'p1': 'Local H - Local H is an American rock band originally formed by guitarist and vocalist Scott Lucas, bassist Matt Garcia, drummer Joe Daniels, and lead guitarist John Sparkman in Zion, Illinois in 1987. The members all met in high school in 1987 and founded Local H three years later.',\n",
       "  'p2': 'For Against - For Against is a United States post-punk/dream pop band from Lincoln, Nebraska. Despite numerous lineup shuffles and some periods of dormancy, the band has produced material steadily from its 1984 inception to the present.',\n",
       "  'a': 'yes',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Are Local H and For Against both from the United States?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Local H is an American rock band.',\n",
       "  's2_c': 'For Against is a post-punk/dream pop band.',\n",
       "  'c_c': 'Local H and For Against are both bands.',\n",
       "  'q_c': 'Are Local H and For Against both bands?'},\n",
       " {'k': 8,\n",
       "  'p1': 'Kansas Song - Kansas Song (We’re From Kansas) is a fight song of the University of Kansas.',\n",
       "  'p2': \"University of Kansas - The University of Kansas, often referred to as KU or Kansas, is a public research university in the U.S. state of Kansas. The main campus in Lawrence, one of the largest college towns in Kansas, is on Mount Oread, the highest elevation in Lawrence. Two branch campuses are in the Kansas City metropolitan area- the Edwards Campus in Overland Park, and the university's medical school and hospital in Kansas City. There are also educational and research sites in Parsons, Topeka, Garden City, Hays, and Leavenworth, and branches of the medical school in Wichita and Salina. The university is one of the 62 members of the Association of American Universities.\",\n",
       "  'a': 'Kansas Song',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What is the name of the fight song of the university whose main campus is in Lawrence, Kansas and whose branch campuses are in the Kansas City metropolitan area?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'University of Kansas',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Kansas Song is a fight song of the University of Kansas.',\n",
       "  '6_p1': 8,\n",
       "  '6_p2': 9,\n",
       "  's2': 'The University of Kansas is a public research university in the U.S. state of Kansas.',\n",
       "  's1_': 'Kansas Song (We’re From Kansas) is a fight song of the University of Kansas.',\n",
       "  's2_': 'The University of Kansas is a public research university in the U.S. state of Kansas.',\n",
       "  'c': 'Kansas Song is a fight song of the University of Kansas which is a public research university in the U.S. state of Kansas.',\n",
       "  'c-b': 'Kansas Song is a fight song of a public research university in the U.S. state of Kansas.',\n",
       "  'c-a': 'A certain fight song is a fight song of a public research university in the U.S. state of Kansas.',\n",
       "  'q': 'What fight song is a fight song of a public research university in the U.S. state of Kansas?'},\n",
       " {'k': 9,\n",
       "  'p1': 'David Weissman - David Weissman is a screenwriter and director. His film credits include \"The Family Man\" (2000), \"Evolution\" (2001), and \"\"When in Rome\"\" (2010).',\n",
       "  'p2': \"The Family Man - The Family Man is a 2000 American romantic comedy-drama film directed by Brett Ratner, written by David Diamond and David Weissman, and starring Nicolas Cage and Téa Leoni. Cage's production company, Saturn Films, helped produce the film. The film centers on a man who sees what could have been had he made a different decision 13 years prior.\",\n",
       "  'a': 'David Weissman',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What screenwriter with credits for \"Evolution\" co-wrote a film starring Nicolas Cage and Téa Leoni?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'The Family Man',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'David Weissman is a screenwriter and director whose film credits include \"The Family Man\"',\n",
       "  '6_p1': 10,\n",
       "  '6_p2': 11,\n",
       "  's2': 'The Family Man is a film starring Nicolas Cage.',\n",
       "  's1_': 'David Weissman is a screenwriter and director whose film credits include \"The Family Man\"',\n",
       "  's2_': 'The Family Man is a film starring Nicolas Cage.',\n",
       "  'c': 'David Weissman is a screenwriter and director whose film credits include \"The Family Man\" which is a film starring Nicolas Cage.',\n",
       "  'c-b': 'David Weissman is a screenwriter and director whose film credits include a film starring Nicolas Cage.',\n",
       "  'c-a': 'A certain screenwriter and director whose film credits include a film starring Nicolas Cage.',\n",
       "  'q': 'Who is the screenwriter and director whose film credits include a film starring Nicolas Cage?'},\n",
       " {'k': 10,\n",
       "  'p1': 'End of Days (film) - End of Days is a 1999 American fantasy action horror thriller film directed by Peter Hyams and starring Arnold Schwarzenegger, Gabriel Byrne, Robin Tunney, Kevin Pollak, Rod Steiger, CCH Pounder, and Udo Kier. The film follows former New York Police Department detective Jericho Cane (Schwarzenegger) after he saves a banker (Byrne) from an assassin, finds himself embroiled in a religious conflict, and must protect an innocent young woman (Tunney) who is chosen by evil forces to conceive the Antichrist with Satan.',\n",
       "  'p2': 'Oh My God (Guns N\\' Roses song) - \"Oh My God\" is a song by Guns N\\' Roses released in 1999 on the soundtrack to the film \"End of Days\". The song was sent out to radio stations in November 1999 as a promo for the soundtrack and the band. Despite being the band\\'s first recorded release in almost five years, it was never issued as a stand-alone single for public retail.',\n",
       "  'a': '1999',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What year did Guns N Roses perform a promo for a movie starring Arnold Schwarzenegger as a former New York Police detective?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'End of Days',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'End of Days is a 1999 American fantasy action horror thriller film directed by Peter Hyams and starring Arnold Schwarzenegger.',\n",
       "  '6_p1': 12,\n",
       "  '6_p2': 13,\n",
       "  's2': 'Oh My God is a song by Guns N\\' Roses released in 1999 on the soundtrack to the film \"End of Days\".',\n",
       "  's1_': 'End of Days is a 1999 American fantasy action horror thriller film starring Arnold Schwarzenegger.',\n",
       "  's2_': 'Oh My God is a song by Guns N\\' Roses released in 1999 on the soundtrack to the film \"End of Days\".',\n",
       "  'c': \"Oh My God is a song by Guns N' Roses released in 1999 on the soundtrack to the film End of Days which is a 1999 American fantasy action horror thriller film directed by Peter Hyams and starring Arnold Schwarzenegger.\",\n",
       "  'c-b': \"Oh My God is a song by Guns N' Roses released in 1999 on the soundtrack to a film which is a 1999 American fantasy action horror thriller film directed by Peter Hyams and starring Arnold Schwarzenegger.\",\n",
       "  'c-a': \"Oh My God is a song by Guns N' Roses released in a certain year on the soundtrack to a film which is an American fantasy action horror thriller film directed by Peter Hyams and starring Arnold Schwarzenegger.\",\n",
       "  'q': \"Oh My God is a song by Guns N' Roses released in what year on the soundtrack to a film which is an American fantasy action horror thriller film directed by Peter Hyams and starring Arnold Schwarzenegger?\"},\n",
       " {'k': 11,\n",
       "  'p1': '888 7th Avenue - 888 7th Avenue is a 628 ft (191m) tall modern-style office skyscraper in Midtown Manhattan which was completed in 1969 and has 46 floors. Emery Roth & Sons designed the building, which is tied with Central Park Place for the 65th tallest building in New York City. It currently carries the Vornado Realty Trust corporate headquarters. Previously known as the Arlen Building, its namesake being the company responsible for its construction, Arlen Realty & Development Corporation. The Red Eye Grill is located in the building at street level.',\n",
       "  'p2': 'Random House Tower - The Random House Tower, also known as the Park Imperial Apartments, is a 52-story mixed-use tower in New York City, United States, that is used as the headquarters of book publisher Random House and a luxury apartment complex. The book publisher entrance is on Broadway and goes up to 27 floors, while the apartment complex entrance is on West 56th Street.',\n",
       "  'a': 'no',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Are Random House Tower and 888 7th Avenue both used for real estate?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': '888 7th Avenue is a skyscraper with 46 floors.',\n",
       "  's2_c': 'Random House Tower is a 52-story mixed-use tower.',\n",
       "  'c_c': '888 7th Avenue and Random House Tower both have 52 stories.',\n",
       "  'q_c': 'Do 888 7th Avenue and Random House Tower have 52 stories?'},\n",
       " {'k': 12,\n",
       "  'p1': \"1995–96 Manchester United F.C. season - The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football. United finished the season by becoming the first English team to win the Double (league title and FA Cup) twice. Their triumph was made all the more remarkable by the fact that Alex Ferguson had sold experienced players Paul Ince, Mark Hughes and Andrei Kanchelskis before the start of the season, and not made any major signings. Instead, he had drafted in young players like Nicky Butt, David Beckham, Paul Scholes and the Neville brothers, Gary and Phil.\",\n",
       "  'p2': 'Alex Ferguson - Sir Alexander Chapman Ferguson, CBE (born 31 December 1941) is a Scottish former football manager and player who managed Manchester United from 1986 to 2013. He is regarded by many players, managers and analysts to be one of the greatest and most successful managers of all time.',\n",
       "  'a': 'from 1986 to 2013',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'The football manager who recruited David Beckham managed Manchester United during what timeframe?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Alex Ferguson',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Alex Ferguson is a Scottish former football manager and player who managed Manchester United from 1986 to 2013.',\n",
       "  '6_p1': 14,\n",
       "  '6_p2': None,\n",
       "  's1': \"The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football.\",\n",
       "  's1_': \"The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football.\",\n",
       "  'c': \"The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by Scottish former football manager and player Alex Ferguson.\",\n",
       "  'c-b': \"The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by Scottish former football manager and player.\",\n",
       "  'c-a': \"The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by Scottish former football manager and player.\",\n",
       "  'q': \"The 1995–96 season was Manchester United's fourth season in the Premier League, and their 21st consecutive season in the top division of English football, managed by which Scottish former football manager and player?\"},\n",
       " {'k': 13,\n",
       "  'p1': 'Brown State Fishing Lake - Brown State Fishing Lake (sometimes also known as Brown State Fishing Lake And Wildlife Area) is a protected area in Brown County, Kansas in the United States. The lake is 62 acres (0.25 km²) in area and up to 13 feet (4 m) deep. The area was formerly known as Brown County State Park, and is 8 miles (13 km) east of Hiawatha, Kansas.',\n",
       "  'p2': 'Brown County, Kansas - Brown County (county code BR) is a county located in the northeast portion of the U.S. state of Kansas. As of the 2010 census, the county population was 9,984. Its county seat and most populous city is Hiawatha. Brown County is the location of the Kickapoo Indian Reservation of Kansas, the majority of the Sac and Fox Reservation and the majority of the Iowa Reservation of Kansas and Nebraska.',\n",
       "  'a': '9,984',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Brown State Fishing Lake is in a country that has a population of how many inhabitants ?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Brown County, Kansas',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Brown County, Kansas has a population of 9,984.',\n",
       "  '6_p1': 15,\n",
       "  '6_p2': None,\n",
       "  's1': 'Brown State Fishing Lake is in Brown County, Kansas.',\n",
       "  's1_': 'Brown State Fishing Lake is in Brown County, Kansas.',\n",
       "  'c': 'Brown State Fishing Lake is in Brown County, Kansas which has a population of 9,984.',\n",
       "  'c-b': 'Brown State Fishing Lake is in a county which has a population of 9,984.',\n",
       "  'c-a': 'Brown State Fishing Lake is in a county which has a certain population.',\n",
       "  'q': 'What is the population of the county in which Brown State Fishing Lake is located?'},\n",
       " {'k': 14,\n",
       "  'p1': 'America East Conference - The America East Conference is a collegiate athletic conference affiliated with the NCAA Division I, whose members are located mainly in the Northeastern United States. The conference was known as the Eastern College Athletic Conference-North from 1979 to 1988 and the North Atlantic Conference from 1988 to 1996.',\n",
       "  'p2': \"Vermont Catamounts men's soccer - The Vermont Catamounts men's soccer team represents the University of Vermont in all NCAA Division I men's college soccer competitions. The team competes in the America East Conference.\",\n",
       "  'a': 'the North Atlantic Conference',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': \"The Vermont Catamounts men's soccer team currently competes in a conference that was formerly known as what from 1988 to 1996?\",\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'America East Conference',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'The America East Conference was known as the North Atlantic Conference from 1988 to 1996.',\n",
       "  '6_p2': 16,\n",
       "  '6_p1': None,\n",
       "  's2': \"Vermont Catamounts men's soccer team competes in the America East Conference.\",\n",
       "  's2_': \"Vermont Catamounts men's soccer team competes in the America East Conference.\",\n",
       "  'c': \"The America East Conference was known as the North Atlantic Conference from 1988 to 1996 and the Vermont Catamounts men's soccer team competes in it.\",\n",
       "  'c-b': \"The Vermont Catamounts men's soccer team competes in a conference that was known as the North Atlantic Conference from 1988 to 1996.\",\n",
       "  'c-a': \"The Vermont Catamounts men's soccer team competes in a conference that was known as from 1988 to 1996.\",\n",
       "  'q': \"The Vermont Catamounts men's soccer team competes in what conference that was known as from 1988 to 1996?\"},\n",
       " {'k': 15,\n",
       "  'p1': 'Giuseppe Verdi - Giuseppe Fortunino Francesco Verdi (] ; 9 or 10 October 1813 – 27 January 1901) was an Italian opera composer.',\n",
       "  'p2': 'Ambroise Thomas - Charles Louis Ambroise Thomas (5 August 1811 – 12 February 1896) was a French composer, best known for his operas \"Mignon\" (1866) and \"Hamlet\" (1868, after Shakespeare) and as Director of the Conservatoire de Paris from 1871 till his death.',\n",
       "  'a': 'yes',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Are Giuseppe Verdi and Ambroise Thomas both Opera composers ?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Giuseppe Verdi was an Italian opera composer.',\n",
       "  's2_c': 'Ambroise Thomas was a French composer.',\n",
       "  'c_c': 'Ambroise Thomas was a French composer and Giuseppe Verdi was an Italian opera composer.',\n",
       "  'q_c': 'Were Ambroise Thomas and Giuseppe Verdi both opera composers?'},\n",
       " {'k': 16,\n",
       "  'p1': 'Richard Nixon - Richard Milhous Nixon (January 9, 1913 – April 22, 1994) was the 37th President of the United States from 1969 until 1974, when he resigned from office, the only U.S. president to do so. He had previously served as the 36th Vice President of the United States from 1953 to 1961, and prior to that as a U.S. Representative and also Senator from California.',\n",
       "  'p2': \"Roger O. Egeberg - Roger Olaf Egeberg, M.D. (13 November 1902 – 13 September 1997 Washington, D.C.) was an American medical educator, administrator and advocate of public health. He was General Douglas MacArthur's personal physician during World War II in the Pacific theater. His other roles included Assistant Secretary for Health and Scientific Affairs in the Department of Health, Education, and Welfare (now the United States Department of Health and Human Services) during the Nixon administration and Dean of the University of Southern California's medical school.\",\n",
       "  'a': '1969 until 1974',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Roger O. Egeberg was Assistant Secretary for Health and Scientific Affairs during the administration of a president that served during what years?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Nixon administration',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Richard Nixon served as the 37th President of the United States from 1969 until 1974.',\n",
       "  '6_p2': 17,\n",
       "  '6_p1': None,\n",
       "  's2': 'Roger O. Egeberg served as Assistant Secretary for Health and Scientific Affairs in the Department of Health, Education, and Welfare during the Nixon administration.',\n",
       "  's2_': 'Roger O. Egeberg served as Assistant Secretary for Health and Scientific Affairs in the Department of Health, Education, and Welfare during the Nixon administration.',\n",
       "  'c': 'Roger O. Egeberg served as Assistant Secretary for Health and Scientific Affairs in the Department of Health, Education, and Welfare during the Nixon administration which was headed by Richard Nixon.',\n",
       "  'c-b': 'Roger O. Egeberg served as Assistant Secretary for Health and Scientific Affairs in the Department of Health, Education, and Welfare during the administration which was headed by Richard Nixon.',\n",
       "  'c-a': 'Roger O. Egeberg served as Assistant Secretary for Health and Scientific Affairs in the Department of Health, Education, and Welfare during the administration which was headed by Richard Nixon.',\n",
       "  'q': 'Roger O. Egeberg served as Assistant Secretary for Health and Scientific Affairs in the Department of Health, Education, and Welfare during the administration which was headed by Richard Nixon?'},\n",
       " {'k': 17,\n",
       "  'p1': 'Henry Roth - Henry Roth (February 8, 1906 – October 13, 1995) was an American novelist and short story writer.',\n",
       "  'p2': 'Robert Erskine Childers - Robert Erskine Childers DSC (25 June 1870 – 24 November 1922), universally known as Erskine Childers, was a British writer, whose works included the influential novel \"The Riddle of the Sands\", and a Fenian revolutionary who smuggled guns to Ireland in his sailing yacht \"Asgard\". He was executed by the authorities of the nascent Irish Free State during the Irish Civil War. He was the son of British Orientalist scholar Robert Caesar Childers; the cousin of Hugh Childers and Robert Barton; and the father of the fourth President of Ireland, Erskine Hamilton Childers.',\n",
       "  'a': 'Robert Erskine Childers DSC',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Which writer was from England, Henry Roth or Robert Erskine Childers?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Henry Roth was an American novelist and short story writer.',\n",
       "  's2_c': 'Robert Erskine Childers was a British writer, whose works included the influential novel \"The Riddle of the Sands\".',\n",
       "  'c_c': 'Both Henry Roth and Robert Erskine Childers were writers.',\n",
       "  'q_c': 'Who amongst Henry Roth and Robert Erskine Childers was a writer?'},\n",
       " {'k': 18,\n",
       "  'p1': 'Sergio Pérez - Sergio Pérez Mendoza (    ; born 26 January 1990) also known as \"Checo\" Pérez, is a Mexican racing driver, currently driving for Force India.',\n",
       "  'p2': 'Formula One drivers from Mexico - There have been six Formula One drivers from Mexico who have taken part in races since the championship began in 1950. Pedro Rodríguez is the most successful Mexican driver being the only one to have won a grand prix. Sergio Pérez, the only other Mexican to finish on the podium, currently races with Sahara Force India F1 Team .',\n",
       "  'a': 'Pedro Rodríguez',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Which other Mexican Formula One race car driver has held the podium besides the Force India driver born in 1990?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Sergio Pérez',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Pedro Rodr ⁇ guez is the most successful Mexican driver being the only one to have won a grand prix.',\n",
       "  '6_p1': 18,\n",
       "  '6_p2': 19,\n",
       "  's1': 'Sergio Pérez is a Mexican racing driver, currently driving for Force India.',\n",
       "  's1_': 'Sergio Pérez is a Mexican racing driver, currently driving for Force India.',\n",
       "  's2_': 'Sergio Pérez is the only other Mexican to finish on the podium.',\n",
       "  'c': 'Pedro Rodr ?? guez is the most successful Mexican driver being the only one to have won a grand prix, Sergio Pérez is a Mexican racing driver, currently driving for Force India.',\n",
       "  'c-b': 'Pedro Rodr ⁇ guez is the most successful Mexican driver being the only one to have won a grand prix.',\n",
       "  'c-a': 'The most successful Mexican driver being the only one to have won a grand prix is Pedro Rodr ⁇ guez.',\n",
       "  'q': 'Who is the most successful Mexican driver being the only one to have won a grand prix?'},\n",
       " {'k': 19,\n",
       "  'p1': 'Jim Cummings - James Jonah Cummings (born November 3, 1952) is an American voice actor and singer, who has appeared in almost 400 roles. He is known for voicing the title character from \"Darkwing Duck\", Dr. Robotnik from \"Sonic the Hedgehog\", and Pete. His other characters include Winnie the Pooh, Tigger, and the Tasmanian Devil. He has performed in numerous Disney and DreamWorks animations including \"Aladdin\", \"The Lion King\", \"Balto\", \"Antz\", \"The Road to El Dorado\", \"Shrek\", and \"The Princess and the Frog\". He has also provided voice-over work for video games, such as \"Icewind Dale\", \"Fallout\", \"\", \"Baldur\\'s Gate\", \"Mass Effect 2\", \"\", \"\", \"\", and \"Splatterhouse\".',\n",
       "  'p2': 'A Rather Blustery Day - \"A Rather Blustery Day\" is a whimsical song from the Walt Disney musical film featurette, \"Winnie the Pooh and the Blustery Day\". It was written by Robert & Richard Sherman and sung by Jim Cummings as \"Pooh\".',\n",
       "  'a': 'Sonic',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'This singer of A Rather Blustery Day also voiced what hedgehog?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Jim Cummings',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Jim Cummings is known for voicing Dr. Robotnik from \"Sonic the Hedgehog\".',\n",
       "  '6_p2': 20,\n",
       "  '6_p1': None,\n",
       "  's2': 'A Rather Blustery Day is a whimsical song sung by Jim Cummings as \"Pooh\".',\n",
       "  's2_': 'A Rather Blustery Day is a whimsical song sung by Jim Cummings as \"Pooh\".',\n",
       "  'c': 'Jim Cummings is known for voicing Dr. Robotnik from \"Sonic the Hedgehog\" and he sang A Rather Blustery Day as \"Pooh\".',\n",
       "  'c-b': 'A certain actor is known for voicing Dr. Robotnik from \"Sonic the Hedgehog\" and he sang A Rather Blustery Day as \"Pooh\".',\n",
       "  'c-a': 'A certain actor is known for voicing Dr. Robotnik from a certain video game and he sang A Rather Blustery Day as Pooh.',\n",
       "  'q': 'Which actor is known for voicing Dr. Robotnik from a certain video game and he sang A Rather Blustery Day as Pooh?'},\n",
       " {'k': 20,\n",
       "  'p1': 'Apple Remote - The Apple Remote is a remote control device released in or after October 2005 by Apple Inc. for use with a number of its products which use infrared capabilities. The device was originally designed to interact with the Front Row media program on the iSight iMac G5 and is compatible with some later desktop and portable Macintosh computers. The first three generations of Apple TV used the Apple Remote as their primary control mechanism. It has now been replaced with the Siri Remote in the fourth generation. Prior to the Apple Remote, Apple produced several nameless IR remotes for products such as the Macintosh TV, TV tuner expansion boards, and the PowerCD drive.',\n",
       "  'p2': \"Front Row (software) - Front Row is a discontinued media center software application for Apple's Macintosh computers and Apple TV for navigating and viewing video, photos, podcasts, and music from a computer, optical disc, or the Internet through a 10-foot user interface (similar to Windows Media Center and Kodi). The software relies on iTunes and iPhoto and is controlled by an Apple Remote or the keyboard function keys. The first version was released October 2005, with two major revisions since. Front Row was removed and discontinued in Mac OS X 10.7.\",\n",
       "  'a': 'keyboard function keys',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Aside from the Apple Remote, what other device can control the program Apple Remote was originally designed to interact with?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Front Row',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Front Row is a discontinued media center software application controlled by an Apple Remote or the keyboard function keys.',\n",
       "  '6_p1': 21,\n",
       "  '6_p2': None,\n",
       "  's1': 'The Apple Remote was originally designed to interact with the Front Row media program.',\n",
       "  's1_': 'The Apple Remote was originally designed to interact with the Front Row media program.',\n",
       "  'c': 'Front Row is a discontinued media center software application controlled by an Apple Remote or the keyboard function keys.',\n",
       "  'c-b': 'A discontinued media center software application controlled by an Apple Remote or the keyboard function keys.',\n",
       "  'c-a': 'A discontinued media center software application controlled by an Apple Remote or the keys on a certain device.',\n",
       "  'q': 'What is the name of the device controlled by a discontinued media center software application controlled by an Apple Remote or the keys on it?'},\n",
       " {'k': 21,\n",
       "  'p1': 'Wolf Alice - Wolf Alice are a four-piece alternative rock band from North London, formed initially as a two-person band in 2010. Its members since 2012 are Ellie Rowsell (vocals, guitar), Joff Oddie (guitars, vocals), Theo Ellis (bass), and Joel Amey (drums, vocals).',\n",
       "  'p2': 'Badly Drawn Boy - Damon Michael Gough (born 2 October 1969, in Dunstable, Bedfordshire), known by the stage name Badly Drawn Boy, is an English indie singer-songwriter and multi-instrumentalist.',\n",
       "  'a': 'Badly Drawn Boy',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Which performance act has a higher instrument to person ratio, Badly Drawn Boy or Wolf Alice? ',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Wolf Alice are a four-piece alternative rock band from North London.',\n",
       "  's2_c': 'Badly Drawn Boy is an English indie singer-songwriter and multi-instrumentalist.',\n",
       "  'c_c': 'Both Wolf Alice and Badly Drawn Boy are from London.',\n",
       "  'q_c': 'Which band is from London, Wolf Alice or Badly Drawn Boy?'},\n",
       " {'k': 22,\n",
       "  'p1': \"Peter Schmeichel - Peter Bolesław Schmeichel MBE (] ; born 18 November 1963) is a Danish former professional footballer who played as a goalkeeper, and was voted the IFFHS World's Best Goalkeeper in 1992 and 1993. He is best remembered for his most successful years at English club Manchester United, whom he captained to the 1999 UEFA Champions League to complete the Treble, and for winning UEFA Euro 1992 with Denmark.\",\n",
       "  'p2': 'Kasper Schmeichel - Kasper Peter Schmeichel (] ; born 5 November 1986) is a Danish professional footballer who plays as a goalkeeper for Premier League club Leicester City and the Denmark national team. He is the son of former Manchester United and Danish international goalkeeper Peter Schmeichel.',\n",
       "  'a': \"World's Best Goalkeeper\",\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What was the father of Kasper Schmeichel voted to be by the IFFHS in 1992?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Schmeichel',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': \"Peter Schmeichel was voted the IFFHS World's Best Goalkeeper in 1992 and 1993.\",\n",
       "  '6_p1': 22,\n",
       "  '6_p2': 23,\n",
       "  's2': 'Kasper Schmeichel is the son of former Manchester United and Danish international goalkeeper Peter Schmeichel.',\n",
       "  's1_': \"Peter Schmeichel is a Danish former professional footballer who played as a goalkeeper, and was voted the IFFHS World's Best Goalkeeper in 1992 and 1993.\",\n",
       "  's2_': 'Kasper Schmeichel is the son of former Manchester United and Danish international goalkeeper Peter Schmeichel.',\n",
       "  'c': \"Kasper Schmeichel is the son of former Manchester United and Danish international goalkeeper Peter Schmeichel who was voted the IFFHS World's Best Goalkeeper in 1992 and 1993.\",\n",
       "  'c-b': \"Kasper Schmeichel is the son of former Manchester United and Danish international goalkeeper Peter Schmeichel who was voted the IFFHS World's Best Goalkeeper in 1992 and 1993.\",\n",
       "  'c-a': \"Kasper Schmeichel is the son of former Manchester United and Danish international goalkeeper Peter Schmeichel who was voted the IFFHS World's Best Goalkeeper in 1992 and 1993.\",\n",
       "  'q': \"Kasper Schmeichel is the son of former Manchester United and Danish international goalkeeper Peter Schmeichel who was voted the IFFHS World's Best Goalkeeper in 1992 and 1993.\"},\n",
       " {'k': 23,\n",
       "  'p1': 'These Boots Are Made for Walkin\\' - \"These Boots Are Made for Walkin\\' is a hit song written by Lee Hazlewood and recorded by Nancy Sinatra. It charted January 22, 1966 and reached No. 1 in the United States \"Billboard\" Hot 100 and in the UK Singles Chart.',\n",
       "  'p2': 'Lee Hazlewood - Barton Lee Hazlewood (July 9, 1929 – August 4, 2007) was an American country and pop singer, songwriter, and record producer, most widely known for his work with guitarist Duane Eddy during the late 1950s and singer Nancy Sinatra in the 1960s.',\n",
       "  'a': 'Barton Lee Hazlewood',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': \"Who was the writer of These Boots Are Made for Walkin' and who died in 2007?\",\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Lee Hazlewood',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 24,\n",
       "  '6_p2': 25,\n",
       "  's1': \"These Boots Are Made for Walkin' is a hit song written by Lee Hazlewood.\",\n",
       "  's2': 'Lee Hazlewood was an American country and pop singer, record producer.',\n",
       "  's1_': \"These Boots Are Made for Walkin' is a hit song written by Lee Hazlewood.\",\n",
       "  's2_': 'Lee Hazlewood was an American country and pop singer, record producer.',\n",
       "  'c': \"Lee Hazlewood was an American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walkin'.\",\n",
       "  'c-b': \"The American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walkin'.\",\n",
       "  'c-a': 'The American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walking.',\n",
       "  'q': 'Who is the American country and pop singer, record producer who wrote the hit song These Boots Are Made for Walking?'},\n",
       " {'k': 24,\n",
       "  'p1': \"Virginia Commonwealth University - Virginia Commonwealth University (VCU) is a public research university located in Richmond, Virginia. VCU was founded in 1838 as the medical department of Hampden–Sydney College, becoming the Medical College of Virginia in 1854. In 1968, the Virginia General Assembly merged MCV with the Richmond Professional Institute, founded in 1917, to create Virginia Commonwealth University. In 2015, more than 31,000 students pursue 226 degree and certificate programs through VCU's 13 schools and one college. The VCU Health System supports the university's health care education, research and patient care mission.\",\n",
       "  'p2': \"2011–12 VCU Rams men's basketball team - The 2011–12 VCU Rams men's basketball team represented Virginia Commonwealth University during the 2011–12 NCAA Division I men's basketball season. It was the 44th season of the University fielding a men's basketball program. Led by third-year head coach Shaka Smart, the Rams were coming off a season marked by a run to the Final Four. Expected to finish lower in the CAA regular season standings, the Rams finished as regular season runners-up with a 15–3 conference record, before winning the 2012 CAA Men's Basketball Championship against Drexel, 59–56, earning their 11th ever berth into the NCAA Tournament. It was also the first season since 1984–85 that the Rams consecutively qualified for the NCAA Tournament.\",\n",
       "  'a': '1838',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': \"The 2011–12 VCU Rams men's basketball team, led by third year head coach Shaka Smart, represented Virginia Commonwealth University which was founded in what year?\",\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Virginia Commonwealth University',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Virginia Commonwealth University was founded in 1838.',\n",
       "  '6_p2': 26,\n",
       "  '6_p1': None,\n",
       "  's2': \"2011–12 VCU Rams men's basketball team represented Virginia Commonwealth University.\",\n",
       "  's2_': \"2011–12 VCU Rams men's basketball team represented Virginia Commonwealth University.\",\n",
       "  'c': \"2011–12 VCU Rams men's basketball team represented Virginia Commonwealth University which was founded in 1838.\",\n",
       "  'c-b': \"2011–12 VCU Rams men's basketball team represented the university which was founded in 1838.\",\n",
       "  'c-a': \"2011–12 VCU Rams men's basketball team represented the university which was founded in a certain year.\",\n",
       "  'q': \"2011–12 VCU Rams men's basketball team represented the university which was founded in what year?\"},\n",
       " {'k': 25,\n",
       "  'p1': 'Dictyosperma - Dictyosperma is a monotypic genus of flowering plant in the palm family found in the Mascarene Islands in the Indian Ocean (Mauritius, Réunion and Rodrigues). The sole species, Dictyosperma album, is widely cultivated in the tropics but has been farmed to near extinction in its native habitat. It is commonly called princess palm or hurricane palm, the latter owing to its ability to withstand strong winds by easily shedding leaves. It is closely related to, and resembles, palms in the \"Archontophoenix\" genus. The genus is named from two Greek words meaning \"net\" and \"seed\" and the epithet is Latin for \"white\", the common color of the crownshaft at the top of the trunk.',\n",
       "  'p2': 'Huernia - The genus Huernia (family Apocynaceae, subfamily Asclepiadoideae) consists of stem succulents from Eastern and Southern Africa, first described as a genus in 1810. The flowers are five-lobed, usually somewhat more funnel- or bell-shaped than in the closely related genus \"Stapelia\", and often striped vividly in contrasting colours or tones, some glossy, others matt and wrinkled depending on the species concerned. To pollinate, the flowers attract flies by emitting a scent similar to that of carrion. The genus is considered close to the genera \"Stapelia\" and \"Hoodia\". The name is in honour of Justin Heurnius (1587–1652) a Dutch missionary who is reputed to have been the first collector of South African Cape plants. His name was actually mis-spelt by the collector.',\n",
       "  'a': 'yes',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Are both Dictyosperma, and Huernia described as a genus?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'bleh',\n",
       "  's2_c': 'bleh',\n",
       "  'c_c': 'Both bleh and a sneeze are considered to be unpleasant.',\n",
       "  'q_c': 'Are both bleh and a sneeze considered to be unpleasant?'},\n",
       " {'k': 26,\n",
       "  'p1': 'Kaiser Ventures - Kaiser Ventures (formerly Kaiser Steel) corporation, headquartered in Ontario, California. It was founded by Henry J. Kaiser to provide steel plate for the Pacific Coast shipbuilding industry, which expanded during World War II, then shrank, then expanded again during the Korean War. California Shipbuilding Corporation on Terminal Island, California, was one of these shipyards which built hundreds of Liberty ships and Victory ships in World War II, and was also a project of Henry Kaiser. Kaiser Steel was noted for making the most of its costly steelmaking inputs, and it captured, along with the U.S. Steel plant in Utah, much of the Pacific Coast steel market by the late 1950s. Its assets included steelmaking plants in Napa, California (that it acquired from Basalt Rock Company in 1955) and Fontana, California (now operated by California Steel Industries), and a former open-pit iron ore mine at Eagle Mountain, California. Steel manufactured was also used in Kaiser Motors.',\n",
       "  'p2': 'Henry J. Kaiser - Henry John Kaiser (May 9, 1882 – August 24, 1967) was an American industrialist who became known as the father of modern American shipbuilding. He established the Kaiser Shipyards, which built Liberty ships during World War II, after which he formed Kaiser Aluminum and Kaiser Steel. Kaiser organized Kaiser Permanente health care for his workers and their families. He led Kaiser-Frazer followed by Kaiser Motors, automobile companies known for the safety of their designs. Kaiser was involved in large construction projects such as civic centers and dams, and invested in real estate. With his wealth, he established the Kaiser Family Foundation, a nonprofit, non-partisan, charitable organization.',\n",
       "  'a': 'Henry J. Kaiser',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Kaiser Ventures corporation was founded by an American industrialist who became known as the father of modern American shipbuilding?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Henry J. Kaiser',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 27,\n",
       "  '6_p2': 28,\n",
       "  's1': 'Kaiser Ventures was founded by Henry J. Kaiser.',\n",
       "  's2': 'Henry J. Kaiser established Kaiser Shipyards.',\n",
       "  's1_': 'Kaiser Ventures was founded by Henry J. Kaiser.',\n",
       "  's2_': 'Henry J. Kaiser established Kaiser Shipyards.',\n",
       "  'c': 'Henry J. Kaiser founded Kaiser Ventures and Kaiser Shipyards.',\n",
       "  'c-b': 'A certain person founded Kaiser Ventures and Kaiser Shipyards.',\n",
       "  'c-a': 'A certain person founded Kaiser Ventures and Kaiser Shipyards.',\n",
       "  'q': 'Who founded Kaiser Ventures and Kaiser Shipyards?'},\n",
       " {'k': 27,\n",
       "  'p1': 'Tunnels &amp; Trolls - Tunnels & Trolls (abbreviated \"T&T\") is a fantasy role-playing game designed by Ken St. Andre and first published in 1975 by Flying Buffalo. The second modern role-playing game published, it was written by Ken St. Andre to be a more accessible alternative to \"Dungeons & Dragons\" and is suitable for solitaire, group, and play-by-mail gameplay.',\n",
       "  'p2': 'Arena of Khazan - Arena of Khazan is a 1979 role-playing game adventure for \"Tunnels & Trolls\" published by Flying Buffalo.',\n",
       "  'a': 'Arena of Khazan',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What is the name for the adventure in \"Tunnels and Trolls\", a game designed by Ken St. Andre?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Tunnels & Trolls',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Arena of Khazan is a 1979 role-playing game adventure for \"Tunnels & Trolls\" published by Flying Buffalo.',\n",
       "  '6_p1': 29,\n",
       "  '6_p2': None,\n",
       "  's1': 'Tunnels & Trolls is a fantasy role-playing game designed by Ken St. Andre.',\n",
       "  's1_': 'Tunnels & Trolls is a fantasy role-playing game designed by Ken St. Andre.',\n",
       "  'c': 'Arena of Khazan is a 1979 role-playing game adventure for \"Tunnels & Trolls\", designed by Ken St. Andre and published by Flying Buffalo.',\n",
       "  'c-b': 'Arena of Khazan is a 1979 role-playing game adventure designed by Ken St. Andre and published by Flying Buffalo.',\n",
       "  'c-a': 'A 1979 role-playing game adventure designed by Ken St. Andre and published by Flying Buffalo.',\n",
       "  'q': 'What is the name of a 1979 role-playing game adventure designed by Ken St. Andre and published by Flying Buffalo?'},\n",
       " {'k': 28,\n",
       "  'p1': 'Poison (American band) - Poison is an American rock band that achieved great commercial success in the mid-1980s to the mid-1990s. Poison has sold over 40 million records worldwide and has sold 15 million records in the United States alone. The band has also charted ten singles to the Top 40 of the \"Billboard\" Hot 100, including six Top 10 singles and the Hot 100 number-one, \"Every Rose Has Its Thorn\". The band\\'s breakthrough debut album, the multi-platinum \"Look What the Cat Dragged In\", was released in 1986 and they hit their peak with the second album, \"Open Up and Say... Ahh!\", which became the band\\'s most successful album, being certified 5x platinum in the US. The popularity continued into the new decade with their third consecutive multi-platinum selling album, \"Flesh & Blood\".',\n",
       "  'p2': 'Shut Up, Make Love - \"Shut Up, Make Love\" is a song by American rock band Poison; released in 2000, it was the first single presented to radio stations from their 2000 \"Crack a Smile...and More!\" album. The song also appears on the second promo single \"Be the One\" as a B-side. \"Crack a Smile\" is Poison\\'s fifth studio album, released on March 14, 2000, and charting at #131 on The Billboard 200. The song was the first Poison single with lead guitarist Blues Saraceno.',\n",
       "  'a': '2000',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'When was Poison\\'s album \"Shut Up, Make Love\" released?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Poison',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Shut Up, Make Love is a song by American rock band Poison released in 2000.',\n",
       "  '6_p1': 30,\n",
       "  '6_p2': None,\n",
       "  's1': 'Poison is an American band that achieved great commercial success.',\n",
       "  's1_': 'Poison is an American band that achieved great commercial success.',\n",
       "  'c': 'Shut Up, Make Love is a song by American rock band Poison released in 2000, Poison is an American band that achieved great commercial success.',\n",
       "  'c-b': 'Shut Up, Make Love is a song by American rock band Poison released in 2000, Poison is an American band that achieved great commercial success.',\n",
       "  'c-a': 'Shut Up, Make Love is a song by American rock band Poison released in a certain year, Poison is an American band that achieved great commercial success.',\n",
       "  'q': 'Shut Up, Make Love is a song by American rock band Poison released in what year, Poison is an American band that achieved great commercial success?'},\n",
       " {'k': 29,\n",
       "  'p1': 'Hayden (musician) - Paul Hayden Desser (born February 12, 1971) who records as Hayden, is a Canadian singer-songwriter from Thornhill, Ontario.',\n",
       "  'p2': 'Buck-Tick - Buck-Tick (stylized as BUCK-TICK) is a Japanese rock band, formed in Fujioka, Gunma in 1983. The group has consisted of Atsushi Sakurai (lead vocals), Hisashi Imai (guitar), Hidehiko Hoshino (guitar), Yutaka Higuchi (bass) and Toll Yagami (drums) since 1985. In their three decade career, the band has released 20 studio albums, nearly all reaching the top ten on the charts, of which three in the late eighties and early nineties topped them. They are commonly credited as one of the founders of the visual kei movement.',\n",
       "  'a': 'Fujioka, Gunma',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Hayden is a singer-songwriter from Canada, but where does Buck-Tick hail from?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Buck-Tick',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Buck-Tick is a Japanese rock band, formed in Fujioka, Gunma.',\n",
       "  '6_p1': 31,\n",
       "  '6_p2': None,\n",
       "  's1': 'Hayden is a Canadian singer-songwriter from Thornhill, Ontario.',\n",
       "  's1_': 'Hayden is a Canadian singer-songwriter from Thornhill, Ontario.',\n",
       "  'c': 'Hayden is a Canadian singer-songwriter from Thornhill, Ontario and Buck-Tick is a Japanese rock band, formed in Fujioka, Gunma.',\n",
       "  'c-b': 'Hayden is a Canadian singer-songwriter from Thornhill, Ontario and Buck-Tick is a Japanese rock band, formed in Fujioka, Gunma.',\n",
       "  'c-a': 'Hayden is a Canadian singer-songwriter from Thornhill, Ontario and Buck-Tick is a Japanese rock band, formed in a certain place.',\n",
       "  'q': 'Hayden is a Canadian singer-songwriter from Thornhill, Ontario and Buck-Tick is a Japanese rock band, formed where?'},\n",
       " {'k': 30,\n",
       "  'p1': 'Charles Nungesser - Charles Eugène Jules Marie Nungesser, MC (15 March 1892 – presumably on or after 8 May 1927) was a French ace pilot and adventurer, best remembered as a rival of Charles Lindbergh. Nungesser was a renowned ace in France, ranking third highest in the country with 43 air combat victories during World War I.',\n",
       "  'p2': 'L\\'Oiseau Blanc - L\\'Oiseau Blanc (commonly known in the English-speaking world as The White Bird ) was a French Levasseur PL.8 biplane that disappeared in 1927, during an attempt to make the first non-stop transatlantic flight between Paris and New York City to compete for the Orteig Prize. The aircraft was flown by French World War I aviation heroes, Charles Nungesser and François Coli. The aircraft took off from Paris on 8 May 1927 and was last seen over Ireland. Less than two weeks later, Charles Lindbergh successfully made the New York–Paris journey and claimed the prize, flying the \"Spirit of St. Louis\".',\n",
       "  'a': 'Charles Eugène',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': \"Which  French ace pilot and adventurer fly L'Oiseau Blanc\",\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Charles Nungesser',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Charles Eugène Jules Marie Nungesser was a rival of Charles Lindbergh.',\n",
       "  '6_p1': 32,\n",
       "  '6_p2': 33,\n",
       "  's2': \"L'Oiseau Blanc was flown by Charles Nungesser.\",\n",
       "  's1_': 'Charles Nungesser was a French ace pilot and adventurer, best remembered as a rival of Charles Lindbergh.',\n",
       "  's2_': \"L'Oiseau Blanc was flown by Charles Nungesser.\",\n",
       "  'c': \"Charles Eugène Jules Marie Nungesser was a rival of Charles Lindbergh and he flew L'Oiseau Blanc.\",\n",
       "  'c-b': \"Charles Eugène Jules Marie Nungesser was a rival of Charles Lindbergh and he flew L'Oiseau Blanc.\",\n",
       "  'c-a': \"A certain person was a rival of Charles Lindbergh and he flew L'Oiseau Blanc.\",\n",
       "  'q': \"Who was a rival of Charles Lindbergh and who flew L'Oiseau Blanc?\"},\n",
       " {'k': 31,\n",
       "  'p1': 'In the Realm of the Hackers - In The Realm of the Hackers is a 2003 Australian documentary directed by Kevin Anderson about the prominent hacker community, centered in Melbourne, Australia in the late 1980s until early 1990. The storyline is centered on the Australian teenagers going by the hacker names \"Electron\" and \"Phoenix\", who were members of an elite computer hacking group called \\'The Realm\\' and hacked into some of the most secure computer networks in the world, including those of the US Naval Research Laboratory, Lawrence Livermore National Laboratory, a government lab charged with the security of the US nuclear stockpile, and NASA. The film runs for 55 minutes and was inspired by the book \"Underground\", by Melbourne-based writer and academic Suelette Dreyfus.',\n",
       "  'p2': 'Freakonomics (film) - Freakonomics- The Movie is a 2010 American documentary film based on the 2005 book \"Freakonomics- A Rogue Economist Explores the Hidden Side of Everything\" by economist Steven D. Levitt and writer Stephen J. Dubner. The film premiered at the Tribeca Film Festival in April 2010 with a theatrical release later in the year. The film received a lukewarm reception and a 65% fresh rating at Rottentomatoes.',\n",
       "  'a': 'no',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Are Freakonomics and In the Realm of the Hackers both American documentaries?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'In the Realm of the Hackers is an Australian documentary.',\n",
       "  's2_c': 'Freakonomics is an American documentary.',\n",
       "  'c_c': 'In the Realm of the Hackers is an Australian documentary while Freakonomics is an American documentary.',\n",
       "  'q_c': 'Are In the Realm of the Hackers and Freakonomics both documentaries?'},\n",
       " {'k': 32,\n",
       "  'p1': 'Letters to Cleo - Letters to Cleo are an alternative rock band from Boston, Massachusetts, best known for the 1994 single, \"Here & Now\", from their full-length debut album, \"Aurora Gory Alice\". The band\\'s members are Kay Hanley, Greg McKenna, Michael Eisenstein, Stacy Jones, Scott Riebling, and later, Tom Polce.',\n",
       "  'p2': \"Screaming Trees - Screaming Trees was an American rock band formed in Ellensburg, Washington in 1985 by vocalist Mark Lanegan, guitarist Gary Lee Conner, bass player Van Conner and drummer Mark Pickerel. Pickerel had been replaced by Barrett Martin by the time the band reached its most successful period. Although widely associated with grunge, the band's sound incorporated hard rock and psychedelic elements. During Screaming Trees' existence the band released seven studio albums, five EPs, and three compilations.\",\n",
       "  'a': 'Letters to Cleo',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Which band, Letters to Cleo or Screaming Trees, had more members?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Letters to Cleo are an alternative rock band from Boston, Massachusetts.',\n",
       "  's2_c': 'Screaming Trees were an American rock band.',\n",
       "  'c_c': 'Screaming Trees and Letters to Cleo are both rock bands.',\n",
       "  'q_c': 'Which band is more similar, Screaming Trees or Letters to Cleo?'},\n",
       " {'k': 33,\n",
       "  'p1': 'Socialist Revolutionary Party - The Socialist Revolutionary Party, or Party of Socialists-Revolutionaries (the SRs; Russian- Партия социалистов-революционеров (ПСР), эсеры , \"esery\") was a major political party in early 20th century Russia and a key player in the Russian Revolution. Its general ideology was revolutionary socialism of democratic socialist and agrarian socialist forms. After the February Revolution of 1917, it shared power with other liberal and democratic socialist forces within the Russian Provisional Government. In November 1917, it won a plurality of the national vote in Russia\\'s first-ever democratic elections (to the Russian Constituent Assembly), but the October Revolution had changed the political landscape and the Bolsheviks disbanded the Constituent Assembly in January 1918. The SRs soon split into pro-Bolshevik and anti-Bolshevik factions. The anti-Bolshevik faction of this party, known as the Right SRs, which remained loyal to the Provisional Government leader Alexander Kerensky was defeated and destroyed by the Bolsheviks in the course of the Russian Civil War and subsequent persecution.',\n",
       "  'p2': 'Russian Civil War - The Russian Civil War (Russian- Гражда́нская война́ в Росси́и , \"Grazhdanskaya voyna v Rossiyi\" ; November 1917 – October 1922) was a multi-party war in the former Russian Empire immediately after the Russian Revolutions of 1917, as many factions vied to determine Russia\\'s political future. The two largest combatant groups were the Red Army, fighting for the Bolshevik form of socialism led by Vladimir Lenin, and the loosely allied forces known as the White Army, which included diverse interests favoring monarchism, capitalism and alternative forms of socialism, each with democratic and antidemocratic variants. In addition, rival militant socialists and nonideological Green armies fought against both the Bolsheviks and the Whites. Eight foreign nations intervened against the Red Army, notably the Allied Forces and the pro-German armies. The Red Army defeated the White Armed Forces of South Russia in Ukraine and the army led by Admiral Aleksandr Kolchak in Siberia in 1919. The remains of the White forces commanded by Pyotr Nikolayevich Wrangel were beaten in Crimea and evacuated in late 1920. Lesser battles of the war continued on the periphery for two more years, and minor skirmishes with the remnants of the White forces in the Far East continued well into 1923. The war ended in 1923 in the sense that Red control of the newly formed Soviet Union was now assured, although armed national resistance in Central Asia was not completely crushed until 1934. There were an estimated 7,000,000–12,000,000 casualties during the war, mostly civilians. The Russian Civil War has been described by some as the greatest national catastrophe that Europe had yet seen.',\n",
       "  'a': 'October 1922',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Alexander Kerensky was defeated and destroyed by the Bolsheviks in the course of a civil war that ended when ?',\n",
       "  'qtype': 'confused',\n",
       "  'b': 'bleh',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': \"was <extra_id_1>  <extra_id_2>  <extra_id_3>  <extra_id_4>  <extra_id_5>  <extra_id_6>  <extra_id_7>  <extra_id_8>  <extra_id_9>  <extra_id_10>  <extra_id_11>  <extra_id_12>  <extra_id_13>  <extra_id_14>  <extra_id_15> a multi-party war in the former Russian Empire immediately after the Russian Revolutions of 1917, as many factions vied to determine Russia's political future. The Russian Civil War was a multi-party war in the former Russian Empire immediately after the Russian Revolutions of 1917, as many factions vied to determine Russia's political future. <extra_id_20>  <extra_id_21>  <extra_id_22>  <extra_id_23>  <extra_id_24>\",\n",
       "  '6_p1': 34,\n",
       "  '6_p2': None,\n",
       "  's1': 'The Socialist Revolutionary Party was a major political party in early 20th century Russia.',\n",
       "  's1_': 'The Socialist Revolutionary Party was a major political party in early 20th century Russia.',\n",
       "  'c': 'The Socialist Revolutionary Party was a major political party in early 20th century Russia which was involved in the Russian Civil War.',\n",
       "  'c-b': 'The Socialist Revolutionary Party was a major political party in early 20th century Russia which was involved in the Russian Civil War.',\n",
       "  'c-a': 'The Socialist Revolutionary Party was a major political party in early 20th century Russia which was involved in the Russian Civil War.',\n",
       "  'q': 'The Socialist Revolutionary Party was a major political party in early 20th century Russia which was involved in what war?',\n",
       "  's1_c': 'bleh',\n",
       "  's2_c': 'bleh',\n",
       "  'c_c': 'Both bleh and a sneeze are considered to be unpleasant.',\n",
       "  'q_c': 'In what year were both bleh and a sneeze considered to be unpleasant?'},\n",
       " {'k': 34,\n",
       "  'p1': 'Carlo Rovelli - Carlo Rovelli (born 3 May 1956) is an Italian theoretical physicist and writer who has worked in Italy, the United States and since 2000, in France. His work is mainly in the field of quantum gravity, where he is among the founders of the loop quantum gravity theory. He has also worked in the history and philosophy of science. He collaborates regularly with several Italian newspapers, in particular the cultural supplements of \"Il Corriere della Sera\", \"Il Sole 24 Ore\" and \"La Repubblica\".',\n",
       "  'p2': 'Seven Brief Lessons on Physics - Seven Brief Lessons on Physics (Italian- \"\" ) is a short book by the Italian physicist Carlo Rovelli. Originally published in Italian in 2014, the book has been translated into 41 languages. It is an Italian bestseller, with more than 300,000 copies sold in Rovelli\\'s home country.',\n",
       "  'a': '2000',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Seven Brief Lessons on Physics was written by an Italian physicist that has worked in France since what year?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Carlo Rovelli',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Carlo Rovelli is an Italian theoretical physicist and writer who has worked in Italy, the United States and since 2000, in France.',\n",
       "  '6_p2': 35,\n",
       "  '6_p1': None,\n",
       "  's2': 'Seven Brief Lessons on Physics is a short book by the Italian physicist Carlo Rovelli.',\n",
       "  's2_': 'Seven Brief Lessons on Physics is a short book by the Italian physicist Carlo Rovelli.',\n",
       "  'c': 'Carlo Rovelli is an Italian theoretical physicist and writer who has worked in Italy, the United States and since 2000, in France and he wrote Seven Brief Lessons on Physics which is a short book.',\n",
       "  'c-b': 'An Italian theoretical physicist and writer who has worked in Italy, the United States and since 2000, in France and he wrote Seven Brief Lessons on Physics which is a short book.',\n",
       "  'c-a': 'An Italian theoretical physicist and writer who has worked in Italy, the United States and since a certain year, in France and he wrote Seven Brief Lessons on Physics which is a short book.',\n",
       "  'q': 'An Italian theoretical physicist and writer who has worked in Italy, the United States and since what year, in France and he wrote Seven Brief Lessons on Physics which is a short book, what is his name?'},\n",
       " {'k': 35,\n",
       "  'p1': 'Livesey Hall War Memorial - The Livesey Hall War Memorial commemorates the fallen of World War I and World War II who had been employed by the South Suburban Gas Company of London. It is also a tribute to those employees who served in the wars. The monument was designed and executed by British sculptor Sydney March, of the March family of artists.',\n",
       "  'p2': 'World War II casualties - World War II was the deadliest military conflict in history in absolute terms of total casualties. Over 60 million people were killed, which was about 3% of the 1940 world population (est. 2.3 billion). The tables below give a detailed country-by-country count of human losses. World War II fatality statistics vary, with estimates of total deaths ranging from 50 million to more than 80 million. The higher figure of over 80 million includes deaths from war-related disease and famine. Civilians killed totalled 50 to 55 million, including 19 to 28 million from war-related disease and famine. Military deaths from all causes totalled 21 to 25 million, including deaths in captivity of about 5 million prisoners of war.',\n",
       "  'a': 'World War II',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'The Livesey Hal War Memorial commemorates the fallen of which war, that had over 60 million casualties?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'World War II',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 36,\n",
       "  '6_p2': 37,\n",
       "  's1': 'Livesey Hall War Memorial commemorates the fallen of World War II.',\n",
       "  's2': 'World War II was the deadliest military conflict in history in absolute terms of total casualties.',\n",
       "  's1_': 'Livesey Hall War Memorial commemorates the fallen of World War II.',\n",
       "  's2_': 'World War II was the deadliest military conflict in history in absolute terms of total casualties.',\n",
       "  'c': 'Livesey Hall War Memorial commemorates the fallen of World War II which was the deadliest military conflict in history in absolute terms of total casualties.',\n",
       "  'c-b': 'Livesey Hall War Memorial commemorates the fallen of the deadliest military conflict in history in absolute terms of total casualties.',\n",
       "  'c-a': 'Livesey Hall War Memorial commemorates the fallen of the deadliest military conflict in history in absolute terms of total casualties.',\n",
       "  'q': 'Which military conflict does Livesey Hall War Memorial commemorate in absolute terms of total casualties?'},\n",
       " {'k': 36,\n",
       "  'p1': 'Elko Regional Airport - Elko Regional Airport (IATA- EKO, ICAO- KEKO, FAA LID- EKO) , formerly Elko Municipal Airport, is a mile west of downtown Elko, in Elko County, Nevada.',\n",
       "  'p2': 'Gerald R. Ford International Airport - Gerald R. Ford International Airport (IATA- GRR, ICAO- KGRR, FAA LID- GRR) is a commercial airport in Cascade Township approximately 13 mi southeast of Grand Rapids, Michigan. The facility is owned by the Kent County Board of Commissioners and managed by an independent authority. The Federal Aviation Administration (FAA) National Plan of Integrated Airport Systems for 2017–2021 categorized it as a small hub primary commercial service facility.',\n",
       "  'a': 'no',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Are both Elko Regional Airport and Gerald R. Ford International Airport located in Michigan?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Elko Regional Airport is a mile west of downtown Elko, in Elko County, Nevada.',\n",
       "  's2_c': 'Gerald R. Ford International Airport is a small hub primary commercial service facility.',\n",
       "  'c_c': 'Gerald R. Ford International Airport is a small hub primary commercial service facility and Elko Regional Airport is not.',\n",
       "  'q_c': 'Are Gerald R. Ford International Airport and Elko Regional Airport both small hub primary commercial service facilities?'},\n",
       " {'k': 37,\n",
       "  'p1': 'Ralph Hefferline - Ralph Franklin Hefferline (15 February 1910 in Muncie, Indiana – 16 March 1974) was a psychology professor at Columbia University.',\n",
       "  'p2': \"Columbia University - Columbia University (Columbia; officially Columbia University in the City of New York), established in 1754, is a private Ivy League research university in Upper Manhattan, New York City, often cited as one of the world's most prestigious universities.\",\n",
       "  'a': 'New York City',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Ralph Hefferline was a psychology professor at a university that is located in what city?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Columbia University',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Ralph Hefferline was a psychology professor at Columbia University in New York City.',\n",
       "  '6_p1': 38,\n",
       "  '6_p2': 39,\n",
       "  's2': \"Columbia University is often cited as one of the world's prestigious universities.\",\n",
       "  's1_': 'Ralph Franklin Hefferline was a psychology professor at Columbia University.',\n",
       "  's2_': \"Columbia University is often cited as one of the world's prestigious universities.\",\n",
       "  'c': \"Ralph Hefferline was a psychology professor at Columbia University in New York City which is often cited as one of the world's prestigious universities.\",\n",
       "  'c-b': \"Ralph Hefferline was a psychology professor at a university in New York City which is often cited as one of the world's prestigious universities.\",\n",
       "  'c-a': \"Ralph Hefferline was a psychology professor at a university which is often cited as one of the world's prestigious universities.\",\n",
       "  'q': \"Ralph Hefferline was a psychology professor at a university which is often cited as one of the world's prestigious universities.\"},\n",
       " {'k': 38,\n",
       "  'p1': 'Manchester Terrier - The Manchester Terrier is a breed of dog of the smooth-haired terrier type.',\n",
       "  'p2': 'Scotch Collie - The Scotch Collie is a landrace breed of dog which originated from the highland regions of Scotland. The breed consisted of both the long-haired (now known as Rough) Collie and the short-haired (now known as Smooth) Collie. It is generally believed to have descended from a variety of ancient herding dogs, some dating back to the Roman occupation, which may have included Roman Cattle Dogs, Native Celtic Dogs and Viking Herding Spitzes. Other ancestors include the Gordon and Irish Setters.',\n",
       "  'a': 'Scotch Collie',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': \"Which dog's ancestors include Gordon and Irish Setters: the Manchester Terrier or the Scotch Collie?\",\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Manchester Terrier is a breed of dog of the smooth-haired terrier type.',\n",
       "  's2_c': 'Scotch Collie is a landrace breed of dog which originated from the highland of Scotland',\n",
       "  'c_c': 'Manchester Terrier and Scotch Collie are both breeds of dog.',\n",
       "  'q_c': 'Which breed of dog is more common, the Manchester Terrier or the Scotch Collie?'},\n",
       " {'k': 39,\n",
       "  'p1': 'Sachin Warrier - Sachin Warrier is a playback singer and composer in the Malayalam cinema industry from Kerala. He became notable with the song \"Muthuchippi Poloru\" from the film Thattathin Marayathu. He made his debut with the movie Malarvaadi Arts Club. He was working as a software engineer in Tata Consultancy Services in Kochi. Later he resigned from the job to concentrate more on music. His latest work is as a composer for the movie Aanandam.',\n",
       "  'p2': 'Tata Consultancy Services - Tata Consultancy Services Limited (TCS) is an Indian multinational information technology (IT) service, consulting and business solutions company Headquartered in Mumbai, Maharashtra. It is a subsidiary of the Tata Group and operates in 46 countries.',\n",
       "  'a': 'Mumbai',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Where is the company that Sachin Warrier worked for as a software engineer headquartered? ',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Tata Consultancy Services',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Tata Consultancy Services Limited is headquartered in Mumbai.',\n",
       "  '6_p1': 40,\n",
       "  '6_p2': None,\n",
       "  's1': 'Sachin Warrier was working as a software engineer in Tata Consultancy Services.',\n",
       "  's1_': 'Sachin Warrier was working as a software engineer in Tata Consultancy Services.',\n",
       "  'c': 'Sachin Warrier was working as a software engineer in Tata Consultancy Services which is headquartered in Mumbai.',\n",
       "  'c-b': 'Sachin Warrier was working as a software engineer in a company which is headquartered in Mumbai.',\n",
       "  'c-a': 'Sachin Warrier was working as a software engineer in a company which is headquartered in a certain city.',\n",
       "  'q': 'Sachin Warrier was working as a software engineer in a company which is headquartered in which city?'},\n",
       " {'k': 40,\n",
       "  'p1': 'Masakazu Katsura - Masakazu Katsura (桂 正和 , Katsura Masakazu , born December 10, 1962) is a Japanese manga artist, known for several works of manga, including \"Wing-man\", \"Shadow Lady\", \"DNA²\", \"Video Girl Ai\", \"I\"s\", and \"Zetman\". He has also worked as character designer for \"\", \"Tiger & Bunny\" and \"\".',\n",
       "  'p2': 'I&quot;s - I\"s (アイズ , Aizu ) is a Japanese manga series written and illustrated by Masakazu Katsura. The story\\'s main character is 16-year-old high school student Ichitaka Seto who is in love with his classmate Iori Yoshizuki, but too shy to tell her. Again and again he plans to tell her his true feelings, but each time something (usually a misunderstanding of some kind) gets in the way. Things become even more complicated when Itsuki Akiba returns to Japan; she is a girl Ichitaka was friends with in their childhood before she moved to the United States, and who had a huge crush on him.',\n",
       "  'a': '1962',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'A Japanese manga series based on a 16 year old high school student Ichitaka Seto, is written and illustrated by someone born in what year?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Masakazu Katsura',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Masakazu Katsura was born in 1962.',\n",
       "  '6_p2': 41,\n",
       "  '6_p1': None,\n",
       "  's2': 'I&quot;s - I\"s is a Japanese manga series illustrated by Masakazu Katsura.',\n",
       "  's2_': 'I&quot;s - I\"s is a Japanese manga series illustrated by Masakazu Katsura.',\n",
       "  'c': 'Masakazu Katsura was born in 1962 and I&quot;s - I\"s is a Japanese manga series illustrated by him.',\n",
       "  'c-b': 'A Japanese manga series illustrated by a certain person who was born in 1962.',\n",
       "  'c-a': 'A Japanese manga series illustrated by a certain person who was born in a certain year.',\n",
       "  'q': 'Who illustrated a Japanese manga series born in what year?'},\n",
       " {'k': 41,\n",
       "  'p1': 'Battle of Adwa - The Battle of Adwa (Amharic- አድዋ; Amharic translated- Adowa, or sometimes by the Italian name \"Adua\") was fought on 2 March 1896/Yekatit 23, 1889 according to Ethiopian calendar between the Ethiopian Empire and the Kingdom of Italy near the town of Adwa, Ethiopia, in Tigray. This climactic battle of the First Italo-Ethiopian War, was a decisive defeat for Italy and secured Ethiopian sovereignty.',\n",
       "  'p2': \"Giuseppe Arimondi - Giuseppe Edoardo Arimondi, OSML, OMS, OCI (Savigliano, 26 April 1846 – Adwa, 1 March 1896) was an Italian general, mostly known for his role during the First Italo-Ethiopian War. He was one of the few European commanders who gained a victory over the Mahdists before Kitchener's Expedition, soundly defeating them at Agordat in 1893. After a long and successfull colonial service, he died in combat at Adwa, and was posthumously awarded the Gold Medal of Military Valor.\",\n",
       "  'a': 'sovereignty',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'The battle in which Giuseppe Arimondi lost his life secured what for Ethiopia?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Adwa',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'The Battle of Adwa secured Ethiopian sovereignty.',\n",
       "  '6_p2': 42,\n",
       "  '6_p1': None,\n",
       "  's2': 'Giuseppe Arimondi died in Adwa.',\n",
       "  's2_': 'Giuseppe Arimondi died in Adwa.',\n",
       "  'c': 'Giuseppe Arimondi died in Adwa, in the Battle of Adwa which secured Ethiopian sovereignty.',\n",
       "  'c-b': 'Giuseppe Arimondi died in the Battle of Adwa which secured Ethiopian sovereignty.',\n",
       "  'c-a': 'Giuseppe Arimondi died in the Battle of Adwa which secured Ethiopian sovereignty.',\n",
       "  'q': 'Giuseppe Arimondi died in the Battle of Adwa which secured Ethiopian sovereignty?'},\n",
       " {'k': 42,\n",
       "  'p1': 'Nelson Rockefeller - Nelson Aldrich Rockefeller (July 8, 1908 – January 26, 1979) was an American businessman and politician. He served as the 41st Vice President of the United States from 1974 to 1977, and previously as the 49th Governor of New York (1959–1973). He also served as Assistant Secretary of State for American Republic Affairs for Presidents Franklin D. Roosevelt and Harry S. Truman as well as Under Secretary of Health, Education and Welfare under Dwight D. Eisenhower. A member of the wealthy Rockefeller family, he was also a noted art collector, as well as administrator of Rockefeller Center in Manhattan, New York.',\n",
       "  'p2': 'Alfred Balk - Alfred Balk (July 24, 1930 – November 25, 2010) was an American reporter, nonfiction author and magazine editor who wrote groundbreaking articles about housing segregation, the Nation of Islam, the environment and Illinois politics. His refusal to identify a confidential source led to a landmark court case. During a career-long emphasis on media improvement, he served on the Twentieth Century Fund\\'s task force that established a National News Council, consulted for several foundations, served as secretary of New York Governor Nelson Rockefeller\\'s Committee on the Employment of Minority Groups in the News Media, and produced a film, \"That the People Shall Know- The Challenge of Journalism\", narrated by Walter Cronkite. He wrote and co-authored books on a variety of topics, ranging from the tax exempt status of religious organizations to globalization to the history of radio.',\n",
       "  'a': 'Nelson Rockefeller',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Alfred Balk served as the secretary of the Committee on the Employment of Minority Groups in the News Media under which United States Vice President?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Nelson Rockefeller',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 43,\n",
       "  '6_p2': 44,\n",
       "  's1': 'Nelson Rockefeller served as the 41st Vice President of the United States.',\n",
       "  's2': \"Alfred Balk served as secretary of Nelson Rockefeller's Committee on the Employment of Minority Groups in the News Media.\",\n",
       "  's1_': 'Nelson Rockefeller served as the 41st Vice President of the United States.',\n",
       "  's2_': \"Alfred Balk served as secretary of Nelson Rockefeller's Committee on the Employment of Minority Groups in the News Media.\",\n",
       "  'c': \"Alfred Balk served as secretary of Nelson Rockefeller's Committee on the Employment of Minority Groups in the News Media who served as the 41st Vice President of the United States.\",\n",
       "  'c-b': 'Alfred Balk served as secretary of the 41st Vice President of the United States who served as the Committee on the Employment of Minority Groups in the News Media headed by the 41st Vice President of the United States.',\n",
       "  'c-a': 'Alfred Balk served as secretary of the 41st Vice President of the United States who served as the Committee on the Employment of Minority Groups in the News Media headed by a certain person.',\n",
       "  'q': 'Who served as the Committee on the Employment of Minority Groups in the News Media headed by Alfred Balk?'},\n",
       " {'k': 43,\n",
       "  'p1': 'Yellowcraigs - Yellowcraig, less commonly known as Broad Sands Bay, is a coastal area of forest, beach and grassland in East Lothian, south-east Scotland. Yellowcraig is partly within the Firth of Forth Site of Special Scientific Interest (SSSI). It is bordered to the north by the Firth of Forth, to the south by the village of Dirleton and Dirleton Castle, to the east by the North Berwick West Links golf course, and to the west by the Archerfield Estate and Links golf courses.',\n",
       "  'p2': 'Dirleton Castle - Dirleton Castle is a medieval fortress in the village of Dirleton, East Lothian, Scotland. It lies around 2 mi west of North Berwick, and around 19 mi east of Edinburgh. The oldest parts of the castle date to the 13th century, and it was abandoned by the end of the 17th century.',\n",
       "  'a': 'Yellowcraig',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'A medieval fortress in Dirleton, East Lothian, Scotland borders on the south side of what coastal area?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Dirleton Castle',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Yellowcraig is bordered to the south by Dirleton Castle.',\n",
       "  '6_p2': 45,\n",
       "  '6_p1': None,\n",
       "  's2': 'Dirleton Castle is a medieval fortress in the village of Dirleton, East Lothian, Scotland.',\n",
       "  's2_': 'Dirleton Castle is a medieval fortress in the village of Dirleton, East Lothian, Scotland.',\n",
       "  'c': 'Yellowcraig is bordered to the south by Dirleton Castle which is a medieval fortress in the village of Dirleton, East Lothian, Scotland.',\n",
       "  'c-b': 'Yellowcraig is bordered to the south by a medieval fortress in the village of Dirleton, East Lothian, Scotland.',\n",
       "  'c-a': 'A certain place is bordered to the south by a medieval fortress in the village of Dirleton, East Lothian, Scotland.',\n",
       "  'q': 'What place is bordered to the south by a medieval fortress in the village of Dirleton, East Lothian, Scotland?'},\n",
       " {'k': 44,\n",
       "  'p1': 'World Summit of Nobel Peace Laureates - The World Summit of Nobel Peace Laureates was initiated by Mikhail Gorbachev in the 90s, as a forum in which the Nobel Peace Laureates and the Peace Laureate Organizations could come together to address global issues with a view to encourage and support peace and human well being in the world. Its Permanent Secretariat is an independent, non-profit, ECOSOC non-governmental organization, based in Rome, operating on a permanent basis. A permanent staff, mainly composed of volunteers, promotes the work of the Nobel Peace Prize Winners and organizes the World Summit of Nobel Peace Laureates on a yearly basis. To date, the Permanent Secretariat has organized 16 Summits, the most recent having been held in February 2017 in the city of Bogota, Colombia. This was the first time that the Summit took place in Latin America.',\n",
       "  'p2': \"Mikhail Gorbachev - Mikhail Sergeyevich Gorbachev, ( ; Russian- Михаи́л Серге́евич Горбачёв ; ] ; born 2 March 1931) is a former Soviet statesman. He was the eighth and last leader of the Soviet Union, having been General Secretary of the Communist Party of the Soviet Union from 1985 until 1991. He was the country's head of state from 1988 until 1991 (titled as Chairman of the Presidium of the Supreme Soviet from 1988 to 1989, as Chairman of the from 1989 to 1990, and as Soviet Union]] from 1990 to 1991).\",\n",
       "  'a': 'Organizations could come together to address global issues',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What type of forum did a former Soviet statesman initiate?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Mikhail Gorbachev',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'The World Summit of Nobel Peace Laureates was initiated by Mikhail Gorbachev.',\n",
       "  '6_p2': 46,\n",
       "  '6_p1': None,\n",
       "  's2': 'Mikhail Gorbachev was the eighth and last leader of the Soviet Union.',\n",
       "  's2_': 'Mikhail Gorbachev was the eighth and last leader of the Soviet Union.',\n",
       "  'c': 'Mikhail Gorbachev was the eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates.',\n",
       "  'c-b': 'The eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates.',\n",
       "  'c-a': 'The eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates.',\n",
       "  'q': 'Who was the eighth and last leader of the Soviet Union who initiated The World Summit of Nobel Peace Laureates?'},\n",
       " {'k': 45,\n",
       "  'p1': 'Silene - Silene is a genus of flowering plants in the family Caryophyllaceae. Containing approximately 700 species, it is the largest genus in the family. Common names include campion (which is shared with the related genus \"Lychnis\") and catchfly. Many \"Silene\" species are widely distributed, particularly in the northern hemisphere.',\n",
       "  'p2': 'Ferocactus - Ferocactus is a genus of large barrel-shaped cacti, mostly with large spines and small flowers. There are about 30 species included in the genus. They are found in the southwestern United States and northwestern Mexico.',\n",
       "  'a': 'yes',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Are Ferocactus and Silene both types of plant?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Silene is a genus of flowering plants.',\n",
       "  's2_c': 'Ferocactus is a genus of cacti.',\n",
       "  'c_c': 'Both Silene and Ferocactus are types of plant.',\n",
       "  'q_c': 'Are Silene and Ferocactus types of plant?'},\n",
       " {'k': 46,\n",
       "  'p1': 'No. 2 Squadron RAAF - No. 2 Squadron is a Royal Australian Air Force (RAAF) squadron that operates from RAAF Base Williamtown, near Newcastle, New South Wales. From its formation in 1916 as part of the Australian Flying Corps, it has flown a variety of aircraft types including fighters, bombers, and Airborne Early Warning & Control (AEW&C). During World War I, the squadron operated on the Western Front conducting fighter sweeps and ground-attack missions. It was disbanded in mid-1919, following the end of hostilities. The squadron was briefly re-raised in 1922 as part of the newly independent RAAF, but was disbanded after only a couple of months and not reformed until 1937. It saw action as a bomber unit in the South West Pacific theatre of World War II and, equipped with English Electric Canberra jets, in the Malayan Emergency and the Vietnam War. The squadron was again disbanded in 1982, following the retirement of the Canberra. It was re-formed in 2000 to operate the Boeing 737 AEW&C \"Wedgetail\". One of the six Boeing 737s was deployed to the Middle East in September 2014, as part of Australia\\'s contribution to the military coalition against ISIS.',\n",
       "  'p2': \"English Electric Canberra - The English Electric Canberra is a British first-generation jet-powered medium bomber that was manufactured during the 1950s. It was developed by English Electric during the mid-to-late 1940s in response to a 1944 Air Ministry requirement for a successor to the wartime de Havilland Mosquito fast-bomber. Amongst the performance requirements for the type was the demand for an outstanding high altitude bombing capability in addition to flying at high speeds. These were partly accomplished by making use of newly developed jet propulsion technology. When the Canberra was introduced to service with the Royal Air Force (RAF), the type's first operator, in May 1951, it became the service's first jet-powered bomber aircraft.\",\n",
       "  'a': 'English Electric Canberra',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Which British first-generation jet-powered medium bomber was used in the South West Pacific theatre of World War II?',\n",
       "  'qtype': 'confused',\n",
       "  'b': 'Electric Canberra - - -  Squadron RAAF - No. 2 Squadron is a Royal Australian Air Force (RAAF) squadron that operates from RAAF Base Williamtown, near Newcastle, New South Wales. It was formed in 1916 as part of the Australian Flying Corps, it has flown a variety of aircraft types including fighters, bombers, and Airborne Early Warning & Control (AEW&C).',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 47,\n",
       "  '6_p2': 48,\n",
       "  's1': 'bleh',\n",
       "  's2': 'The English Electric Canberra is a British first-generation jet-powered medium bomber that was manufactured during the 1950s.',\n",
       "  's1_': 'bleh',\n",
       "  's2_': 'The English Electric Canberra is a British first-generation jet-powered medium bomber that was manufactured during the 1950s.',\n",
       "  'c': 'The English Electric Canberra is a British first-generation jet-powered medium bomber that was manufactured during the 1950s.',\n",
       "  'c-b': 'The English Electric Canberra is a British first-generation jet-powered medium bomber that was manufactured during the 1950s.',\n",
       "  'c-a': 'A certain aircraft is a British first-generation jet-powered medium bomber that was manufactured during the 1950s.',\n",
       "  'q': 'What aircraft is a British first-generation jet-powered medium bomber that was manufactured during the 1950s?',\n",
       "  's1_c': 'is',\n",
       "  's2_c': 'Electric Canberra - <extra_id_2>- <extra_id_3>- <extra_id_4> Squadron RAAF - No. 2 Squadron is a Royal Australian Air Force (RAAF) squadron that operates from RAAF Base Williamtown, near Newcastle, New South Wales. It was formed in 1916 as part of the Australian Flying Corps, it has flown a variety of aircraft types including fighters, bombers, and Airborne Early Warning & Control (AEW&C).',\n",
       "  'c_c': 'No. 2 Squadron RAAF is a Royal Australian Air Force (RAAF) squadron that operates from RAAF Base Williamtown, near Newcastle, New South Wales and it has flown a variety of aircraft types including fighters, bombers, and Airborne Early Warning & Control (AEW&C).',\n",
       "  'q_c': 'What is the aircraft type of No. 2 Squadron RAAF that operates from RAAF Base Williamtown, near Newcastle, New South Wales?'},\n",
       " {'k': 47,\n",
       "  'p1': '2009 Colorado Buffaloes football team - The 2009 Colorado Buffaloes football team represented the University of Colorado in the 2009 NCAA Division I FBS college football season. The Buffaloes were led by fourth year head coach Dan Hawkins and played their home games at Folsom Field. The Buffaloes finished the season with a record of 3–9 and 2–6 in Big 12 play.',\n",
       "  'p2': '2009 Big 12 Conference football season - The 2009 Big 12 Conference football season was the 14th season for the Big 12, as part of the 2009 NCAA Division I FBS football season.',\n",
       "  'a': '2009 Big 12 Conference',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Which year and which conference was the 14th season for this conference as part of the NCAA Division that the Colorado Buffaloes played in with a record of 2-6 in conference play?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': '2009 Big 12 Conference',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 49,\n",
       "  '6_p2': 50,\n",
       "  's1': 'The 2009 Colorado Buffaloes football team finished the season with a record of 3–9 and 2–6 in Big 12 play.',\n",
       "  's2': 'The 2009 Big 12 Conference football season was the 14th season for the Big 12.',\n",
       "  's1_': 'The 2009 Colorado Buffaloes football team finished the season with a record of 3–9 and 2–6 in Big 12 play.',\n",
       "  's2_': 'The 2009 Big 12 Conference football season was the 14th season for the Big 12.',\n",
       "  'c': 'The 2009 Colorado Buffaloes football team finished the season with a record of 3–9 and 2–6 in Big 12 play.',\n",
       "  'c-b': 'The 2009 Colorado Buffaloes football team finished the season with a record of 3–9 and 2–6 in Big 12 play.',\n",
       "  'c-a': 'The 2009 Colorado Buffaloes football team finished the season with a record of 3–9 and 2–6 in Big 12 play.',\n",
       "  'q': 'What is the 2009 Colorado Buffaloes football team’s record, including its record in Big 12 play?'},\n",
       " {'k': 48,\n",
       "  'p1': 'Carrefour - Carrefour S.A. (] ) is a French multinational retailer headquartered in Boulogne Billancourt, France, in the Hauts-de-Seine Department near Paris. It is one of the largest hypermarket chains in the world (with 1,462 hypermarkets at the end of 2016). Carrefour operates in more than 30 countries, in Europe, the Americas, Asia and Africa. Carrefour means \"crossroads\" and \"public square\" in French. The company is a component of the Euro Stoxx 50 stock market index.',\n",
       "  'p2': 'Euromarché - Euromarché (\"Euromarket\") was a French hypermarket chain. The first store opened in 1968 in Saint-Michel-sur-Orge. In June 1991, the group was rebought by its rival, Carrefour, for 5,2 billion francs.',\n",
       "  'a': '1,462',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'In 1991 Euromarché was bought by a chain that operated how any hypermarkets at the end of 2016?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Carrefour',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Carrefour is a French multinational retailer with 1,462 hypermarkets.',\n",
       "  '6_p2': 51,\n",
       "  '6_p1': None,\n",
       "  's2': 'Euromarché was rebought by Carrefour.',\n",
       "  's2_': 'Euromarché was rebought by Carrefour.',\n",
       "  'c': 'Carrefour is a French multinational retailer with 1,462 hypermarkets and it rebought Euromarché.',\n",
       "  'c-b': 'A French multinational retailer with 1,462 hypermarkets rebought Euromarché.',\n",
       "  'c-a': 'A French multinational retailer with a certain number of hypermarkets rebought Euromarché.',\n",
       "  'q': 'What is the number of hypermarkets owned by the retailer that rebought Euromarché?'},\n",
       " {'k': 49,\n",
       "  'p1': 'Indianapolis Motor Speedway - The Indianapolis Motor Speedway is an automobile racing circuit located in Speedway, Indiana, (an enclave suburb of Indianapolis) in the United States. It is the home of the Indianapolis 500 and the Brickyard 400, and formerly the home of the United States Grand Prix. It is located on the corner of 16th Street and Georgetown Road, approximately 6 mi west of Downtown Indianapolis.',\n",
       "  'p2': '1957 Indianapolis 500 - The 41st International 500-Mile Sweepstakes was held at the Indianapolis Motor Speedway on Thursday, May 30, 1957. The event was part of the 1957 USAC National Championship Trail and it was race 3 of 8 in the 1957 World Championship of Drivers.',\n",
       "  'a': 'Indianapolis Motor Speedway',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What race track in the midwest hosts a 500 mile race eavery May?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Indianapolis Motor Speedway',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 52,\n",
       "  '6_p2': 53,\n",
       "  's1': 'The Indianapolis Motor Speedway is an automobile racing circuit located in Speedway, Indiana, in the United States.',\n",
       "  's2': 'The 41st International 500-Mile Sweepstakes was held at the Indianapolis Motor Speedway.',\n",
       "  's1_': 'The Indianapolis Motor Speedway is an automobile racing circuit located in Speedway, Indiana, in the United States.',\n",
       "  's2_': 'The 41st International 500-Mile Sweepstakes was held at the Indianapolis Motor Speedway.',\n",
       "  'c': 'The Indianapolis Motor Speedway is an automobile racing circuit located in Speedway, Indiana, in the United States, where the 41st International 500-Mile Sweepstakes was held.',\n",
       "  'c-b': 'The Indianapolis Motor Speedway is an automobile racing circuit located in Speedway, Indiana, in the United States, where the 41st International 500-Mile Sweepstakes was held.',\n",
       "  'c-a': 'A certain automobile racing circuit is located in Speedway, Indiana, in the United States, where the 41st International 500-Mile Sweepstakes was held.',\n",
       "  'q': 'What automobile racing circuit is located in Speedway, Indiana, in the United States, where the 41st International 500-Mile Sweepstakes was held?'},\n",
       " {'k': 50,\n",
       "  'p1': 'Franco Corelli - Franco Corelli (8 April 1921 – 29 October 2003) was an Italian tenor who had a major international opera career between 1951 and 1976. Associated in particular with the spinto and dramatic tenor roles of the Italian repertory, he was celebrated universally for his powerhouse voice, electrifying top notes, clear timbre, passionate singing and remarkable performances. Dubbed the \"Prince of tenors\", Corelli possessed handsome features and a charismatic stage presence which endeared him to audiences. He had a long and fruitful partnership with the Metropolitan Opera in New York City between 1961 and 1975. He also appeared on the stages of most of the major opera houses in Europe and with opera companies throughout North America.',\n",
       "  'p2': 'Tosca (1956 film) - Tosca is a 1956 Italian musical film directed by Carmine Gallone and starring Franca Duval, Afro Poli and Franco Corelli. It is based on the opera \"Tosca\" by Giacomo Puccini. It was made at Cinecittà in Rome.',\n",
       "  'a': 'Rome',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'In what city did the \"Prince of tenors\" star in a film based on an opera by Giacomo Puccini?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Franco Corelli',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Tosca is a 1956 Italian musical film made in Rome starring Franco Corelli.',\n",
       "  '6_p1': 54,\n",
       "  '6_p2': None,\n",
       "  's1': 'Franco Corelli was an Italian tenor who had a major international opera career between 1951 and 1976.',\n",
       "  's1_': 'Franco Corelli was an Italian tenor who had a major international opera career between 1951 and 1976.',\n",
       "  'c': 'Tosca is a 1956 Italian musical film made in Rome starring Franco Corelli who was an Italian tenor who had a major international opera career between 1951 and 1976.',\n",
       "  'c-b': 'Tosca is a 1956 Italian musical film made in Rome starring an Italian tenor who had a major international opera career between 1951 and 1976.',\n",
       "  'c-a': 'Tosca is a 1956 Italian musical film made in a certain city starring an Italian tenor who had a major international opera career between 1951 and 1976.',\n",
       "  'q': 'Tosca is a 1956 Italian musical film made in which city starring an Italian tenor who had a major international opera career between 1951 and 1976?'},\n",
       " {'k': 51,\n",
       "  'p1': 'On My Mind (Ellie Goulding song) - \"On My Mind\" is a song by English singer Ellie Goulding from her third studio album \"Delirium\" (2015). It was released as the album\\'s lead single on 17 September 2015. It was written by Goulding, Max Martin, Savan Kotecha and Ilya Salmanzadeh. \"On My Mind\" is an electropop and R&B song whose instrumentation consists of scratchy guitars, trap drums, slapped beats and sharp, syncopated electronica. Lyrically, \"On My Mind\" talks about a one-night stand with someone the protagonist shouldn\\'t be with, having a dichotomy between heart and head. Though firmly denied by Goulding, many critics considered it an answer song to Ed Sheeran\\'s \"Don\\'t\".',\n",
       "  'p2': 'Delirium (Ellie Goulding album) - Delirium is the third studio album by English singer and songwriter Ellie Goulding, released on 6 November 2015 by Polydor Records. Music critics were generally impressed by the overall production of the record, although they were ambivalent in regards to its originality. It debuted at number three on the UK Albums Chart and the US \"Billboard\" 200, earning Goulding her highest-charting record in the latter country and her highest first-week sales figures in both territories. The album spawned three singles- \"On My Mind\", \"Army\" and \"Something in the Way You Move\".',\n",
       "  'a': 'Max Martin, Savan Kotecha and Ilya Salmanzadeh',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Ellie Goulding worked with what other writers on her third studio album, Delirium?',\n",
       "  'qtype': 'confused',\n",
       "  'b': '-      Ellie Goulding, 2015,   techa,  Ellie Goulding, 2015,  Ellie Goulding, 2015,  Ellie Goulding, 2015,    My Mind is a song by English singer Ellie Goulding from her third studio album \"Delirium\" (2015). It was written by Max Martin, Savan Kotecha and Ilya Sal',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': ('bleh', 'bleh'),\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': '\"On My Mind\" is a song by English singer Ellie Goulding from her third studio album \"Delirium\" (2015). It was written by Max Martin, Savan Kotecha and Ilya Salmanzadeh.',\n",
       "  '6_p1': 55,\n",
       "  '6_p2': 56,\n",
       "  's2': 'Delirium is the third studio album by English singer and songwriter Ellie Goulding, released on 6 November 2015.',\n",
       "  's1_': '\"On My Mind\" is a song by English singer Ellie Goulding from her third studio album \"Delirium\" (2015). It was written by Max Martin, Savan Kotecha and Ilya Salmanzadeh.',\n",
       "  's2_': 'Delirium is the third studio album by English singer and songwriter Ellie Goulding, released on 6 November 2015.',\n",
       "  'c': 'Delirium is the third studio album by English singer and songwriter Ellie Goulding, released on 6 November 2015, contains the song \"On My Mind\" written by Max Martin, Savan Kotecha and Ilya Salmanzadeh.',\n",
       "  'c-b': 'Delirium is the third studio album by English singer and songwriter Ellie Goulding, released on 6 November 2015, contains the song \"On My Mind\" written by Max Martin, Savan Kotecha and Ilya Salmanzadeh.',\n",
       "  'c-a': 'Delirium is the third studio album by English singer and songwriter Ellie Goulding, released on 6 November 2015, contains the song \"On My Mind\" written by a certain person.',\n",
       "  'q': 'Delirium is the third studio album by English singer and songwriter Ellie Goulding, released on 6 November 2015, contains the song \"On My Mind\" written by who?',\n",
       "  's1_c': 'My Mind\" is a song by English singer Ellie Goulding from her third studio album \"Delirium\" (2015). It was written by who?',\n",
       "  's2_c': 'ulding, Max Martin, <extra_id_2>  <extra_id_3>  <extra_id_4>  <extra_id_5>ding <extra_id_6>ding <extra_id_7>d <extra_id_8> My Mind\" is <extra_id_9>  <extra_id_10>, <extra_id_11>ulding <extra_id_12>ul <extra_id_13>ul <extra_id_14>ul <extra_id_15>ulding <extra_id_16>ulding <extra_id_17>ulding <extra_id_18>  <extra_id_19>ulding <extra_id_20>  <extra_id_21>s <extra_id_22>d <extra_id_23>s <extra_id_24>ding <extra_id_25>  <extra_id_26>s <extra_id_27> a',\n",
       "  'c_c': 'Go <extra_id_1>, Goulding, Goul <extra_id_2> Goulding, Goulding, Goulding, Goulding, Goulding, Goulding, Goul <extra_id_3> Goul <extra_id_4> Goulding, Goulding, Goulding, Goulding, Go <extra_id_16>, Go <extra_id_17>, Go <extra_id_18>, Goulding <extra_id_19> Go <extra_id_20>, Go <extra_id_21>, Goulding, Goulding, Goulding <extra_id_22> Goul <extra_id_23>ing <extra_id_24> Goul <extra_id_25>s <extra_id_26>',\n",
       "  'q_c': 'Who amongst Max Martin, Savan Kotecha and Ilya Salmanzadeh invented the word Goulding?'},\n",
       " {'k': 52,\n",
       "  'p1': 'Marion, South Australia - Marion is a suburb in the City of Marion in Adelaide around 10 km south-west of the CBD. Founded as a rural village in 1838 on the banks of the Sturt River, Marion was found to have rich soil and the population expanded rapidly. Colonel William Light laid out the plan for the village, as he had done with the City of Adelaide itself.',\n",
       "  'p2': 'Westminster School, Adelaide - Westminster School is an independent, Uniting Church, Early Learning to Year 12, coeducational, day and boarding school located at Marion, South Australia, 12 km south of Adelaide. Founded as a Methodist day and boarding school for boys, the school was opened by the Prime Minister Robert Menzies in 1961 and is named after Westminster School in London. The school became co-educational in 1978, and has a current enrolment of around 1150 students.',\n",
       "  'a': 'Marion, South Australia',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Which Australian city founded in 1838 contains a boarding school opened by a Prime Minister of Australia and named after a school in London of the same name.',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Marion, South Australia',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 57,\n",
       "  '6_p2': 58,\n",
       "  's1': 'Marion is a suburb in the City of Marion in Adelaide around 10 km south-west of the CBD.',\n",
       "  's2': 'Westminster School is located at Marion, South Australia.',\n",
       "  's1_': 'Marion is a suburb in the City of Marion in Adelaide around 10 km south-west of the CBD.',\n",
       "  's2_': 'Westminster School is located at Marion, South Australia.',\n",
       "  'c': 'Marion is a suburb in the City of Marion in Adelaide around 10 km south-west of the CBD and Westminster School is located at Marion, South Australia.',\n",
       "  'c-b': 'A suburb in the City of Marion in Adelaide around 10 km south-west of the CBD and Westminster School is located at a certain suburb.',\n",
       "  'c-a': 'A suburb in the City of Marion in Adelaide around 10 km south-west of the CBD and Westminster School is located at a certain suburb.',\n",
       "  'q': 'In what suburb in the City of Marion in Adelaide around 10 km south-west of the CBD is Westminster School located?'},\n",
       " {'k': 53,\n",
       "  'p1': 'Drifting (motorsport) - Drifting is a driving technique where the driver intentionally oversteers, with loss of traction in the rear wheels or all tires, while maintaining control and driving the car through the entirety of a corner. Car drifting is caused when the rear slip angle is greater than the front slip angle, to such an extent that often the front wheels are pointing in the opposite direction to the turn (e.g. car is turning left, wheels are pointed right or vice versa, also known as opposite lock or counter-steering).',\n",
       "  'p2': 'D1NZ - D1NZ is a production car drifting series in New Zealand. There is currently no New Zealand drifting championship so D1NZ is the premier series running as a clubsport advanced event under MotorSport New Zealand. It began early in 2003 as a small competition consisting of several drifting teams from all over the country in order to organise and regulate Drifting events on the circuit. Since then it has continued to grow larger and more professional, now involving professional drivers with sponsored cars. The series now includes two championships- Pro and Pro-sport. New competitors generally need to prove themselves in the Pro-sport championship before entering the Pro championship (unless coming from an equivalent international series). Organizers aim to promote and educate Drifting in New Zealand, with a regulated safe environment. Despite similar moniker, the series is not related to the Japanese D1 Grand Prix series. It has been recognised as the longest running drifting series in the world.',\n",
       "  'a': 'Drifting',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'D1NZ is a series based on what oversteering technique?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Drifting',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 59,\n",
       "  '6_p2': 60,\n",
       "  's1': 'Drifting is a driving technique where the driver intentionally oversteers.',\n",
       "  's2': 'D1NZ is a production car drifting series in New Zealand.',\n",
       "  's1_': 'Drifting is a driving technique where the driver intentionally oversteers.',\n",
       "  's2_': 'D1NZ is a production car drifting series in New Zealand.',\n",
       "  'c': 'D1NZ is a production car drifting series in New Zealand which is a driving technique where the driver intentionally oversteers.',\n",
       "  'c-b': 'D1NZ is a production car drifting series in New Zealand which is a driving technique where the driver intentionally oversteers.',\n",
       "  'c-a': 'D1NZ is a production car drifting series in New Zealand which is a driving technique where the driver intentionally oversteers.',\n",
       "  'q': 'D1NZ is a production car drifting series in New Zealand which is a driving technique where the driver intentionally oversteers?'},\n",
       " {'k': 54,\n",
       "  'p1': 'Jerry Glanville - Jerry Michael Glanville (born October 14, 1941) is a former American football player and coach, former NASCAR driver and owner, and sportscaster. He served as head coach of the Houston Oilers from 1986 to 1990 and the Atlanta Falcons from 1990 to 1994, compiling a career NFL record of 63–73. From 2007 to 2009, he was the Head Football Coach at Portland State University, tallying a mark of 9–24. Glanville has worked as an analyst on HBO\\'s \"Inside the NFL\", CBS\\'s \"The NFL Today\"/\"NFL on CBS\" and Fox\\'s coverage of the NFL. He has also raced on the Automobile Racing Club of America circuit. Glanville also briefly served as a consultant and liaison for the United Football League in 2011.',\n",
       "  'p2': 'Keith Bostic (American football) - William Keith Bostic (born January 17, 1961) is a former professional American football player who played for seven seasons in the National Football League (NFL). During his career he played safety for the Houston Oilers and the Cleveland Browns. Bostic served as the Oiler defensive captain under Jerry Glanville. He earned one Pro Bowl selection and missed another based on a tiebreaker for the last safety chosen. In his Pro Bowl season, he led the American Football Conference in interceptions.',\n",
       "  'a': 'Keith Bostic',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'who is younger Keith Bostic or Jerry Glanville ?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Jerry Glanville',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Keith Bostic served as the Oiler defensive captain under Jerry Glanville.',\n",
       "  '6_p1': 61,\n",
       "  '6_p2': None,\n",
       "  's1': 'Jerry Glanville is a former American football player and coach.',\n",
       "  's1_': 'Jerry Glanville is a former American football player and coach.',\n",
       "  'c': 'Keith Bostic served as the Oiler defensive captain under Jerry Glanville who is a former American football player and coach.',\n",
       "  'c-b': 'Keith Bostic served as the Oiler defensive captain under a former American football player and coach.',\n",
       "  'c-a': 'A certain player served as the Oiler defensive captain under a former American football player and coach.',\n",
       "  'q': 'Who served as the Oiler defensive captain under a former American football player and coach?'},\n",
       " {'k': 55,\n",
       "  'p1': 'Boston, Lincolnshire - Boston ( ) is a town and small port in Lincolnshire, on the east coast of England. It is the largest town of the wider Borough of Boston local government district. The borough had a total population of 66,900, at the ONS mid 2015 estimates, while the town itself had a population of 35,124 at the 2001 census. It is due north of Greenwich on the Prime Meridian.',\n",
       "  'p2': 'Kirton End - Kirton End is a hamlet in the civil parish of Kirton in the Boston district of Lincolnshire, England. It lies on the B1391 road, 4 mi south-west from Boston, and 1.5 mi north-east from Kirton.',\n",
       "  'a': '35,124',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'According to the 2001 census, what was the population of the city in which Kirton End is located?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Boston',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Boston has a population of 35,124.',\n",
       "  '6_p2': 62,\n",
       "  '6_p1': None,\n",
       "  's2': 'Kirton End is a hamlet in the civil parish of Kirton in the Boston district of Lincolnshire, England.',\n",
       "  's2_': 'Kirton End is a hamlet in the civil parish of Kirton in the Boston district of Lincolnshire, England.',\n",
       "  'c': 'Boston has a population of 35,124 and Kirton End is a hamlet in the civil parish of Kirton in the Boston district of Lincolnshire, England.',\n",
       "  'c-b': 'A place with a population of 35,124 is in the Lincolnshire, England district.',\n",
       "  'c-a': 'A place with a certain population is in the Lincolnshire, England district.',\n",
       "  'q': 'What place with a certain population is in the Lincolnshire, England district?'},\n",
       " {'k': 56,\n",
       "  'p1': 'Ajuga - Ajuga , also known as bugleweed, ground pine, carpet bugle, or just bugle, is a genus of 40 species annual and perennial herbaceous flowering plantsin the mint family Lamiaceae, with most species native to Europe, Asia, and Africa, but also two species in southeastern Australia. They grow to 5–50 cm tall, with opposite leaves.',\n",
       "  'p2': 'Cypress - Cypress is a conifer tree or shrub of northern temperate regions that belongs to the family Cupressaceae. The word \"cypress\" is derived from Old French \"cipres\", which was imported from Latin \"cypressus\", the latinisation of the Greek κυπάρισσος (\"kyparissos\").',\n",
       "  'a': 'no',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Are both Cypress and Ajuga genera?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Ajuga is a genus of 40 species annual and perennial herbaceous flowering plants.',\n",
       "  's2_c': 'Cypress is a conifer tree or shrub of northern temperate climates.',\n",
       "  'c_c': 'Ajuga and Cypress are both types of plant.',\n",
       "  'q_c': 'Are Ajuga and Cypress both types of plant?'},\n",
       " {'k': 57,\n",
       "  'p1': '1992–93 Charlotte Hornets season - The 1992–93 NBA season was the fifth season for the Charlotte Hornets in the National Basketball Association. In the 1992 NBA draft, the Hornets selected center Alonzo Mourning out of Georgetown with the second overall pick. With the addition of Mourning, along with second-year star Larry Johnson and Muggsy Bogues, the Hornets struggled around .500 for most of the season, but won 9 of their final 12 games finishing their season third in the Central Division with a 44–38 record, and qualified for their first ever playoff appearance. The team became the first of the four expansion franchises of the late 1980s to win a playoff series, when Mourning sunk a 20 ft jumper at the buzzer in Game 4 of the first round playoff series against the 4th-seeded Boston Celtics. However, in the semifinals, the Hornets were eliminated by the New York Knicks in five games.',\n",
       "  'p2': 'Muggsy Bogues - Tyrone Curtis \"Muggsy\" Bogues (born January 9, 1965) is an American retired basketball player. The shortest player ever to play in the National Basketball Association, the 5 ft Bogues played point guard for four teams during his 14-season career in the NBA. He is best known for his ten seasons with the Charlotte Hornets, and also played for the Washington Bullets, Golden State Warriors, and Toronto Raptors. After his NBA career, he served as head coach of the now-defunct WNBA team Charlotte Sting.',\n",
       "  'a': 'shortest player ever to play in the National Basketball Association',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What distinction is held by the former NBA player who was a member of the Charlotte Hornets during their 1992-93 season and was head coach for the WNBA team Charlotte Sting?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Muggsy Bogues',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Muggsy Bogues is the shortest player ever to play in the National Basketball Association.',\n",
       "  '6_p1': 63,\n",
       "  '6_p2': None,\n",
       "  's1': 'Muggsy Bogues played for the 1992–93 Charlotte Hornets season.',\n",
       "  's1_': 'Muggsy Bogues played for the 1992–93 Charlotte Hornets season.',\n",
       "  'c': 'Muggsy Bogues is the shortest player ever to play in the National Basketball Association, he played for the 1992–93 Charlotte Hornets season.',\n",
       "  'c-b': 'The shortest player ever to play in the National Basketball Association, he played for the 1992–93 Charlotte Hornets season.',\n",
       "  'c-a': 'The shortest player ever to play in the National Basketball Association, he played for the 1992–93 Charlotte Hornets season.',\n",
       "  'q': 'Who is the shortest player ever to play in the National Basketball Association, who played for the 1992–93 Charlotte Hornets season?'},\n",
       " {'k': 58,\n",
       "  'p1': 'Alien (soundtrack) - The iconic, avant-garde score to the film \"Alien\" was composed by Jerry Goldsmith and is considered by some to be one of his best, most visceral scores. Rather than focusing on themes, Goldsmith creates a bleak and dissonant soundscape that fits the film\\'s dark and intense atmosphere, with only a few \"romantic\" cues.',\n",
       "  'p2': \"Alien (film) - Alien is a 1979 science-fiction horror film directed by Ridley Scott, and starring Sigourney Weaver, Tom Skerritt, Veronica Cartwright, Harry Dean Stanton, John Hurt, Ian Holm and Yaphet Kotto. The film's title refers to a highly aggressive extraterrestrial creature that stalks and attacks the crew of a spaceship. Dan O'Bannon, drawing upon previous works of science fiction and horror, wrote the screenplay from a story he co-authored with Ronald Shusett. The film was produced by Gordon Carroll, David Giler and Walter Hill through their company Brandywine Productions, and was distributed by 20th Century Fox. Giler and Hill revised and made additions to the script. Shusett was executive producer. The eponymous Alien and its accompanying elements were designed by the Swiss artist H. R. Giger, while concept artists Ron Cobb and Chris Foss designed the more human aspects of the film.\",\n",
       "  'a': 'Ronald Shusett',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What is the name of the executive producer of the film that has a score composed by Jerry Goldsmith?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Alien',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': \"Alien is a 1979 science-fiction horror film written by Dan O'Bannon, drawing upon previous works of science fiction and horror, and executive produced by Ronald Shusett.\",\n",
       "  '6_p1': 64,\n",
       "  '6_p2': None,\n",
       "  's1': 'The iconic, avant-garde score to the film \"Alien\" was composed by Jerry Goldsmith.',\n",
       "  's1_': 'The iconic, avant-garde score to the film \"Alien\" was composed by Jerry Goldsmith.',\n",
       "  'c': \"Alien is a 1979 science-fiction horror film written by Dan O'Bannon, drawing upon previous works of science fiction and horror, and executive produced by Ronald Shusett, and its iconic, avant-garde score was composed by Jerry Goldsmith.\",\n",
       "  'c-b': \"A 1979 science-fiction horror film written by Dan O'Bannon, drawing from previous works of science fiction and horror, and executive produced by Ronald Shusett, and its iconic, avant-garde score was composed by Jerry Goldsmith.\",\n",
       "  'c-a': \"A 1979 science-fiction horror film written by Dan O'Bannon, drawing from previous works of science fiction and horror, and executive produced by a certain person, and its iconic, avant-garde score was composed by Jerry Goldsmith.\",\n",
       "  'q': \"Who executive produced a 1979 science-fiction horror film written by Dan O'Bannon, drawing from previous works of science fiction and horror, and whose iconic, avant-garde score was composed by Jerry Goldsmith?\"},\n",
       " {'k': 59,\n",
       "  'p1': 'Emma Bull - Emma Bull (born December 13, 1954) is an American science fiction and fantasy author. Her novels include the Hugo- and Nebula-nominated \"Bone Dance\" and the urban fantasy \"War for the Oaks\". She is also known for a series of anthologies set in Liavek, a shared universe that she created with her husband Will Shetterly. As a singer, songwriter, and guitarist, she has been a member of the Minneapolis-based folk/rock bands Cats Laughing and The Flash Girls.',\n",
       "  'p2': \"Virginia Woolf - Adeline Virginia Woolf (née Stephen; 25 January 1882 – 28 March 1941) was an English writer who is considered one of the foremost modernists of the twentieth century, and a pioneer in the use of stream of consciousness as a narrative device. Born in an affluent household in Kensington, London, she attended the King's College London and was acquainted with the early reformers of women's higher education.\",\n",
       "  'a': 'Adeline Virginia Woolf',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Who was born earlier, Emma Bull or Virginia Woolf?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Emma Bull is an American science fiction and fantasy author.',\n",
       "  's2_c': 'Virginia Woolf was a pioneer of the use of stream of consciousness as a narrative device.',\n",
       "  'c_c': 'Emma Bull is an American science fiction and fantasy author and Virginia Woolf was a pioneer in the use of stream of consciousness as a narrative device.',\n",
       "  'q_c': 'Who amongst Virginia Woolf and Emma Bull was a pioneer in the use of stream of consciousness as a narrative device?'},\n",
       " {'k': 60,\n",
       "  'p1': 'What Are Little Boys Made Of? - \"What Are Little Boys Made Of?\" is a popular nursery rhyme dating from the early 19th century. It has a Roud Folk Song Index number of 821.',\n",
       "  'p2': 'What Are Little Girls Made Of? - \"What Are Little Girls Made Of?\" is episode seven of the of the American science fiction television series, \"\". It was first broadcast October 20, 1966, and repeated, two months later, on December 22, the first episode of the series to be repeated on NBC. It was written by Robert Bloch and directed by James Goldstone. The title of the episode is taken from the fourth line of the 19th century nursery rhyme, \"What Are Little Boys Made Of?.\"',\n",
       "  'a': '821',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What was the Roud Folk Song Index of the nursery rhyme inspiring What Are Little Girls Made Of?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'What Are Little Boys Made Of?',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'What Are Little Boys Made Of? has Roud Folk Song Index number 821.',\n",
       "  '6_p2': 65,\n",
       "  '6_p1': None,\n",
       "  's2': 'What Are Little Girls Made Of? is an episode of the television series \"\".',\n",
       "  's2_': 'What Are Little Girls Made Of? is an episode of the television series \"\".',\n",
       "  'c': 'What Are Little Girls Made Of? is an episode of the television series and it has Roud Folk Song Index number 821.',\n",
       "  'c-b': 'What Are Little Girls Made Of? is an episode of the television series and it has Roud Folk Song Index number 821.',\n",
       "  'c-a': 'What Are Little Girls Made Of? is an episode of the television series and it has Roud Folk Song Index number 821.',\n",
       "  'q': 'What Are Little Girls Made Of? is an episode of the television series and it has Roud Folk Song Index number 821?'},\n",
       " {'k': 61,\n",
       "  'p1': 'Scott Parkin - Scott Parkin (born 1969, Garland, Texas is an anti-war, environmental and global justice organizer, former community college history instructor, and a founding member of the Houston Global Awareness Collective. He has been a vocal critic of the American invasion of Iraq, and of corporations such as Exxonmobil and Halliburton. Since 2006, he has worked as an campaigner for the Rainforest Action Network, organizing campaigns against Bank of America, Citibank, TXU and the Keystone XL Pipeline. He also organizes with Rising Tide North America.',\n",
       "  'p2': \"Halliburton - The Halliburton Company, an American multinational corporation. One of the world's largest oil field service companies, it has operations in more than 70 countries. It owns hundreds of subsidiaries, affiliates, branches, brands, and divisions worldwide and employs approximately 50,000 people.\",\n",
       "  'a': 'more than 70 countries',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Scott Parkin has been a vocal critic of Exxonmobil and another corporation that has operations in how many countries ?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Halliburton',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Halliburton has operations in more than 70 countries.',\n",
       "  '6_p1': 66,\n",
       "  '6_p2': None,\n",
       "  's1': 'Scott Parkin has been a vocal critic of the American invasion of Iraq, and of corporations such as Halliburton.',\n",
       "  's1_': 'Scott Parkin has been a vocal critic of the American invasion of Iraq, and of corporations such as Halliburton.',\n",
       "  'c': 'Scott Parkin has been a vocal critic of the American invasion of Iraq, and of corporations such as Halliburton which has operations in more than 70 countries.',\n",
       "  'c-b': 'Scott Parkin has been a vocal critic of the American invasion of Iraq, and of corporations which have operations in more than 70 countries.',\n",
       "  'c-a': 'Scott Parkin has been a vocal critic of the American invasion of Iraq, and of corporations which have operations in a certain number of countries.',\n",
       "  'q': 'Scott Parkin has been a vocal critic of the American invasion of Iraq, and of corporations which have operations in how many countries?'},\n",
       " {'k': 62,\n",
       "  'p1': 'Rose McGowan - Rose Arianna McGowan (born September 5, 1973) is an Italian-born American actress, film producer, director and singer. She is best known to television audiences for having played Paige Matthews in The WB supernatural drama series \"Charmed\" from 2001 to 2006.',\n",
       "  'p2': 'Jawbreaker (film) - Jawbreaker is a 1999 American black comedy film written and directed by Darren Stein. The film stars Rose McGowan, Rebecca Gayheart, and Julie Benz as girls in an exclusive clique in their high school. Charlotte Ayanna has a non-speaking cameo role as the murdered leader of the group. The film was inspired by the film \"Heathers\", and is often compared to it, particularly the plot involving a popular female clique, and the ostensibly accidental killing of one of its members.',\n",
       "  'a': 'Charmed',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What WB supernatrual drama series was Jawbreaker star Rose Mcgowan best known for being in?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Rose McGowan',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Rose McGowan is best known for having played Paige Matthews in The WB supernatural drama series \"Charmed\"',\n",
       "  '6_p2': 67,\n",
       "  '6_p1': None,\n",
       "  's2': 'Jawbreaker is a 1999 American black comedy film starring Rose McGowan.',\n",
       "  's2_': 'Jawbreaker is a 1999 American black comedy film starring Rose McGowan.',\n",
       "  'c': 'Rose McGowan is best known for having played Paige Matthews in The WB supernatural drama series \"Charmed\" and Jawbreaker is a 1999 American black comedy film starring her.',\n",
       "  'c-b': 'The actor best known for having played Paige Matthews in The WB supernatural drama series \"Charmed\" and Jawbreaker is a 1999 American black comedy film starring her.',\n",
       "  'c-a': 'The actor best known for having played Paige Matthews in The WB supernatural drama series \"Charmed\" and Jawbreaker is a 1999 American black comedy film starring her.',\n",
       "  'q': 'Who is the actor best known for having played Paige Matthews in The WB supernatural drama series \"Charmed\" and Jawbreaker is a 1999 American black comedy film starring her?'},\n",
       " {'k': 63,\n",
       "  'p1': 'Vince Phillips - Vincent Edwards \"Vince\" Phillips (born July 23, 1963) is an American former professional boxer who competed from 1989 to 2007. He held the IBF junior welterweight title from 1997 to 1999, notably scoring an upset victory against then-undefeated Kostya Tszyu to become champion.',\n",
       "  'p2': 'International Boxing Federation - The International Boxing Federation (IBF) is one of four major organizations recognized by the International Boxing Hall of Fame (IBHOF) which sanction world championship boxing bouts, alongside the World Boxing Association (WBA), World Boxing Council (WBC) and World Boxing Organization (WBO).',\n",
       "  'a': 'International Boxing Hall of Fame',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Vince Phillips held a junior welterweight title by an organization recognized by what larger Hall of Fame?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'International Boxing Federation',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Vince Phillips held the IBF junior welterweight title.',\n",
       "  '6_p1': 68,\n",
       "  '6_p2': 69,\n",
       "  's2': 'The International Boxing Federation (IBF) is one of four major organizations recognized by the International Boxing Hall of Fame (IBHOF) which sanction world championship boxing bouts.',\n",
       "  's1_': 'Vince Phillips held the IBF junior welterweight title.',\n",
       "  's2_': 'The International Boxing Federation (IBF) is one of four major organizations recognized by the International Boxing Hall of Fame (IBHOF) which sanction world championship boxing bouts.',\n",
       "  'c': 'Vince Phillips held the IBF junior welterweight title which is one of four major organizations recognized by the International Boxing Hall of Fame (IBHOF) which sanction world championship boxing bouts.',\n",
       "  'c-b': 'Vince Phillips held the junior welterweight title which is one of four major organizations recognized by the International Boxing Hall of Fame which sanction world championship boxing bouts.',\n",
       "  'c-a': 'Vince Phillips held the junior welterweight title which is one of four major organizations recognized by a certain entity which sanction world championship boxing bouts.',\n",
       "  'q': 'Which entity recognizes Vince Phillips’ junior welterweight title which is held by one of four major organizations which sanction world championship boxing bouts?'},\n",
       " {'k': 64,\n",
       "  'p1': 'I Don\\'t Wanna Know - \"I Don\\'t Wanna Know\" is a song by American R&B artist Mario Winans, featuring a rap by P. Diddy, re-recorded background vocals by Enya and sampling the synthesizer riff from Enya\\'s song \"Boadicea\". It reached number 1 in the United Kingdom and in Netherlands, and number 2 in the United States, stuck for eight consecutive weeks behind the Usher singles \"Yeah\" and \"Burn\", respectively. It remains Winans\\' only major success to date. As a result of its eight weeks at number 2, \"I Don\\'t Wanna Know\" stands in joint fifth place for the most weeks at number 2 by a song which did not go to number 1, behind Foreigner\\'s \"Waiting for a Girl Like You\", Missy Elliott\\'s \"Work It\", Donna Lewis\\'s \"I Love You Always Forever\" and Shania Twain\\'s \"You\\'re Still the One\".',\n",
       "  'p2': 'Yeah! (Usher song) - \"Yeah!\" is a song by American singer Usher. He co-wrote the song with Sean Garrett, Patrick J. Que Smith, Robert McDowell, LRoc, Ludacris, Da\\'Ron, and Lil Jon. It also features guest vocals from Lil Jon and Ludacris, with the former also producing the song as well as incorporating crunk and R&B—which he coined as crunk&B—in the song\\'s production. The song was released as the lead single from Usher\\'s fourth studio album \"Confessions\" (2004) on January 27, 2004, after Usher was told by Arista Records, his label at the time, to record more tracks for the album.',\n",
       "  'a': 'Usher',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What is the name of the singer who\\'s song was released as the lead single from the album \"Confessions\", and that had popular song stuck behind for eight consecutive weeks?',\n",
       "  'qtype': 'confused',\n",
       "  'b': 'bleh',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': '\"I Don\\'t Wanna Know\" is a song by American R&B artist Mario Winans, featuring a rap by P. Diddy, re-recorded background vocals by Enya and sampling the synthesizer riff from Enya\\'s song \"Boadicea\" which was stuck for eight consecutive weeks behind the Usher singles \"Yeah\" and \"Burn\".',\n",
       "  '6_p1': 70,\n",
       "  '6_p2': 71,\n",
       "  's2': 'Yeah! is a song by Usher.',\n",
       "  's1_': 'I Don\\'t Wanna Know is a song by Mario Winans, featuring a rap by P. Diddy, re-recorded background vocals by Enya and sampling the synthesizer riff from Enya\\'s song \"Boadicea\".',\n",
       "  's2_': 'Yeah! is a song by Usher.',\n",
       "  'c': '\"I Don\\'t Wanna Know\" is a song by American R&B artist Mario Winans, featuring a rap by P. Diddy, re-recorded background vocals by Enya and sampling the synthesizer riff from Enya\\'s song \"Boadicea\" which was stuck for eight consecutive weeks behind the Usher singles \"Yeah!\" and \"Burn\".',\n",
       "  'c-b': '\"I Don\\'t Wanna Know\" is a song by American R&B artist Mario Winans, featuring a rap by P. Diddy, re-recorded background vocals by Enya and sampling the synthesizer riff from Enya\\'s song \"Boadicea\" which was stuck for eight consecutive weeks behind the Usher singles \"Yeah!\" and \"Burn\".',\n",
       "  'c-a': '\"I Don\\'t Wanna Know\" is a song by American R&B artist Mario Winans, featuring a rap by P. Diddy, re-recorded background vocals by Enya and sampling the synthesizer riff from Enya\\'s song \"Boadicea\" which was stuck for eight consecutive weeks behind the Usher singles \"Yeah!\" and \"Burn\".',\n",
       "  'q': '\"I Don\\'t Wanna Know\" is a song by American R&B artist Mario Winans, featuring a rap by P. Diddy, re-recorded background vocals by Enya and sampling the synthesizer riff from Enya\\'s song \"Boadicea\" which was stuck for eight consecutive weeks behind the Usher singles \"Yeah!\" and \"Burn\"?',\n",
       "  's1_c': 'bleh',\n",
       "  's2_c': 'bleh',\n",
       "  'c_c': 'Both bleh and a sneeze are considered to be unpleasant.',\n",
       "  'q_c': 'Who amongst the following is considered to be unpleasant, the usher or the bleh?'},\n",
       " {'k': 65,\n",
       "  'p1': 'Brian Doyle-Murray - Brian Doyle-Murray (born Brian Murray, October 31, 1945) is an American actor, voice artist, comedian and screenwriter. He is the older brother of actor/comedian Bill Murray, and the two have acted together in several films, including \"Caddyshack\", \"Scrooged\", \"Ghostbusters II\", \"The Razor\\'s Edge\", and \"Groundhog Day\". He co-starred on the TBS sitcom on \"Sullivan & Son,\" where he played the foul-mouthed Hank Murphy. he also appeared in the Cartoon Network original animated series \"The Marvelous Misadventures of Flapjack\" as the surly Captain K\\'Nuckles and a pirate ghost, The Flying Dutchman from the Nickelodeon animated series, \"SpongeBob SquarePants\", he appears in a recurring role as Don Ehlert on the ABC sitcom \"The Middle\".',\n",
       "  'p2': 'The Hard Easy (Adventure Time) - \"The Hard Easy\" is the twenty-third episode of the fourth season of the American animated television series \"Adventure Time\". The episode was written and storyboarded by Tom Herpich and Skyler Page, from a story by Patrick McHale, Kent Osborne, and Pendleton Ward. It originally aired on Cartoon Network on October 1, 2012. The episode guest stars Brian Doyle-Murray as Prince Huge and Jonathan Katz as the Mudscamp elder.',\n",
       "  'a': 'Bill Murray',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'who is the younger brother of The episode guest stars of The Hard Easy ',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Brian Doyle-Murray',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Brian Doyle-Murray is the older brother of Bill Murray.',\n",
       "  '6_p1': 72,\n",
       "  '6_p2': 73,\n",
       "  's2': 'The Hard Easy is the twenty-third episode of the fourth season of the American animated television series \"Adventure Time\" guest stars Brian Doyle-Murray.',\n",
       "  's1_': 'Brian Doyle-Murray is an actor, voice artist, comedian and screenwriter.',\n",
       "  's2_': 'The Hard Easy is the twenty-third episode of the fourth season of the American animated television series \"Adventure Time\" guest stars Brian Doyle-Murray.',\n",
       "  'c': 'Brian Doyle-Murray is the older brother of Bill Murray and The Hard Easy is the twenty-third episode of the fourth season of the American animated television series \"Adventure Time\" guest stars Brian Doyle-Murray.',\n",
       "  'c-b': 'The Hard Easy is the twenty-third episode of the fourth season of the American animated television series \"Adventure Time\" guest stars the older brother of Bill Murray.',\n",
       "  'c-a': 'The Hard Easy is the twenty-third episode of the fourth season of the American animated television series \"Adventure Time\" guest stars the older brother of a certain person.',\n",
       "  'q': 'The Hard Easy is the twenty-third episode of the fourth season of the American animated television series \"Adventure Time\" guest stars the older brother of who?'},\n",
       " {'k': 66,\n",
       "  'p1': \"2017–18 Wigan Athletic F.C. season - The 2017–18 season is Wigan Athletic's 86th year in existence and their first back in League One, after being relegated the previous season. Along with competing in the league, the club will also participate in the FA Cup, EFL Cup and EFL Trophy. Paul Cook was appointed as manager on 31 May 2017, signing a 3 year deal, following the departure of much of the previous coaching staff the day before.\",\n",
       "  'p2': \"EFL Cup - The EFL Cup (referred to historically, and colloquially, as simply the League Cup), currently known as the Carabao Cup for sponsorship reasons, is an annual knockout football competition in men's domestic English football. Organised by the English Football League (EFL), it is open to any club within the top four levels of the English football league system – 92 clubs in total – comprising the top level Premier League, and the three divisions of the English Football League's own league competition (Championship, League One and League Two).\",\n",
       "  'a': 'Carabao Cup',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'The 2017–18 Wigan Athletic F.C. season will be a year in which the team competes in the league cup known as what for sponsorship reasons?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'EFL Cup',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Wigan Athletic will participate in the EFL Cup.',\n",
       "  '6_p1': 74,\n",
       "  '6_p2': 75,\n",
       "  's2': \"The EFL Cup is an annual knockout football competition in men's domestic English football.\",\n",
       "  's1_': 'Wigan Athletic will participate in the EFL Cup in 2017–18 season.',\n",
       "  's2_': \"The EFL Cup is an annual knockout football competition in men's domestic English football.\",\n",
       "  'c': \"Wigan Athletic will participate in the EFL Cup which is an annual knockout football competition in men's domestic English football.\",\n",
       "  'c-b': \"Wigan Athletic will participate in an annual knockout football competition in men's domestic English football.\",\n",
       "  'c-a': \"Wigan Athletic will participate in an annual knockout football competition in men's domestic English football.\",\n",
       "  'q': \"What is the annual knockout football competition in men's domestic English football that Wigan Athletic will participate in?\"},\n",
       " {'k': 67,\n",
       "  'p1': 'Tara Strong - Tara Strong (born Tara Lyn Charendoff; February 12, 1973) is a Canadian–American actress who has done voice work for numerous animations and video games and performed in various live-action productions. Many of her major voice roles include animated series such as \"Rugrats\", \"The Powerpuff Girls\", \"The Fairly OddParents\", \"Drawn Together\", \"Teen Titans\" and the spin-off series \"Teen Titans Go!\", and \"\", as well as video games such as \"Mortal Kombat X\", \"Final Fantasy X-2\", and the \"\" series. Her portrayals have garnered nominations in the Annie Awards and Daytime Emmys, and an award from the Academy of Interactive Arts & Sciences.',\n",
       "  'p2': 'Teen Titans Go! (TV series) - Teen Titans Go! is an American animated television series based on the DC Comics fictional superhero team, the \"Teen Titans\". The series was announced following the popularity of DC Nation\\'s New Teen Titans shorts.',\n",
       "  'a': 'Teen Titans Go!',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Which of Tara Strong major voice role in animated series is an American animated television series based on the DC Comics fictional superhero team, the \"Teen Titans\"?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Tara Strong',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Tara Strong has done voice work for Teen Titans Go!',\n",
       "  '6_p1': 76,\n",
       "  '6_p2': 77,\n",
       "  's2': 'Tara Strong is a voice actor in Teen Titans Go!',\n",
       "  's1_': 'Tara Strong is an actress who has done voice work for numerous animations and video games.',\n",
       "  's2_': 'Tara Strong is a voice actor in Teen Titans Go!',\n",
       "  'c': 'Tara Strong is a voice actor in Teen Titans Go! who has done voice work for the show Teen Titans Go!',\n",
       "  'c-b': 'A voice actor in Teen Titans Go! who has done voice work for the show Teen Titans Go!',\n",
       "  'c-a': 'A voice actor in a certain show who has done voice work for a certain show.',\n",
       "  'q': 'What show has a voice actor who has done voice work for it?'},\n",
       " {'k': 68,\n",
       "  'p1': '122nd SS-Standarte - The 122nd SS-Standarte was a regimental command of the Allgemeine-SS that was formed in the city of Strasbourg during World War II. The Standarte was activated on November 12, 1940, and reached battalion strength by the end of the year. The command was a successor to the previously disbanded 121st SS-Standarte, also situated in Strasbourg.',\n",
       "  'p2': 'Strasbourg - Strasbourg ( , ] ; Alsatian- \"Strossburi\"; German- \"Straßburg\" ] ) is the capital and largest city of the Grand Est region of France and is the official seat of the European Parliament. Located close to the border with Germany in the historic region of Alsace, it is the capital of the Bas-Rhin département. In 2014, the city proper had 276,170 inhabitants and both the Eurométropole de Strasbourg (Greater Strasbourg) and the Arrondissement of Strasbourg had 484,157 inhabitants. Strasbourg\\'s metropolitan area had a population of 773,347 in 2013 (not counting the section across the border in Germany), making it the ninth largest metro area in France and home to 13% of the Grand Est region\\'s inhabitants. The transnational Eurodistrict Strasbourg-Ortenau had a population of 915,000 inhabitants in 2014.',\n",
       "  'a': '276,170 inhabitants',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What is the inhabitant of the city where  122nd SS-Standarte was formed in2014',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Strasbourg',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Strasbourg is the capital and largest city of the Grand Est region of France and is the official seat of the European Parliament.',\n",
       "  '6_p1': 78,\n",
       "  '6_p2': None,\n",
       "  's1': 'The 122nd SS-Standarte was formed in the city of Strasbourg.',\n",
       "  's1_': 'The 122nd SS-Standarte was formed in the city of Strasbourg.',\n",
       "  'c': 'The 122nd SS-Standarte was formed in the city of Strasbourg which is the capital and largest city of the Grand Est region of France and is the official seat of the European Parliament.',\n",
       "  'c-b': 'The 122nd SS-Standarte was formed in the city which is the capital and the largest city of the Grand Est region of France and is the official seat of the European Parliament.',\n",
       "  'c-a': 'The 122nd SS-Standarte was formed in the city which is the capital and the largest city of the Grand Est region of France and is the official seat of the European Parliament.',\n",
       "  'q': 'The 122nd SS-Standarte was formed in the city which is the capital and the largest city of the Grand Est region of France and is the official seat of the European Parliament?'},\n",
       " {'k': 69,\n",
       "  'p1': \"Koningsdag - Koningsdag (] ) or King's Day is a national holiday in the Kingdom of the Netherlands. Celebrated on 27 April (26 April if the 27th is a Sunday), the date marks the birth of King Willem-Alexander.\",\n",
       "  'p2': \"Oranjegekte - Oranjegekte (Orange craze) or Oranjekoorts (Orange fever) is a phenomenon in the Netherlands that occurs during major sporting events, especially international football championships, and during Koningsdag, an annual holiday celebrating the king's birthday. It manifests itself in the wearing of orange clothing such as T-shirts, caps and scarfs; lavish attention for sports and sports fans in the media; and the decoration of cars, rooms, houses, shops, and even entire streets in orange, the traditional color of the Dutch royal family, the House of Orange-Nassau.\",\n",
       "  'a': 'orange',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What color clothing do people of the Netherlands wear during Oranjegekte or to celebrate the national holiday Koningsdag? ',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Koningsdag',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Koningsdag is a national holiday in the Kingdom of the Netherlands.',\n",
       "  '6_p1': 79,\n",
       "  '6_p2': 80,\n",
       "  's2': 'Oranjegekte occurs during Koningsdag.',\n",
       "  's1_': 'Koningsdag is a national holiday in the Kingdom of the Netherlands.',\n",
       "  's2_': 'Oranjegekte occurs during Koningsdag.',\n",
       "  'c': 'Oranjegekte occurs during Koningsdag which is a national holiday in the Kingdom of the Netherlands.',\n",
       "  'c-b': 'Oranjegekte occurs during a national holiday in the Kingdom of the Netherlands.',\n",
       "  'c-a': 'A certain national holiday occurs in the Kingdom of the Netherlands.',\n",
       "  'q': 'What national holiday occurs in the Kingdom of the Netherlands?'},\n",
       " {'k': 70,\n",
       "  'p1': 'Tromeo and Juliet - Tromeo and Juliet is a 1996 American independent transgressive romantic comedy film and a loose adaptation of William Shakespeare\\'s \"Romeo & Juliet\" from Troma Entertainment. The film was directed by Lloyd Kaufman from a screenplay by Kaufman and James Gunn, who also served as associate director.',\n",
       "  'p2': 'James Gunn - James Gunn (born August 5) is an American screenwriter, director, producer, novelist, actor, and musician. He started his career as a screenwriter in the mid-1990s, writing the scripts for \"Tromeo and Juliet\" (1996), \"Scooby-Doo\" (2002) and its sequel \"\" (2004), and the 2004 version of \"Dawn of the Dead\". He then started working also as a director, starting with \"Slither\" (2006). He subsequently wrote and directed the web series \"James Gunn\\'s PG Porn\", and the superhero films \"Super\" (2010), \"Guardians of the Galaxy\" (2014) and \"Guardians of the Galaxy Vol. 2\" (2017).',\n",
       "  'a': 'Tromeo and Juliet',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What was the name of the 1996 loose adaptation of William Shakespeare\\'s \"Romeo & Juliet\" written by James Gunn?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'James Gunn',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Tromeo and Juliet is a 1996 American independent transgressive romantic comedy film directed by Lloyd Kaufman from a screenplay by Kaufman and James Gunn.',\n",
       "  '6_p1': 81,\n",
       "  '6_p2': 82,\n",
       "  's2': 'James Gunn is an American screenwriter, director, producer, novelist, actor, and musician.',\n",
       "  's1_': 'Tromeo and Juliet is a 1996 American independent transgressive romantic comedy film directed by Lloyd Kaufman from a screenplay by Kaufman and James Gunn.',\n",
       "  's2_': 'James Gunn is an American screenwriter, director, producer, novelist, actor, and musician.',\n",
       "  'c': 'Tromeo and Juliet is a 1996 American independent transgressive romantic comedy film directed by Lloyd Kaufman from a screenplay by James Gunn who is an American screenwriter, director, producer, novelist, actor, and musician.',\n",
       "  'c-b': 'Tromeo and Juliet is a 1996 American independent transgressive romantic comedy film directed by Lloyd Kaufman from a screenplay by an American screenwriter, director, producer, novelist, actor, and musician.',\n",
       "  'c-a': 'A 1996 American independent transgressive romantic comedy film directed by Lloyd Kaufman from a screenplay by an American screenwriter, director, producer, novelist, actor, and musician.',\n",
       "  'q': 'What is the name of the 1996 American independent transgressive romantic comedy film directed by Lloyd Kaufman from a screenplay by an American screenwriter, director, producer, novelist, actor, and musician?'},\n",
       " {'k': 71,\n",
       "  'p1': 'Robert Suettinger - Robert Suettinger was United States President Bill Clinton\\'s national intelligence officer for East Asia at the National Intelligence Council (NIC) from 1997-1998. While there, Suettinger oversaw the preparation of national intelligence estimates for the director of the Central Intelligence Agency. His areas of specialty included the People\\'s Republic of China and the North Korean nuclear weapons program. After working in the Clinton administration, Suettinger has been a senior analyst for the Brookings Institution where he wrote the book \"Beyond Tiananmen – The Politics of U.S.-China Relations, 1989-2000\". He also is a senior advisor at The Stimson Center.',\n",
       "  'p2': 'Bill Clinton - William Jefferson Clinton (born William Jefferson Blythe III; August 19, 1946) is an American politician who served as the 42nd President of the United States from 1993 to 2001. Prior to the presidency, he was the Governor of Arkansas during two separate terms, from 1979 to 1981, and again from 1983 to 1992. Before that, he served as Arkansas attorney general, from 1977 to 1979. A member of the Democratic Party, Clinton was ideologically a New Democrat and many of his policies reflected a centrist \"Third Way\" political philosophy.',\n",
       "  'a': 'William Jefferson Clinton',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Robert Suettinger was the national intelligence officer under which former Governor of Arkansas?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Bill Clinton',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 83,\n",
       "  '6_p2': 84,\n",
       "  's1': \"Robert Suettinger was Bill Clinton's national intelligence officer for East Asia.\",\n",
       "  's2': 'Bill Clinton served as the 42nd President of the United States.',\n",
       "  's1_': \"Robert Suettinger was Bill Clinton's national intelligence officer for East Asia.\",\n",
       "  's2_': 'Bill Clinton served as the 42nd President of the United States.',\n",
       "  'c': \"Robert Suettinger was Bill Clinton's national intelligence officer for East Asia and Bill Clinton served as the 42nd President of the United States.\",\n",
       "  'c-b': 'Robert Suettinger was the national intelligence officer for East Asia for the 42nd President of the United States.',\n",
       "  'c-a': 'Robert Suettinger was the national intelligence officer for East Asia for the 42nd President of the United States.',\n",
       "  'q': 'Who was the 42nd President of the United States who was Robert Suettinger who was the national intelligence officer for East Asia?'},\n",
       " {'k': 72,\n",
       "  'p1': 'John John Florence - John \"John\" Alexander Florence (born 18 October 1992) is an American professional surfer. He is known as \"one of the most dominant Pipe surfers of his era\" and has most recently won the title of 2016 World Surf League Men\\'s Champion, thus naming him the best surfer in the world. He is the first Hawaiian champion since 3 times world champion Andy Irons.',\n",
       "  'p2': 'MEO Rip Curl Pro Portugal - MEO Rip Curl Pro Portugal formerly known as \"Moche Rip Curl Pro Portugal\" or \"Rip Curl Pro Portugal\" is a professional surfing competition of the ASP World Tour held every year in October at the Supertubos beach in Peniche, Portugal. The event was first founded in 2009 as \"Rip Curl Search Portugal\". The defending champion is the 2016 WSL champion, John John Florence who won the event in 2016.',\n",
       "  'a': 'John John Florence',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What American professional Hawaiian surfer born 18 October 1992 won the Rip Curl Pro Portugal?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'John John Florence',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 85,\n",
       "  '6_p2': 86,\n",
       "  's1': 'John John Florence is an American professional surfer.',\n",
       "  's2': 'MEO Rip Curl Pro Portugal is a professional surfing competition of the ASP World Tour won by John John Florence in 2016.',\n",
       "  's1_': 'John John Florence is an American professional surfer.',\n",
       "  's2_': 'MEO Rip Curl Pro Portugal is a professional surfing competition of the ASP World Tour won by John John Florence in 2016.',\n",
       "  'c': 'MEO Rip Curl Pro Portugal is a professional surfing competition of the ASP World Tour won by American professional surfer John John Florence in 2016.',\n",
       "  'c-b': 'MEO Rip Curl Pro Portugal is a professional surfing competition of the ASP World Tour won by American professional surfer in 2016.',\n",
       "  'c-a': 'MEO Rip Curl Pro Portugal is a professional surfing competition of the ASP World Tour won by American professional surfer in 2016.',\n",
       "  'q': 'Which American professional surfer won the MEO Rip Curl Pro Portugal, a professional surfing competition of the ASP World Tour, in 2016?'},\n",
       " {'k': 73,\n",
       "  'p1': \"Sela Ward - Sela Ann Ward (born July 11, 1956) is an American actress, author and producer, best known for her roles on television beginning in the early '80s.\",\n",
       "  'p2': 'Bobbi Bacha - Bobbi Bacha is a Texas Private Investigator portrayed in 2004 TV Sony Pictures Movie \"Suburban Madness\" played by actress Sela Ward. Bobbi Bacha also was involved and worked on the case of New York millionaire Robert Durst who was charged with murder in Galveston, Texas for killing his neighbor Morris Black but was found not guilty by a Galveston County Jury.',\n",
       "  'a': 'Ann',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What is the middle name of the actress who plays Bobbi Bacha in Suburban Madness?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Sela Ward',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Sela Ward is an actress who was born on July 11, 1956.',\n",
       "  '6_p2': 87,\n",
       "  '6_p1': None,\n",
       "  's2': 'Bobbi Bacha is a Texas Private Investigator portrayed in 2004 TV Sony Pictures Movie \"Suburban Madness\" played by actress Sela Ward.',\n",
       "  's2_': 'Bobbi Bacha is a Texas Private Investigator portrayed in 2004 TV Sony Pictures Movie \"Suburban Madness\" played by actress Sela Ward.',\n",
       "  'c': 'Bobbi Bacha is a Texas Private Investigator portrayed in 2004 TV Sony Pictures Movie \"Suburban Madness\" played by actress Sela Ward who was born on July 11, 1956.',\n",
       "  'c-b': 'Bobbi Bacha is a Texas Private Investigator portrayed in 2004 TV Sony Pictures Movie \"Suburban Madness\" played by actress who was born on July 11, 1956.',\n",
       "  'c-a': 'Bobbi Bacha is a Texas Private Investigator portrayed in 2004 TV Sony Pictures Movie \"Suburban Madness\" played by actress who was born on July 11, 1956.',\n",
       "  'q': 'Who played the role of Bobbi Bacha, a Texas Private Investigator in 2004 TV Sony Pictures Movie \"Suburban Madness\"?'},\n",
       " {'k': 74,\n",
       "  'p1': 'Alvaro Mexia - Alvaro Mexia was a 17th-century Spanish explorer and cartographer of the east coast of Florida. Mexia was stationed in St Augustine and was given a diplomatic mission to the native populations living south of St. Augustine and in the Cape Canaveral area. This mission resulted in a \"Period of Friendship\" between the Spanish and the Ais native population.',\n",
       "  'p2': 'Indigenous peoples of Florida - The Indigenous peoples of Florida lived in what is now known as Florida for more than 12,000 years before the time of first contact with Europeans. However, the indigenous Floridians had largely died out by the early 18th century. Some Apalachees migrated to Louisiana, where their descendants now live, some were taken to Cuba and Mexico by the Spanish in the 18th century, and a few may have been absorbed into the Seminole and Miccosukee tribes.',\n",
       "  'a': 'Apalachees',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Alvaro Mexia had a diplomatic mission with which tribe of indigenous people?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Indigenous peoples of Florida',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Alvaro Mexia was a 17th-century Spanish explorer and cartographer of the east coast of Florida, he was given a diplomatic mission to the native populations living south of St Augustine and the Cape Canaveral area.',\n",
       "  '6_p1': 88,\n",
       "  '6_p2': 89,\n",
       "  's2': 'The Indigenous peoples of Florida lived in what is now known as Florida for more than 12,000 years before the first contact with Europeans.',\n",
       "  's1_': 'Alvaro Mexia was a 17th-century Spanish explorer and cartographer of the east coast of Florida.',\n",
       "  's2_': 'The Indigenous peoples of Florida lived in what is now known as Florida for more than 12,000 years before the first contact with Europeans.',\n",
       "  'c': 'Alvaro Mexia was a 17th-century Spanish explorer and cartographer of the east coast of Florida, he was given a diplomatic mission to the native populations living south of St Augustine and the Cape Canaveral area, they lived in what is now known as Florida for more than 12,000 years before the first contact with Europeans.',\n",
       "  'c-b': 'Alvaro Mexia was a 17th-century Spanish explorer and cartographer of the east coast of Florida, he was given a diplomatic mission to the native populations living south of St Augustine and the Cape Canaveral area, they lived in what is now known as Florida for more than 12,000 years before the first contact with Europeans.',\n",
       "  'c-a': 'Alvaro Mexia was a 17th-century Spanish explorer and cartographer of the east coast of Florida, he was given a diplomatic mission to the native populations living south of St Augustine and the Cape Canaveral, they lived in what is now known as Florida for more than 12,000 years before the first contact with Europeans.',\n",
       "  'q': 'Alvaro Mexia was a 17th-century Spanish explorer and cartographer of the east coast of Florida, he was given a diplomatic mission to the native populations living south of St Augustine and the Cape Canaveral, they lived in what is now known as Florida for more than 12,000 years before the first contact with Europeans.'},\n",
       " {'k': 75,\n",
       "  'p1': 'Edmund Leach - Sir Edmund Ronald Leach (7 November 1910 – 6 January 1989) was a British social anthropologist.',\n",
       "  'p2': 'Alfred Gell - Alfred Antony Francis Gell, {\\'1\\'- \", \\'2\\'- \", \\'3\\'- \", \\'4\\'- \"} ( ; June 12, 1945 – January 28, 1997) was a British social anthropologist whose most influential work concerned art, language, symbolism and ritual. He was trained by Edmund Leach (MPhil, Cambridge University) and Raymond Firth (PhD, London School of Economics) and did his fieldwork in Melanesia and tribal India. Gell taught at the London School of Economics, among other places. He was also a Fellow of the British Academy. He died of cancer in 1997, at the age of 51.',\n",
       "  'a': 'British',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What nationality were social anthropologists Alfred Gell and Edmund Leach?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'British',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 90,\n",
       "  '6_p2': 91,\n",
       "  's1': 'Edmund Leach was a British social anthropologist.',\n",
       "  's2': 'Alfred Gell was a Fellow of the British Academy.',\n",
       "  's1_': 'Edmund Leach was a British social anthropologist.',\n",
       "  's2_': 'Alfred Gell was a Fellow of the British Academy.',\n",
       "  'c': 'Alfred Gell was a Fellow of the British Academy and Edmund Leach was not.',\n",
       "  'c-b': 'Alfred Gell was a Fellow of the British Academy and Edmund Leach was not.',\n",
       "  'c-a': 'Alfred Gell was a Fellow of the British Academy and Edmund Leach was not.',\n",
       "  'q': 'Which of the following, Alfred Gell or Edmund Leach, was a Fellow of the British Academy?'},\n",
       " {'k': 76,\n",
       "  'p1': 'George V - George V (George Frederick Ernest Albert; 3 June 1865 – 20 January 1936) was King of the United Kingdom and the British Dominions, and Emperor of India, from 6 May 1910 until his death in 1936.',\n",
       "  'p2': '1925 Birthday Honours - The 1925 Birthday Honours were appointments by King George V to various orders and honours to reward and highlight good works by citizens of the British Empire. The appointments were made to celebrate the official birthday of The King, and were published in \"The London Gazette\" on 3 June 1925.',\n",
       "  'a': '1865',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'In which year was the King who made the 1925 Birthday Honours born?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'King George V',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'King George V was born in 1865.',\n",
       "  '6_p2': 92,\n",
       "  '6_p1': None,\n",
       "  's2': 'The 1925 Birthday Honours were appointments by King George V.',\n",
       "  's2_': 'The 1925 Birthday Honours were appointments by King George V.',\n",
       "  'c': 'The 1925 Birthday Honours were appointments by King George V who was born in 1865 and who received the Victoria Cross.',\n",
       "  'c-b': 'The 1925 Birthday Honours were appointments by a person who was born in 1865 and who received the Victoria Cross.',\n",
       "  'c-a': 'The 1925 Birthday Honours were appointments by a person who was born in a certain year and who received the Victoria Cross.',\n",
       "  'q': 'Who was born in what year and who received the Victoria Cross in the 1925 Birthday Honours?'},\n",
       " {'k': 77,\n",
       "  'p1': 'Sullivan County, New Hampshire - Sullivan County is a county located in the U.S. state of New Hampshire. As of the 2010 census, the population was 43,742, making it the second-least populous county in New Hampshire. Its county seat is Newport.',\n",
       "  'p2': 'East Lempster, New Hampshire - East Lempster is an unincorporated community in the town of Lempster in Sullivan County, New Hampshire, United States.',\n",
       "  'a': 'Newport',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What is the county seat of the county where East Lempster, New Hampshire is located?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Sullivan County',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Sullivan County has its county seat in Newport.',\n",
       "  '6_p2': 93,\n",
       "  '6_p1': None,\n",
       "  's2': 'East Lempster is an unincorporated community in the town of Lempster in Sullivan County, New Hampshire.',\n",
       "  's2_': 'East Lempster is an unincorporated community in the town of Lempster in Sullivan County, New Hampshire.',\n",
       "  'c': 'East Lempster is an unincorporated community in the town of Lempster in Sullivan County, New Hampshire which has its county seat in Newport.',\n",
       "  'c-b': 'East Lempster is an unincorporated community in the town of Lempster in a county which has its county seat in Newport.',\n",
       "  'c-a': 'East Lempster is an unincorporated community in the town of Lempster in a county which has its county seat in a certain place.',\n",
       "  'q': 'East Lempster is an unincorporated community in the town of Lempster in a county which has its county seat where?'},\n",
       " {'k': 78,\n",
       "  'p1': 'Bob Seger - Robert Clark Seger ( , born May 6, 1945) is an American singer-songwriter, guitarist and pianist. As a locally successful Detroit-area artist, he performed and recorded as Bob Seger and the Last Heard and Bob Seger System throughout the 1960s. By the early 1970s, he had dropped the \\'System\\' from his recordings and continued to strive for broader success with various other bands. In 1973, he put together the Silver Bullet Band, with a group of Detroit-area musicians, with whom he became most successful on the national level with the album \"Live Bullet\" (1976), recorded live with the Silver Bullet Band in 1975 at Cobo Hall in Detroit, Michigan. In 1976, he achieved a national breakout with the studio album \"Night Moves\". On his studio albums, he also worked extensively with the Alabama-based Muscle Shoals Rhythm Section, which appeared on several of Seger\\'s best-selling singles and albums.',\n",
       "  'p2': 'Against the Wind (album) - Against the Wind is the eleventh album by American rock singer Bob Seger and his fourth with the Silver Bullet Band. It was released in February 1980. It is Seger\\'s only number-one album to date, spending six weeks at the top of the \"Billboard\" Top LPs chart, knocking Pink Floyd\\'s \"The Wall\" from the top spot.',\n",
       "  'a': 'Bob Seger',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'The Album Against the Wind was the 11th Album of a Rock singer Robert C Seger born may 6 1945. What was the Rock singers stage name ?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Bob Seger',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 94,\n",
       "  '6_p2': 95,\n",
       "  's1': 'Bob Seger is an American singer-songwriter, guitarist and pianist.',\n",
       "  's2': 'Against the Wind is the eleventh album by American rock singer Bob Seger.',\n",
       "  's1_': 'Bob Seger is an American singer-songwriter, guitarist and pianist.',\n",
       "  's2_': 'Against the Wind is the eleventh album by American rock singer Bob Seger.',\n",
       "  'c': 'Against the Wind is the eleventh album by American rock singer Bob Seger who is an American singer-songwriter, guitarist and pianist.',\n",
       "  'c-b': 'Against the Wind is the eleventh album by an American singer-songwriter, guitarist and pianist.',\n",
       "  'c-a': 'Against the Wind is the eleventh album by an American singer-songwriter, guitarist and pianist.',\n",
       "  'q': 'Against the Wind is the eleventh album by an American singer-songwriter, guitarist and pianist?'},\n",
       " {'k': 79,\n",
       "  'p1': 'Rostker v. Goldberg - Rostker v. Goldberg, 453 U.S. 57 (1981) , was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. After extensive hearings, floor debate and committee sessions on the matter, the United States Congress enacted the law, as it had previously been, to apply to men only. Several attorneys, including Robert L. Goldberg, subsequently challenged the gender distinction as unconstitutional. (The named defendant is Bernard D. Rostker, Director of the Selective Service System.) In a 6-3 decision, the Supreme Court held that this gender distinction was not a violation of the equal protection component of the due process clause, and that the Act would stand as passed.',\n",
       "  'p2': \"Conscription in the United States - Conscription in the United States, commonly known as the draft, has been employed by the federal government of the United States in four conflicts- the American Civil War, World War I, World War II, and the Cold War (including both the Korean and Vietnam Wars). The third incarnation of the draft came into being in 1940 through the Selective Training and Service Act. It was the country's first peacetime draft. From 1940 until 1973, during both peacetime and periods of conflict, men were drafted to fill vacancies in the United States Armed Forces that could not be filled through voluntary means. The draft was ended when the United States Armed Forces moved to an all-volunteer military force. However, the Selective Service System remains in place as a contingency plan; all male civilians between the ages of 18 and 25 are required to register so that a draft can be readily resumed if needed.\",\n",
       "  'a': 'Conscription',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Rostker v. Goldberg held that the practice of what way of filling armed forces vacancies was consitutional?',\n",
       "  'qtype': 'confused',\n",
       "  'b': 'bleh',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional.',\n",
       "  '6_p1': 96,\n",
       "  '6_p2': 97,\n",
       "  's2': 'Conscription in the United States has been employed by the federal government in four conflicts.',\n",
       "  's1_': 'Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional.',\n",
       "  's2_': 'Conscription in the United States has been employed by the federal government in four conflicts.',\n",
       "  'c': 'Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in four conflicts.',\n",
       "  'c-b': 'Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in four conflicts.',\n",
       "  'c-a': 'Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in four conflicts.',\n",
       "  'q': 'Rostker v. Goldberg was a decision of the United States Supreme Court holding that the practice of requiring only men to register for the draft was constitutional. Conscription in the United States has been employed by the federal government in how many conflicts?',\n",
       "  's1_c': 'bleh',\n",
       "  's2_c': 'bleh',\n",
       "  'c_c': 'Both bleh and a sneeze are considered to be unpleasant.',\n",
       "  'q_c': 'What is the common thing between a sneeze and bleh?'},\n",
       " {'k': 80,\n",
       "  'p1': 'Handi-Snacks - Handi-Snacks are a snack food product line sold by Mondelez International. The common two-compartment Handi-Snack has a small spread compartment and a larger compartment for crackers; a red plastic spreader is included with crackers.',\n",
       "  'p2': 'Mondelez International - Mondelez International, Inc., styled Mondelēz ( ), is an American multinational confectionery, food, and beverage company based in Illinois which employs about 107,000 people around the world. It consists of the global snack and food brands of the former Kraft Foods Inc after the October 2012 spin-off of its North American grocery operations. The Mondelez name, adopted in 2012, was suggested by Kraft Foods employees and is derived from the words \"mundus\" (Latin for “world”) and \"delez\" (a proxy for the word \"delicious\").',\n",
       "  'a': 'Mondelez International, Inc.',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Handi-Snacks are a snack food product line sold by what American multinational confectionery, food, and beverage company that is based in Illinois?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Mondelez International',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 98,\n",
       "  '6_p2': 99,\n",
       "  's1': 'Handi-Snacks are a snack food product line sold by Mondelez International.',\n",
       "  's2': 'Mondelez International is an American multinational confectionery, food, and beverage company based in Illinois which employs about 107,000 people around the world.',\n",
       "  's1_': 'Handi-Snacks are a snack food product line sold by Mondelez International.',\n",
       "  's2_': 'Mondelez International is an American multinational confectionery, food, and beverage company based in Illinois which employs about 107,000 people around the world.',\n",
       "  'c': 'Handi-Snacks are a snack food product line sold by Mondelez International which is an American multinational confectionery, food, and beverage company based in Illinois which employs about 107,000 people around the world.',\n",
       "  'c-b': 'Handi-Snacks are a snack food product line sold by a company which is an American multinational confectionery, food, and beverage company based in Illinois which employs about 107,000 people around the world.',\n",
       "  'c-a': 'Handi-Snacks are a snack food product line sold by a company which is an American multinational confectionery, food, and beverage company based in Illinois which employs about 107,000 people around the world.',\n",
       "  'q': 'What is the name of the company which is an American multinational confectionery, food, and beverage company based in Illinois which employs about 107,000 people around the world and sells Handi-Snacks, a snack food product line?'},\n",
       " {'k': 81,\n",
       "  'p1': 'Their Lives - Their Lives- The Women Targeted by the Clinton Machine is a book by Candice E. Jackson. Published by conservative publisher World Ahead Publishing on May 31, 2005, it recounts the stories of seven women who crossed paths with Bill Clinton- Monica Lewinsky, Paula Jones, Gennifer Flowers, Kathleen Willey, Elizabeth Gracen, Juanita Broaddrick, and Sally Perdue.',\n",
       "  'p2': 'Monica Lewinsky - Monica Samille Lewinsky (born July 23, 1973) is an American activist, television personality, fashion designer, and former White House intern.',\n",
       "  'a': 'Monica Lewinsky',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What was the name of a woman from the book titled \"Their Lives: The Women Targeted by the Clinton Machine \" and was also a former white house intern?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Monica Lewinsky',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 100,\n",
       "  '6_p2': 101,\n",
       "  's1': 'Their Lives- The Women Targeted by the Clinton Machine is a book by Candice E. Jackson that recounts the stories of seven women who crossed paths with Bill Clinton- Monica Lewinsky.',\n",
       "  's2': 'Monica Lewinsky is an American activist, television personality, fashion designer, and former White House intern.',\n",
       "  's1_': 'Their Lives- The Women Targeted by the Clinton Machine is a book by Candice E. Jackson that recounts the stories of seven women who crossed paths with Bill Clinton- Monica Lewinsky.',\n",
       "  's2_': 'Monica Lewinsky is an American activist, television personality, fashion designer, and former White House intern.',\n",
       "  'c': 'Their Lives- The Women Targeted by the Clinton Machine is a book by Candice E. Jackson that recounts the stories of seven women who crossed paths with Bill Clinton- Monica Lewinsky is an American activist, television personality, fashion designer, and former White House intern.',\n",
       "  'c-b': 'Their Lives- The Women Targeted by the Clinton Machine is a book by Candice E. Jackson that recounts the stories of seven women who crossed paths with Bill Clinton- Monica Lewinsky is an American activist, television personality, fashion designer, and former White intern.',\n",
       "  'c-a': 'Their Lives- The Women Targeted by the Clinton Machine is a book by Candice E. Jackson that recounts the stories of seven women who crossed paths with Bill Clinton- Monica Lewinsky is an American activist, television personality, fashion designer, and former White intern.',\n",
       "  'q': 'Their Lives- The Women Targeted by the Clinton Machine is a book by Candace E. Jackson that recounts the stories of seven women who crossed paths with Bill Clinton- Monica Lewinsky is an American activist, television personality, fashion designer, and former White intern.'},\n",
       " {'k': 82,\n",
       "  'p1': \"Paul Manafort - Paul John Manafort Jr. (born April 1, 1949) is an American lawyer, lobbyist and political consultant. He joined Donald Trump's presidential campaign team in March 2016 and served as campaign manager from June to August 2016. He was previously an adviser to the U.S. presidential campaigns of Republicans Gerald Ford, Ronald Reagan, George H. W. Bush, and Bob Dole. In 1980 Manafort co-founded the Washington, D.C.-based lobbying firm Black, Manafort & Stone, along with principals Charles R. Black Jr., and Roger J. Stone. In 1984 it was renamed Black, Manafort, Stone and Kelly (BMSK) & associates, after Peter G. Kelly was recruited.\",\n",
       "  'p2': 'Trump campaign–Russian meeting - On June 9, 2016, a meeting was held in Trump Tower in New York City between three senior members of the presidential campaign of Donald TrumpDonald Trump Jr., Jared Kushner, and Paul Manafortand at least five other people, including Russian lawyer Natalia Veselnitskaya. The meeting was arranged by publicist and long-time Trump acquantiance, Rob Goldstone on behalf of his client, singer-songwriter Emin Agalarov. It was disclosed to U.S. government officials when Kushner filed a revised version of his security clearance form.',\n",
       "  'a': 'April 1, 1949',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'When was the American lawyer, lobbyist and political consultant who was a senior member of the presidential campaign of Donald Trump born?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Paul Manafort',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Paul Manafort was born on April 1, 1949.',\n",
       "  '6_p2': 102,\n",
       "  '6_p1': None,\n",
       "  's2': 'Paul Manafort was present at a meeting between three senior members of the presidential campaign of Donald Trump and Russian lawyer Natalia Veselnitskaya.',\n",
       "  's2_': 'Paul Manafort was present at a meeting between three senior members of the presidential campaign of Donald Trump and Russian lawyer Natalia Veselnitskaya.',\n",
       "  'c': 'Paul Manafort was present at a meeting between three senior members of the presidential campaign of Donald Trump and Russian lawyer Natalia Veselnitskaya who was born on April 1, 1949.',\n",
       "  'c-b': 'A person was present at a meeting between three senior members of the presidential campaign of Donald Trump and Russian lawyer Natalia Veselnitskaya who was born on April 1, 1949.',\n",
       "  'c-a': 'A person was present at a meeting between three senior members of the presidential campaign of Donald Trump and Russian lawyer Natalia Veselnitskaya who was born on a certain date.',\n",
       "  'q': 'On what date was Russian lawyer Natalia Veselnitskaya who was present at a meeting between three senior members of the presidential campaign of Donald Trump was born?'},\n",
       " {'k': 83,\n",
       "  'p1': 'Crime and Punishment - Crime and Punishment (Russian- Преступлéние и наказáние , \"Prestupleniye i nakazaniye\"; ] ) is a novel by the Russian author Fyodor Dostoyevsky. It was first published in the literary journal \"The Russian Messenger\" in twelve monthly installments during 1866. Later, it was published in a single volume. It is the second of Dostoyevsky\\'s full-length novels following his return from 5 years of exile in Siberia. \"Crime and Punishment\" is considered the first great novel of his \"mature\" period of writing.',\n",
       "  'p2': 'Lourenço Mutarelli - In addition to comic books, Mutarelli has also created plays, books, and all the illustration and animated sequences of the 2004 motion picture \"Nina\", which was based on the novel \"Crime and Punishment\" by Dostoyevsky.',\n",
       "  'a': '1866',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'In what year was the novel that Lourenço Mutarelli based \"Nina\" on based first published?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Crime and Punishment',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Crime and Punishment is a novel by the Russian author Fyodor Dostoyevsky published in 1866.',\n",
       "  '6_p2': 103,\n",
       "  '6_p1': None,\n",
       "  's2': 'Lourenço Mutarelli created plays, books, and all the illustration and animated sequences of the 2004 motion picture \"Nina\", which was based on the novel \"Crime and Punishment\" by Dostoyevsky.',\n",
       "  's2_': 'Lourenço Mutarelli created plays, books, and all the illustration and animated sequences of the 2004 motion picture \"Nina\", which was based on the novel \"Crime and Punishment\" by Dostoyevsky.',\n",
       "  'c': 'Lourenço Mutarelli created plays, books, and all the illustration and animated sequences of the 2004 motion picture \"Nina\", which was based on the novel \"Crime and Punishment\" by the Russian author Fyodor Dostoyevsky.',\n",
       "  'c-b': 'Lourenço Mutarelli created plays, books, and all the illustration and animated sequences of the 2004 motion picture \"Nina\", which was based on the novel by the Russian author Fyodor Dostoyevsky.',\n",
       "  'c-a': 'Lourenço Mutarelli created plays, books, and all the illustration and animated sequences of the 2004 motion picture \"Nina\", which was based on the novel by the Russian author Fyodor Dostoevsky.',\n",
       "  'q': 'Who created plays, books, and all the illustration and animated sequences of the 2004 motion picture \"Nina\" which was based on the novel by the Russian author Fyodor Dostoevsky?'},\n",
       " {'k': 84,\n",
       "  'p1': 'Roald Dahl - Roald Dahl ( , ] ; 13 September 1916 – 23 November 1990) was a British novelist, short story writer, poet, screenwriter, and fighter pilot. His books have sold more than 250 million copies worldwide.',\n",
       "  'p2': 'Mrs. Bixby and the Colonel\\'s Coat - \"Mrs. Bixby and the Colonel\\'s Coat\" is a short story by Roald Dahl that first appeared in the 1959 issue of \"Nugget\". The story is Dahl\\'s variation on a popular anecdote dating back at least to 1939- a married woman receives a glamorous mink coat from a man with whom she had an affair. She hopes to sneak the coat into her home without arousing her husband\\'s suspicions, but soon discovers that her husband has plans of his own.',\n",
       "  'a': '250 million',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': \"How many copies of Roald Dahl's variation on a popular anecdote sold?\",\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Roald Dahl',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Roald Dahl was a writer whose books have sold more than 250 million copies worldwide.',\n",
       "  '6_p2': 104,\n",
       "  '6_p1': None,\n",
       "  's2': \"Mrs. Bixby and the Colonel's Coat is a short story by Roald Dahl.\",\n",
       "  's2_': \"Mrs. Bixby and the Colonel's Coat is a short story by Roald Dahl.\",\n",
       "  'c': \"Roald Dahl was a writer whose books have sold more than 250 million copies worldwide and Mrs. Bixby and the Colonel's Coat is a short story by Roald Dahl.\",\n",
       "  'c-b': \"A writer whose books have sold more than 250 million copies worldwide wrote Mrs. Bixby and the Colonel's Coat.\",\n",
       "  'c-a': \"A writer whose books have sold more than a certain number of copies worldwide wrote Mrs. Bixby and the Colonel's Coat.\",\n",
       "  'q': \"How many copies of Mrs. Bixby and the Colonel's Coat have been sold worldwide?\"},\n",
       " {'k': 85,\n",
       "  'p1': 'Chris Menges - Chris Menges BSC, ASC (born 15 September 1940) is an English cinematographer and film director. He is a member of both the American and British Societies of Cinematographers.',\n",
       "  'p2': 'Aram Avakian - Aram A. Avakian (April 23, 1926 – January 17, 1987) was an Armenian-American film editor and director. His work in the latter role includes \"Jazz on a Summer\\'s Day\" (1960) and the indie film \"End of the Road\" (1970).',\n",
       "  'a': 'director',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'What occupation do Chris Menges and Aram Avakian share?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Chris Menges is an English cinematographer and film director.',\n",
       "  's2_c': 'Aram Avakian was an Armenian-American film editor and director.',\n",
       "  'c_c': 'Both Aram Avakian and Chris Menges were film directors.',\n",
       "  'q_c': 'What job do both Aram Avakian and Chris Menges have in common?'},\n",
       " {'k': 86,\n",
       "  'p1': 'Andrew Jaspan - Andrew Jaspan, (b. 20 April 1952) British-Australian journalist, the co-founder of \"The Conversation\", an independent not-for-profit website of analysis and news from the university and research sector\\u200a and its Editor-in-Chief from The Conversation\\'s launch in March 2011 until March 2017.',\n",
       "  'p2': 'The Conversation (website) - The Conversation is an independent, not-for-profit media outlet that uses content sourced from the academic and research community. Since the Australian website\\'s launch in March 2011, it has expanded into six editions, with the addition of a United Kingdom (UK) version in 2013, United States (US) in 2014, Africa in May 2015, France in September 2015, and Global in September 2016. \"The Conversation\" publishes all content under a Creative Commons license and, as of May 2017, reports a monthly online audience of 5.2 million users onsite, and a reach of 35 million people through creative commons republication.',\n",
       "  'a': 'The Conversation',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Andrew Jaspan was the co-founder of what not-for-profit media outlet?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'The Conversation',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 105,\n",
       "  '6_p2': 106,\n",
       "  's1': 'Andrew Jaspan is the co-founder of The Conversation.',\n",
       "  's2': 'The Conversation is an independent, not-for-profit media outlet that uses content sourced from the academic and research community.',\n",
       "  's1_': 'Andrew Jaspan is the co-founder of The Conversation.',\n",
       "  's2_': 'The Conversation is an independent, not-for-profit media outlet that uses content sourced from the academic and research community.',\n",
       "  'c': 'Andrew Jaspan is the co-founder of The Conversation which is an independent, not-for-profit media outlet that uses content sourced from the academic and research community.',\n",
       "  'c-b': 'Andrew Jaspan is the co-founder of a media outlet that is an independent, not-for-profit media outlet that uses content sourced from the academic and research community.',\n",
       "  'c-a': 'Andrew Jaspan is the co-founder of a media outlet that is an independent, not-for-profit media outlet that uses content sourced from the academic and research community.',\n",
       "  'q': 'Andrew Jaspan is the co-founder of what media outlet that is an independent, not-for-profit media outlet that uses content sourced from the academic and research community?'},\n",
       " {'k': 87,\n",
       "  'p1': 'John Waters - John Samuel Waters Jr. (born April 22, 1946) is an American film director, screenwriter, author, actor, stand-up comedian, journalist, visual artist, and art collector, who rose to fame in the early 1970s for his transgressive cult films.',\n",
       "  'p2': '18th Independent Spirit Awards - The 2002 Independent Spirit Awards, honoring the best in independent filmmaking for 2002, were announced on March 22, 2003. It was hosted by John Waters.',\n",
       "  'a': 'John Waters',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Which American film director hosted the 18th Independent Spirit Awards in 2002?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'John Waters',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': False,\n",
       "  '5_p2': False,\n",
       "  '6_p1': 107,\n",
       "  '6_p2': 108,\n",
       "  's1': 'John Waters is an American film director, screenwriter, author, actor, stand-up comedian, and art collector.',\n",
       "  's2': 'The 2002 Independent Spirit Awards was hosted by John Waters.',\n",
       "  's1_': 'John Waters is an American film director, screenwriter, author, actor, stand-up comedian, and art collector.',\n",
       "  's2_': 'The 2002 Independent Spirit Awards was hosted by John Waters.',\n",
       "  'c': 'John Waters is an American film director, screenwriter, author, actor, stand-up comedian, and art collector who hosted the 2002 Independent Spirit Awards.',\n",
       "  'c-b': 'The host of the 2002 Independent Spirit Awards was an American film director, screenwriter, author, actor, and stand-up comedian.',\n",
       "  'c-a': 'The host of the 2002 Independent Spirit Awards was an American film director, screenwriter, author, and stand-up comedian.',\n",
       "  'q': 'Who was the host of the 2002 Independent Spirit Awards who was also an American film director, screenwriter, author, and stand-up comedian?'},\n",
       " {'k': 88,\n",
       "  'p1': 'Flamingo Las Vegas - Flamingo Las Vegas (formerly The Fabulous Flamingo and Flamingo Hilton Las Vegas) is a hotel and casino located on the Las Vegas Strip in Paradise, Nevada. It is owned and operated by Caesars Entertainment Corporation.',\n",
       "  'p2': \"Why Is There Air? - Why Is There Air? (1965) is Bill Cosby's third album. It was recorded at the Flamingo Hotel in Las Vegas, Nevada. It won the 1966 Grammy Award for Best Comedy Album.\",\n",
       "  'a': 'Las Vegas Strip in Paradise',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': \"Where does the hotel and casino located in which Bill Cosby's third album was recorded?\",\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Flamingo Las Vegas',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': True,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'Flamingo Las Vegas is a hotel and casino located on the Las Vegas Strip in Paradise, Nevada.',\n",
       "  '6_p1': 109,\n",
       "  '6_p2': 110,\n",
       "  's2': 'Why Is There Air? was recorded at the Flamingo Hotel in Las Vegas.',\n",
       "  's1_': 'Flamingo Las Vegas is a hotel and casino located on the Las Vegas Strip in Paradise, Nevada.',\n",
       "  's2_': 'Why Is There Air? was recorded at the Flamingo Hotel in Las Vegas.',\n",
       "  'c': 'Why Is There Air? was recorded at the Flamingo Hotel in Las Vegas which is a hotel and casino located on the Las Vegas Strip in Paradise, Nevada.',\n",
       "  'c-b': 'Why Is There Air? was recorded at a hotel and casino located on the Las Vegas Strip in Paradise, Nevada.',\n",
       "  'c-a': 'Why Is There Air? was recorded at a hotel and casino located on a certain location, in this case, in Paradise, Nevada.',\n",
       "  'q': 'Why Is There Air? was recorded at a hotel and casino located on what location, in this case, in Paradise, Nevada?'},\n",
       " {'k': 89,\n",
       "  'p1': 'Gibson (cocktail) - The Gibson is a mixed drink made with gin and vermouth, and often garnished with a pickled onion. The oldest published recipe for the Gibson is found in the 1908 book, \"The World\\'s Drinks And How To Mix Them\" by William Boothby.',\n",
       "  'p2': 'Zurracapote - Zurracapote (sometimes abbreviated as zurra) is a popular Spanish alcoholic mixed drink, similar to sangría. It consists of red wine mixed with fruit such as peaches and lemons, sugar, and cinnamon. The concoction is then traditionally left to steep for several days, though some recipes call for the addition of other alcoholic beverages, juices, and fruit extracts. The result is a mild-to-medium alcoholic drink, similar to sangría.',\n",
       "  'a': 'no',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Do the drinks Gibson and Zurracapote both contain gin?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'The Gibson is a mixed drink made with gin and vermouth.',\n",
       "  's2_c': 'Zurracapote is a Spanish alcoholic mixed drink, similar to sangr ⁇ a.',\n",
       "  'c_c': 'The Gibson is a mixed drink made with gin and vermouth and Zurracapote is a Spanish alcoholic mixed drink, similar to sangria.',\n",
       "  'q_c': 'Are the Gibson and Zurracapote both mixed drinks?'},\n",
       " {'k': 90,\n",
       "  'p1': 'London International Documentary Festival - The London International Documentary Festival (or LIDF) is an annual documentary film festival that takes place in the months of March and April every year. The event is presented in association with the London Review of Books.',\n",
       "  'p2': 'London Review of Books - The London Review of Books (LRB) is a British journal of literary essays. It is published fortnightly.',\n",
       "  'a': 'March and April',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'In what month is the annual documentary film festival, that is presented by the fortnightly published British journal of literary essays, held? ',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'London Review of Books',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'The London International Documentary Festival takes place in the months of March and April.',\n",
       "  '6_p2': 111,\n",
       "  '6_p1': None,\n",
       "  's2': 'The London Review of Books is a journal published fortnightly.',\n",
       "  's2_': 'The London Review of Books is a journal published fortnightly.',\n",
       "  'c': 'The London International Documentary Festival takes place in the months of March and April and The London Review of Books is a journal published fortnightly.',\n",
       "  'c-b': 'The London International Documentary Festival takes place in the months of March and April and there is a journal published fortnightly.',\n",
       "  'c-a': 'The London International Documentary Festival takes place in the months of a certain year and there is a journal published fortnightly.',\n",
       "  'q': 'The London International Documentary Festival takes place in which year and there is a journal published fortnightly?'},\n",
       " {'k': 91,\n",
       "  'p1': 'Tysons Galleria - Tysons Galleria is a three-level super-regional mall owned by General Growth Properties located at 2001 International Drive, McLean, Virginia, in Tysons Corner. It is the second-largest mall in McLean/Tysons Corner, and one of the largest in the Washington metropolitan area.',\n",
       "  'p2': 'McLean, Virginia - McLean ( ) is a census-designated place (CDP) in Fairfax County in Northern Virginia. McLean is home to many diplomats, businessmen, members of Congress, and high-ranking government officials partially due to its proximity to Washington, D.C. and the Central Intelligence Agency. It is the location of Hickory Hill, the former home of Ethel Kennedy, the widow of Robert F. Kennedy. It is also the location of Salona, the former home of Light-Horse Harry Lee, the Revolutionary War hero. The community had an estimated total population of 53,673 in 2015, according to estimates prepared by the United States Census Bureau. It is located between the Potomac River and the town of Vienna. McLean is known for its luxury homes and its high-end shopping destinations- the Tysons Corner Center and the Tysons Galleria. The two McLean zip codes - 22101 and 22102 - are among the most expensive ZIP Codes in Virginia and the United States.',\n",
       "  'a': 'Fairfax County',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Tysons Galleria is located in what county?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'McLean',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'McLean is a CDP in Fairfax County.',\n",
       "  '6_p1': 112,\n",
       "  '6_p2': None,\n",
       "  's1': 'Tysons Galleria is located in McLean.',\n",
       "  's1_': 'Tysons Galleria is located in McLean.',\n",
       "  'c': 'Tysons Galleria is located in McLean which is a CDP in Fairfax County.',\n",
       "  'c-b': 'Tysons Galleria is located in a CDP in Fairfax County.',\n",
       "  'c-a': 'Tysons Galleria is located in a CDP in a certain county.',\n",
       "  'q': 'Tysons Galleria is located in what CDP in what county?'},\n",
       " {'k': 92,\n",
       "  'p1': 'Bordan Tkachuk - Bordan Tkachuk ( ) is a British business executive, the former CEO of Viglen, also known from his appearances on the BBC-produced British version of \"The Apprentice,\" interviewing for his boss Lord Sugar.',\n",
       "  'p2': 'Viglen - Viglen Ltd provides IT products and services, including storage systems, servers, workstations and data/voice communications equipment and services.',\n",
       "  'a': 'IT products and services',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Bordan Tkachuk was the CEO of a company that provides what sort of products?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Viglen',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Viglen provides IT products and services.',\n",
       "  '6_p1': 113,\n",
       "  '6_p2': None,\n",
       "  's1': 'Bordan Tkachuk is a former CEO of Viglen.',\n",
       "  's1_': 'Bordan Tkachuk is a former CEO of Viglen.',\n",
       "  'c': 'Bordan Tkachuk was the former CEO of Viglen which provides IT products and services.',\n",
       "  'c-b': 'Bordan Tkachuk was the former CEO of a company which provides IT products and services.',\n",
       "  'c-a': 'Bordan Tkachuk was the former CEO of a company which provides a certain kind of thing.',\n",
       "  'q': 'What kind of thing does Bordan Tkachuk, who was the former CEO of a company which provides that kind of thing, provide?'},\n",
       " {'k': 93,\n",
       "  'p1': 'Lev Yilmaz - Levni Yilmaz (born 1973) is a San Francisco based independent film maker, artist and publisher, best known for his “Tales of Mere Existence” animated comic series.',\n",
       "  'p2': 'Pamela B. Green - Pamela B. Green is an American film producer and director known for her work in feature film titles and motion graphics. She is the director and producer of the documentary \"Be Natural- The Untold Story of Alice Guy-Blache\".',\n",
       "  'a': 'Levni Yilmaz',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Which filmmaker was known for animation, Lev Yilmaz or Pamela B. Green?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Levni Yilmaz is a San Francisco based independent film maker, artist and publisher, best known for his “Tales of Mere Existence” animated comic series.',\n",
       "  's2_c': 'Pamela B. Green is an American film producer and director known for her work in feature film titles and motion graphics.',\n",
       "  'c_c': 'Levni Yilmaz is a San Francisco based independent film maker, artist and publisher, best known for his “Tales of Mere Existence” animated comic series and Pamela B. Green is an American film producer and director known for her work in feature film titles and motion graphics.',\n",
       "  'q_c': 'Who amongst Levni Yilmaz and Pamela B. Green is best known for his “Tales of Mere Existence” animated comic series?'},\n",
       " {'k': 94,\n",
       "  'p1': 'Rabat - Rabat (Berber- , Arabic- الرِّبَاط\\u200e \\u200e , Moroccan Arabic- رّباط\\u200e \\u200e ) is the capital city of Morocco and its second largest city with an urban population of approximately 580,000 (2014) and a metropolitan population of over 1.2 million. It is also the capital city of the Rabat-Salé-Kénitra administrative region.',\n",
       "  'p2': \"Moroccan Ambassador to China - The Moroccan ambassador in Beijing is the official representative of the Government in Rabat to the Government of the People's Republic of China.\",\n",
       "  'a': 'Beijing',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'In which city is the ambassador of the Rabat-Salé-Kénitra administrative region to China based?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Rabat',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'The Moroccan ambassador in Beijing is the official representative of the Government in Rabat to the Government in Beijing.',\n",
       "  '6_p1': 114,\n",
       "  '6_p2': None,\n",
       "  's1': 'Rabat is the capital city of Morocco.',\n",
       "  's1_': 'Rabat is the capital city of Morocco.',\n",
       "  'c': 'The Moroccan ambassador in Beijing is the official representative of the Government in Rabat to the Government in Beijing.',\n",
       "  'c-b': 'The Moroccan ambassador in Beijing is the official representative of the Government in a certain city to the Government in another city.',\n",
       "  'c-a': 'The Moroccan ambassador in Beijing is the official representative of the Government in a certain city to the Government in another city.',\n",
       "  'q': 'The Moroccan ambassador in Beijing is the official representative of the Government in which city to the Government in which other city?'},\n",
       " {'k': 95,\n",
       "  'p1': \"Yingkou - Yingkou () is a prefecture-level city of Liaoning province, People's Republic of China. It is a port city of the Bohai Sea, and is the location of the mouth of the Liao River.\",\n",
       "  'p2': \"Fuding -    () is a county-level city in northeastern Ningde prefecture level city, on Fujian's border with Zhejiang province.\",\n",
       "  'a': 'no',\n",
       "  'gold_qtype': 'comparison',\n",
       "  'gold_q': 'Are Yingkou and Fuding the same level of city?',\n",
       "  'qtype': 'comparison',\n",
       "  's1_c': 'Yingkou is a port city of the Bohai Sea.',\n",
       "  's2_c': 'Fuding is a county-level city.',\n",
       "  'c_c': 'Both Yingkou and Fuding are city-level cities.',\n",
       "  'q_c': 'Are both Yingkou and Fuding city-level cities?'},\n",
       " {'k': 96,\n",
       "  'p1': 'Mascogos - The Mascogos (also known as \"negros mascagos\") are an afrodescendant group in Coahuila, Mexico. Centered on the town of El Nacimiento in Múzquiz Municipality, the group are descendants of Black Seminoles escaping the threat of slavery in the United States.',\n",
       "  'p2': 'Black Seminoles - The Black Seminoles are black Indians associated with the Seminole people in Florida and Oklahoma. They are the descendants of free blacks and of escaped slaves (called maroons) who allied with Seminole groups in Spanish Florida. Historically, the Black Seminoles lived mostly in distinct bands near the Native American Seminole. Some were held as slaves of particular Seminole leaders; but they had more freedom than did slaves held by whites in the South and by other Native American tribes, including the right to bear arms.',\n",
       "  'a': 'Coahuila, Mexico',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Where did the descendants of the group of black Indians associated with the Seminole people settle?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Black Seminoles',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': False,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'The Mascogos are an afrodescendant group in Coahuila, Mexico.',\n",
       "  '6_p2': 115,\n",
       "  '6_p1': None,\n",
       "  's2': 'Black Seminoles are black Indians associated with the Seminole people in Florida and Oklahoma.',\n",
       "  's2_': 'Black Seminoles are black Indians associated with the Seminole people in Florida and Oklahoma.',\n",
       "  'c': 'The Mascogos are an afrodescendant group in Coahuila, Mexico and Black Seminoles are black Indians associated with the Seminole people in Florida and Oklahoma.',\n",
       "  'c-b': 'The Mascogos are an afrodescendant group in Coahuila, Mexico and Black Seminoles are black Indians associated with the Seminole people in Florida and Oklahoma.',\n",
       "  'c-a': 'The Mascogos are an afrodescendant group in a certain place and Black Seminoles are black Indians associated with the Seminole people in Florida and Oklahoma.',\n",
       "  'q': 'The Mascogos are an afrodescendant group in what place are the Black Seminoles, who are black Indians associated with the Seminole people in Florida and Oklahoma?'},\n",
       " {'k': 97,\n",
       "  'p1': 'David Huntsinger - David Huntsinger is a pianist, composer, songwriter, and arranger who moved from his native California to Nashville, TN, in 1976 and played for the Rambos. He co-wrote the song, \"Holy Spirit, Thou Art Welcome\", with Dottie Rambo, as well as the children’s musical, \"Down By The Creek Bank\". In 1979 he left the Rambos to pursue a career as a studio pianist. He wrote and arranged music for the 1989 Grammy-winning album \"A Child’s Gift of Lullabyes\", and arranged for and co-produced Andy Griffith’s 1996 Grammy-winning album, \"I Love To Tell The Story- 25 Timeless Hymns\". He has worked with many artists, such as Sandi Patti, Steve Green, Kathy Troccoli, Michael Crawford, Glen Campbell, Carman, Larnelle Harris, Johnny Cash and Dolly Parton. He toured with Vince Gill for a Christmas tour in 1999, and in 2001 for the Amy Grant/Vince Gill Christmas tour. He has also written a number of children’s musicals, as well as produced many albums of his own original works and arrangements. He played the piano for \"An Unfinished Life\", and contributed to two songs heard in \"The Great Debaters\". He arranged for and played piano in the 1997 Christmas album \"Piano Winterlude\" (Unison). He has done several projects for Discovery House Music. In 2010 he composed a number of pieces for a special 25th Anniversary concert for the international television network 3ABN called \"Pillars of Our Faith\".',\n",
       "  'p2': 'Larnelle Harris - Larnelle Steward Harris (born July 6, 1947) is an American gospel singer and songwriter. During his 30-plus years of ministry, Harris has recorded 18 albums, won five Grammy Awards and 11 Dove Awards, and has had several number one songs on the inspirational music charts.',\n",
       "  'a': 'Larnelle Harris',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'David Huntsinger has worked with this gospel singer born in the month of July?',\n",
       "  'qtype': 'confused',\n",
       "  'b': 'bleh',\n",
       "  'a_in_p1': True,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p1': True,\n",
       "  '5_p2': False,\n",
       "  's1': 'David Huntsinger worked with Larnelle Harris. <extra_id_1>  <extra_id_2> a pianist, composer, songwriter, and arranger who moved from his native California to Nashville, TN, in 1976. <extra_id_3>  <extra_id_4> David Huntsinger worked with Larnelle Harris. <extra_id_5>  <extra_id_6>  <extra_id_7> David Huntsinger worked with Larnelle Harris. <extra_id_8> David Huntsinger is a pianist, composer, songwriter, and arranger who worked with many artists <extra_id_9>  <extra_id_10>  <extra_id_11>  <extra_id_12>  <extra_id_13>  <extra_id_14>  <extra_id_15>  <extra_id_16>',\n",
       "  '6_p1': 116,\n",
       "  '6_p2': 117,\n",
       "  's2': 'Larnelle Harris is an American gospel singer and songwriter.',\n",
       "  's1_': 'David Huntsinger is a pianist, composer, songwriter, and arranger.',\n",
       "  's2_': 'Larnelle Harris is an American gospel singer and songwriter.',\n",
       "  'c': 'David Huntsinger worked with Larnelle Harris who is an American gospel singer and songwriter.',\n",
       "  'c-b': 'David Huntsinger worked with Larnelle Harris who is an American gospel singer and songwriter.',\n",
       "  'c-a': 'David Huntsinger worked with an American gospel singer and songwriter.',\n",
       "  'q': 'David Huntsinger worked with which American gospel singer and songwriter?',\n",
       "  's1_c': 'bleh',\n",
       "  's2_c': 'bleh',\n",
       "  'c_c': 'Both bleh and a sneeze are considered to be unpleasant.',\n",
       "  'q_c': 'Who amongst the following is considered to be unpleasant, bleh or a sneeze?'},\n",
       " {'k': 98,\n",
       "  'p1': 'Champ Bailey - Roland \"Champ\" Bailey Jr. (born June 22, 1978) is a former American football cornerback in the National Football League (NFL). He played college football for Georgia, where he earned consensus All-American honors, and was drafted by the Washington Redskins in the first round of the 1999 NFL Draft. He is the brother of former NFL linebacker Boss Bailey.',\n",
       "  'p2': 'Boss Bailey - Rodney \"Boss\" Bailey (born October 14, 1979) is a former American football linebacker who played in the National Football League. He was originally drafted by the Detroit Lions in the second round of the 2003 NFL Draft. He played college football at the University of Georgia. He is the brother of former NFL cornerback Champ Bailey.',\n",
       "  'a': '2003',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'What year was the brother of this first round draft pick by the Washington Redskins drafted?',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Champ Bailey',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Boss Bailey is the brother of Champ Bailey who was drafted in the second round of the 2003 NFL Draft.',\n",
       "  '6_p1': 118,\n",
       "  '6_p2': None,\n",
       "  's1': 'Champ Bailey is a former American football cornerback in the National Football League (NFL).',\n",
       "  's1_': 'Champ Bailey is a former American football cornerback in the National Football League (NFL).',\n",
       "  'c': 'Champ Bailey is a former American football cornerback in the National Football League (NFL) and his brother Boss Bailey was drafted in the second round of the 2003 NFL Draft.',\n",
       "  'c-b': 'Boss Bailey was drafted in the second round of the 2003 NFL Draft and his brother Champ Bailey is a former American football cornerback in the National Football League (NFL).',\n",
       "  'c-a': 'Boss Bailey was drafted in the second round of the 2003 NFL Draft and his brother Champ Bailey is a former American football cornerback in the National Football League (NFL).',\n",
       "  'q': 'Champ Bailey or Boss Bailey was a former American football cornerback in the National Football League (NFL)?'},\n",
       " {'k': 99,\n",
       "  'p1': '1964 Georgia Tech Yellow Jackets football team - The 1964 Georgia Tech Yellow Jackets football team represented the Georgia Institute of Technology during the 1964 college football season. The Yellow Jackets were led by 20th-year head coach Bobby Dodd, and played their home games at Grant Field in Atlanta, Georgia. They competed as independents for the first time since 1920, after dropping from the Southeastern Conference in 1963.',\n",
       "  'p2': 'Bobby Dodd Stadium - Bobby Dodd Stadium at Historic Grant Field is the football stadium located at the corner of North Avenue at Techwood Drive on the campus of the Georgia Institute of Technology in Atlanta. It has been home to the Georgia Tech Yellow Jackets football team, often referred to as the \"Ramblin\\' Wreck\", in rudimentary form since 1905 and as a complete stadium since 1913. The team participates in the NCAA Division I Football Bowl Subdivision as a member of the Atlantic Coast Conference. It is the oldest stadium in the FBS and has been the site of more home wins than any other FBS stadium.',\n",
       "  'a': 'North Avenue at Techwood Drive',\n",
       "  'gold_qtype': 'bridge',\n",
       "  'gold_q': 'Where is the stadium at which 1964 Georgia Tech Yellow Jackets football team played their home game located? ',\n",
       "  'qtype': 'bridge',\n",
       "  'b': 'Bobby Dodd Stadium',\n",
       "  'a_in_p1': False,\n",
       "  'a_in_p2': True,\n",
       "  'b_same_as_a': False,\n",
       "  '5_p2': True,\n",
       "  '5_p1': False,\n",
       "  's2': 'Bobby Dodd Stadium is located at the corner of North Avenue at Techwood Drive.',\n",
       "  '6_p1': 119,\n",
       "  '6_p2': None,\n",
       "  's1': 'Bobby Dodd Stadium was the home stadium of the 1964 Georgia Tech Yellow Jackets football team.',\n",
       "  's1_': 'Bobby Dodd Stadium was the home stadium of the 1964 Georgia Tech Yellow Jackets football team.',\n",
       "  'c': 'Bobby Dodd Stadium was the home stadium of the 1964 Georgia Tech Yellow Jackets football team and it is located at the corner of North Avenue at Techwood Drive.',\n",
       "  'c-b': 'The home stadium of the 1964 Georgia Tech Yellow Jackets football team is located at the corner of North Avenue and Techwood Drive.',\n",
       "  'c-a': 'The home stadium of the 1964 Georgia Tech Yellow Jackets football team is located at the corner of a certain street.',\n",
       "  'q': 'The home stadium of the 1964 Georgia Tech Yellow Jackets football team is located at the corner of which street?'}]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "989c11f7-eca4-4df1-b8d5-82bce32bc6a1",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8951520367183018\n",
      "0.9354561101549054\n"
     ]
    }
   ],
   "source": [
    "one = list(zip(this_is_it_plz_god0, [i['qtype'] for i in test_data]))\n",
    "print(sum([p==gt for p, gt in one])/len(test_data))\n",
    "print(sum([1 for p, gt in one if p in ('comparison', 'bridge')])/len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ccc1e79-88f0-431c-8864-7b230dcd6d6e",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "t5env",
   "language": "python",
   "name": "t5env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}