easse/scripts/reproduction/reproduction_bloom_by_ryan-eta-al-2023.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reproduction of BLOOM model as proposed in Ryan et al. (2023)\n",
    "\n",
    "\n",
    "### Credits:\n",
    "**Original Code**: [https://github.com/XenonMolecule/MultiSim](https://github.com/XenonMolecule/MultiSim)\n",
    "\n",
    "**Original Paper**: Michael Ryan, Tarek Naous, and Wei Xu. 2023. [Revisiting non-English Text Simplification: A Unified Multilingual Benchmark](https://aclanthology.org/2023.acl-long.269/). In *Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)*, pages 4898–4927, Toronto, Canada. Association for Computational Linguistics.\n",
    "\n",
    "\n",
    "### Instructions:\n",
    "To reproduce BLOOM as used in Ryan et al. (2023) for German text simplification, we slightly adapted their code. If you want to use this notebook please follow the installation intructions of the original code. Further, please add the German training and evaluation data to the required directories."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from dotenv import load_dotenv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import subprocess\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from numpy.random import default_rng\n",
    "from easse.sari import corpus_sari\n",
    "from sacrebleu import corpus_bleu\n",
    "import json\n",
    "import requests\n",
    "from collections import defaultdict\n",
    "from tqdm import tqdm\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# pip install ipywidgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "from huggingface_hub import HfFolder\n",
    "\n",
    "\n",
    "#enter your API key, you can make one for free on HF\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load_dotenv()\n",
    "token = HfFolder.get_token()  # os.environ.get(\"HUGGING_FACE_API_TOKEN\")\n",
    "print(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%env LASER=../../LASER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def laser_embed(df, name, split, laser_version=\"\"):\n",
    "    txt_path = \"./laser_embeddings/\" + name + \"_\" + split + \".txt\"\n",
    "    bin_path = \"./laser_embeddings/\" + name + \"_\" + split + \".bin\"\n",
    "    with open(txt_path, 'w') as f:\n",
    "        for txt in df['original']:\n",
    "            f.write(txt.replace('\\n','') + '\\n')\n",
    "    subprocess.run([\"bash\",\"../../LASER/tasks/embed/embed.sh\",txt_path,bin_path,laser_version])\n",
    "    os.remove(txt_path)\n",
    "\n",
    "def load_laser_embeddings(name, split):\n",
    "    dim = 1024\n",
    "    bin_path = \"./laser_embeddings/\" + name + \"_\" + split + \".bin\"\n",
    "\n",
    "    embeddings = np.fromfile(bin_path, dtype=np.float32, count=-1)                                                                          \n",
    "    embeddings.resize(embeddings.shape[0] // dim, dim)\n",
    "\n",
    "    return embeddings\n",
    "\n",
    "def calc_distances_to_neighbors(train_emb, eval_emb, neighbors):\n",
    "    # Find distances to all neighbors\n",
    "    A = train_emb[neighbors, :]\n",
    "    B = eval_emb\n",
    "\n",
    "    dot_product = np.dot(A, B.T).diagonal(0,0,2).T\n",
    "\n",
    "    # Compute the L2 norm of the vectors in A and B\n",
    "    norm_A = np.linalg.norm(A, axis=2)\n",
    "    norm_B = np.linalg.norm(B, axis=1)\n",
    "\n",
    "    # Compute the cosine distance between each pair of vectors using broadcasting\n",
    "    cosine_distances = 1 - (dot_product / (norm_A.T * norm_B).T)\n",
    "\n",
    "    return cosine_distances\n",
    "\n",
    "def generate_preprocessing_sim(name, train_emb, eval_emb, split=\"test\"):\n",
    "    K=20\n",
    "\n",
    "    model = NearestNeighbors(n_neighbors=K,\n",
    "                            metric='cosine',\n",
    "                            algorithm='brute',\n",
    "                            n_jobs=-1)\n",
    "    model.fit(train_emb)\n",
    "\n",
    "    closest_neighbors = model.kneighbors(eval_emb, return_distance=False)\n",
    "\n",
    "    cosine_distances = calc_distances_to_neighbors(train_emb, eval_emb, closest_neighbors)\n",
    "\n",
    "    pd.DataFrame(closest_neighbors).to_csv(\"./few_shot_preprocessing/\" + name + \"_\" + split + \"_similarity.csv\")\n",
    "    pd.DataFrame(cosine_distances).to_csv(\"./few_shot_preprocessing/\" + name + \"_\" + split + \"_similarity_dist.csv\")\n",
    "\n",
    "def generate_preprocessing_rand(name, train_emb, eval_emb, split=\"test\"):\n",
    "    K = 20\n",
    "\n",
    "    rng = np.random.default_rng(3600)\n",
    "    random_neighbors = rng.integers(low=0, high=train_emb.shape[0], size=(eval_emb.shape[0], K))\n",
    "    cosine_distances = calc_distances_to_neighbors(train_emb, eval_emb, random_neighbors)\n",
    "\n",
    "    pd.DataFrame(random_neighbors).to_csv(\"./few_shot_preprocessing/\" + name + \"_\" + split + \"_random.csv\")\n",
    "    pd.DataFrame(cosine_distances).to_csv(\"./few_shot_preprocessing/\" + name + \"_\" + split + \"_random_dist.csv\")\n",
    "\n",
    "\n",
    "def preprocess_dataset(train_path, test_path, name, split=\"test\"):\n",
    "    train = pd.read_csv(train_path)\n",
    "    test = pd.read_csv(test_path)\n",
    "    # train = train[(train['original'].notna()) & (train['simple'].notna()) ]\n",
    "    # test = test[(test['original'].notna()) & (test['simple'].notna()) ]\n",
    "    print(len(train))\n",
    "    print(len(test))\n",
    "\n",
    "    laser_version = \"\"\n",
    "    if name == \"SimplifyUR\":\n",
    "        laser_version = \"urd_Arab\"\n",
    "    \n",
    "    laser_embed(train, name, \"train\", laser_version)\n",
    "    laser_embed(test, name, \"test\", laser_version)\n",
    "\n",
    "    train_embeddings = load_laser_embeddings(name, \"train\")\n",
    "    test_embeddings = load_laser_embeddings(name, \"test\")\n",
    "\n",
    "    generate_preprocessing_sim(name, train_embeddings, test_embeddings, split)\n",
    "    generate_preprocessing_rand(name, train_embeddings, test_embeddings, split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_bleu_sari(df_ref, sentences):\n",
    "\n",
    "    num_refs = df_ref.shape[1]-1\n",
    "\n",
    "    bleu_scores = np.zeros((num_refs))\n",
    "    sari_scores = np.zeros((num_refs))\n",
    "\n",
    "    examples = [{\"original\": [], \"sentences\": [], \"references\": []} for _ in range(num_refs)]\n",
    "\n",
    "    assert df_ref.shape[0] == len(sentences)\n",
    "\n",
    "    for (index,row), sentence in zip(df_ref.iterrows(), sentences):\n",
    "        original = row['original']\n",
    "        simple = sentence\n",
    "        ref_list = []\n",
    "        for col in row.index:\n",
    "            if col != 'original' and type(row[col]) != float:\n",
    "                ref_list.append(row[col])\n",
    "        num_ref = len(ref_list)\n",
    "        examples[num_ref-1]['original'].append(original)\n",
    "        examples[num_ref-1]['sentences'].append(simple)\n",
    "        examples[num_ref-1]['references'].append(ref_list)\n",
    "\n",
    "    counts = np.array([len(e['original']) for e in examples])\n",
    "    total = sum(counts)\n",
    "    weights = np.divide(counts, total)\n",
    "\n",
    "    for i_bleu in range(len(examples)):\n",
    "        if counts[i_bleu] > 0:\n",
    "            references = np.array(examples[i_bleu]['references']).T.tolist()\n",
    "            bleu_scores[i_bleu] = corpus_bleu(\n",
    "                                examples[i_bleu]['sentences'],\n",
    "                                references,\n",
    "                                force = True,\n",
    "                                tokenize = '13a',\n",
    "                                lowercase = True\n",
    "                            ).score\n",
    "            sari_scores[i_bleu] = corpus_sari(\n",
    "                                orig_sents = examples[i_bleu]['original'],\n",
    "                                sys_sents = examples[i_bleu]['sentences'],\n",
    "                                refs_sents = references,\n",
    "                                tokenizer = '13a'\n",
    "                            )\n",
    "    \n",
    "    bleu = np.dot(bleu_scores, weights)\n",
    "    sari = np.dot(sari_scores, weights)\n",
    "\n",
    "    return bleu, sari"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "API_URL = \"https://api-inference.huggingface.co/models/bigscience/bloom\"\n",
    "# headers = {\"Authorization\": f\"Bearer {token}\"}\n",
    "token = \"hf_YOUR-HF-TOKEN\"  # todo: add your hugginface token here or use one of the previous methods to authenticate\n",
    "\n",
    "headers = {\"Authorization\": f\"Bearer {token}\"}\n",
    "\n",
    "# from huggingface_hub import InferenceApi\n",
    "\n",
    "# inference = InferenceApi(\"bigscience/bloom\",token=HfFolder.get_token())\n",
    "\n",
    "def query(payload):\n",
    "    # print(payload)\n",
    "    data = {\"inputs\": payload}\n",
    "    response = requests.post(API_URL, headers=headers, json=data)\n",
    "    return response.json()\n",
    "\n",
    "\n",
    "def load_fewshot_examples(train, test, mapping, offset=0):\n",
    "    output = defaultdict(lambda:[])\n",
    "    for j, (example,ref) in enumerate(zip(test['original'], test['simple'])):\n",
    "        output['original'].append(example)\n",
    "        output['ref'].append(ref)\n",
    "        i_off = 0\n",
    "        # print(j, mapping)\n",
    "        # print(mapping.iloc[j])\n",
    "        for i, idx in enumerate(mapping.iloc[j]):\n",
    "            if i != 0 and i > offset:\n",
    "                output[\"ex\" + str(i_off) + \"_orig\"].append(train.iloc[idx][\"original\"])\n",
    "                output[\"ex\" + str(i_off) + \"_simp\"].append(train.iloc[idx][\"simple\"])\n",
    "                i_off += 1\n",
    "    \n",
    "    out_df = pd.DataFrame(output)\n",
    "    return out_df\n",
    "    \n",
    "def construct_example(example_row, k=3):\n",
    "    output = []\n",
    "    for i_ex in range(k):\n",
    "        output.append(\"Original: \\\"\" + example_row[\"ex\" + str(i_ex) +\"_orig\"] + \"\\\"\\n\")\n",
    "        output.append(\"Simple: \\\"\" + example_row[\"ex\" + str(i_ex) + \"_simp\"] + \"\\\"\\n\\n\")\n",
    "\n",
    "    output.append(\"Original: \\\"\" + example_row[\"original\"] + \"\\\"\\nSimple: \\\"\")\n",
    "    return \"\".join(output)\n",
    "\n",
    "REQUERY_LIMIT = 5\n",
    "def generate_fewshot(example_row, k=3):\n",
    "    ex = construct_example(example_row, k=k)\n",
    "\n",
    "    new = \"\"\n",
    "    new_total = \"\"\n",
    "    for n in range(REQUERY_LIMIT):\n",
    "        response = query(ex)\n",
    "        if type(response) == dict and \"error\" in response.keys():\n",
    "            print(response)\n",
    "        \n",
    "        res = response[0]['generated_text']\n",
    "        new = res[len(ex):]\n",
    "        new_total += res[len(ex):]\n",
    "        if \"\\\"\\n\"\"\" in new_total:\n",
    "            # print(\"newline found\")\n",
    "            return new_total.split(\"\\\"\\n\"\"\")[0]\n",
    "        elif \"Original:\" in new_total:\n",
    "            # print(\"original found\")\n",
    "            return new_total.split(\"Original:\")[0]\n",
    "        else:\n",
    "            ex += new\n",
    "    return new_total\n",
    "\n",
    "def fewshot_eval(train_path, test_path, preprocessed_path, k=3, output_csv=\"\", checkpoint=\"\"):\n",
    "    train = pd.read_csv(train_path)\n",
    "    test = pd.read_csv(test_path)\n",
    "    preprocessed = pd.read_csv(preprocessed_path)\n",
    "    examples = load_fewshot_examples(train, test, preprocessed)\n",
    "    sentences = []\n",
    "    if (not checkpoint == \"\" and os.path.exists(checkpoint)):\n",
    "        ckpt = pd.read_csv(checkpoint)\n",
    "        sentences_pd = list(ckpt['fewshot output'])\n",
    "        sentences = []\n",
    "        for nr_sent, s in enumerate(sentences_pd):\n",
    "            # print(\"check\", s)\n",
    "            if not type(s) == float:\n",
    "                sentences.append(s)\n",
    "            else:\n",
    "                try:\n",
    "                    output = generate_fewshot(examples.iloc[nr_sent], k)\n",
    "                    # print(nr_sent, output)\n",
    "                    sentences.append(output)\n",
    "                except:\n",
    "                    # print(\"---\")\n",
    "                    # print(\"ERROR:  DUMPING GENERATED SENTENCES!\")\n",
    "                    # print()\n",
    "                    # print(sentences)\n",
    "                    # print()\n",
    "                    # print(\"ERROR ON \" + examples.iloc[i]['original'])\n",
    "                    # print(\"---\")\n",
    "                    sentences.append(\"\")\n",
    "            exit = True\n",
    "            for s in sentences_pd[nr_sent:]:\n",
    "                if not type(s) == float:\n",
    "                    exit = False\n",
    "            if exit:\n",
    "                break\n",
    "    for i_tqdm in tqdm(range(len(examples))):\n",
    "        if i_tqdm < len(sentences):\n",
    "            continue\n",
    "        row = examples.iloc[i_tqdm]\n",
    "        try:\n",
    "            output = generate_fewshot(row, k)\n",
    "            sentences.append(output)\n",
    "        except:\n",
    "            # print(\"---\")\n",
    "            # print(\"ERROR:  DUMPING GENERATED SENTENCES!\")\n",
    "            # print()\n",
    "            # print(sentences)\n",
    "            # print()\n",
    "            # print(\"ERROR ON \" + row['original'])\n",
    "            # print(\"---\")\n",
    "            sentences.append(\"\")\n",
    "    if not output_csv == \"\":\n",
    "        output = {\"original\":list(test['original']), \"fewshot output\": sentences}\n",
    "        output_df = pd.DataFrame(output)\n",
    "        output_df.to_csv(output_csv, index=False)\n",
    "    bleu, sari = calc_bleu_sari(test, sentences)\n",
    "    return bleu, sari\n",
    "\n",
    "# Try k-shots to fill in blanks, but if the error persists try k-=1\n",
    "def few_shot_backoff(train_path, test_path, preprocessed_path, k=3, output_csv=\"\", checkpoint=\"\"):\n",
    "    train = pd.read_csv(train_path)\n",
    "    test = pd.read_csv(test_path)\n",
    "    preprocessed = pd.read_csv(preprocessed_path)\n",
    "    examples = load_fewshot_examples(train, test, preprocessed)\n",
    "    sentences = []\n",
    "    if (not checkpoint == \"\" and os.path.exists(checkpoint)):\n",
    "        ckpt = pd.read_csv(checkpoint)\n",
    "        sentences_pd = list(ckpt['fewshot output'])\n",
    "        sentences = []\n",
    "        for i_sent, s in tqdm(enumerate(sentences_pd)):\n",
    "            if not type(s) == float:\n",
    "                sentences.append(s)\n",
    "            else:\n",
    "                curr_k = k\n",
    "                while curr_k >= 0:\n",
    "                    try:\n",
    "                        generated = generate_fewshot(examples.iloc[i_sent], curr_k)\n",
    "                        # print(i_sent, generated)\n",
    "                        sentences.append(generated)\n",
    "                        break\n",
    "                    except:\n",
    "                        curr_k -= 1\n",
    "                if curr_k < 0:\n",
    "                    print(\"ERROR ON INPUT: \" + examples.iloc[i_sent]['original'])\n",
    "        if not output_csv == \"\":\n",
    "            output = {\"original\":list(test['original']), \"fewshot output\": sentences}\n",
    "            output_df = pd.DataFrame(output)\n",
    "            output_df.to_csv(output_csv, index=False)\n",
    "        bleu, sari = calc_bleu_sari(test, sentences)\n",
    "        return bleu, sari\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query(\"The sense of life is \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "result_scores = [\n",
    "    ['name', \"type\", 'k', 'bleu', 'sari', 'bleu', 'sari'], \n",
    "     # ['TextComplexityDE', 'similarity', 0, 10.498930834393574, 35.270106181558134, 10.498930834393574, 35.270106181558134], \n",
    "    ['TextComplexityDE', 'similarity', 10, 7.205176465320519e-20, 27.05798251998486, 21.468699660404642, 39.8973797381036], \n",
    "     ['TextComplexityDE', 'random', 0, 8.295951045328993, 34.93129793635445, 8.295951045328993, 34.93129793635445],\n",
    "    ['TextComplexityDE', 'random', 10, 0.0, 26.674574612964037, 15.973901603823865, 35.50831232720633],\n",
    "    \n",
    "    ['GEOLino', 'random', 0, 29.117056851318097, 28.713748042875253, 29.117056851318097, 28.713748042875253], \n",
    "    ['GEOLino', 'random', 10, 51.56820676420725, 37.18657466545419, 51.56820676420725, 37.18657466545419],\n",
    "    ['GEOLino', 'similarity', 10, 12.141214691406692, 28.219600294595057, 49.97869326480118, 40.63871137249447], \n",
    "    # ['GEOLino', 'random', 0, 29.117056851318097, 28.713748042875253, 29.117056851318097, 28.713748042875253], \n",
    "    \n",
    "     # ['GEOLino-full', 'similarity', 0, 29.501461657873758, 32.11658520872407, 29.501461657873758, 32.11658520872407],\n",
    "    \n",
    "    ['GEOLino-full', 'random-zero', 0, 29.501461657873758, 32.11658520872407, 29.501461657873758, 32.11658520872407],\n",
    "    ['GEOLino-full', 'random', 10, 46.84634239044782, 38.843746957264734, 48.7504291447407, 39.10700124798857],\n",
    "    ['GEOLino-full', 'similarity', 10, 22.540366238748927, 37.8468078899627, 58.02791540402962, 50.112829617920376],\n",
    "    \n",
    "    ['simple-german-corpus', 'random-zero', 0, 4.078741855153335, 31.311442966902234, 4.078741855153335, 31.311442966902234],\n",
    "    ['simple-german-corpus', 'random', 10, 4.123620228852482, 33.49020474271444, 5.022534384992534, 32.49837499073984],\n",
    "    ['simple-german-corpus', 'similarity', 10, 3.179767831264162, 40.01604919508432, 14.107995580454697, 44.685470973766314],\n",
    "     \n",
    "    ['DEplain-APA', 'random', 0, 17.23222418819839, 35.191632840161695, 17.23222418819839, 35.191632840161695],\n",
    "    ['DEplain-APA', 'random', 10, 19.2266957201598, 35.52235825908088, 19.2266957201598, 35.52235825908088],\n",
    "    ['DEplain-APA', 'similarity', 10, 21.616008179234996, 41.29645080601255, 22.205236460328297, 41.20599406417358],\n",
    "    \n",
    "    ['DEplain-web', 'random', 0, 11.444314254398583, 30.76291493800329, 11.444314254398583, 30.76291493800329],\n",
    "    ['DEplain-web', 'random', 10, 3.20, 33.43, 11.54, 30.97],\n",
    "    ['DEplain-web', 'similarity', 10, 0.10, 33.97, 12.07, 37.10],\n",
    "    \n",
    "    ['BiSECT', 'random', 10, 0.0, 26.25786416446166, 16.309267554152836, 37.27606416280923], \n",
    "    ['BiSECT', 'similarity', 10, 0.0, 26.25786416446166, 15.936765840861158, 37.31072088667093],\n",
    "    \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# result_scores = [[\"name\", \"type\", \"k\", \"bleu\", \"sari\", \"bleu\", \"sari\"]]\n",
    "for data_path in [\n",
    "                \"../../data/German/GEOLino Corpus_\", \n",
    "                \"../../data/German/TextComplexityDE Parallel Corpus_\",\n",
    "                \"../../data/German/GEOLino-full_\", \n",
    "                \"../../data/German/TextComplexityDE-full_\", # only zero-shot\n",
    "                 \"../../data/German/simple-german-corpus_\", \n",
    "                  \"../../data/German/DEplain-APA_\", \n",
    "                  \"../../data/German/DEplain-web_\", \n",
    "                 \"../../data/German/BiSECT_\", \n",
    "                 \"../../data/German/ABGB_\", # only zero-shot\n",
    "                 \"../../data/German/APA-LHA-or-a2_\",\n",
    "                 \"../../data/German/APA-LHA-or-b1_\",\n",
    "                 ]:\n",
    "    train_set = data_path + \"train.csv\"\n",
    "    test_set = data_path + \"test.csv\"\n",
    "    \n",
    "    name = data_path.split(\"/\")[-1][:-1].split(\" \")[0]\n",
    "    print(name)\n",
    "    if \"TextComplexityDE-full_\" in data_path or \"ABGB\" in data_path:\n",
    "        train_set = data_path + \"test.csv\"\n",
    "    \n",
    "    if not os.path.exists(\"../../fewshot-outputs/\"+name):\n",
    "        os.makedirs(\"../../fewshot-outputs/\"+name)\n",
    "    \n",
    "    preprocess_dataset(train_set, test_set, name)\n",
    "    \n",
    "    for demonstration, k in [(\"random\", 0), (\"random\", 10), (\"similarity\", 10)]:\n",
    "        split = \"test\"\n",
    "\n",
    "        \n",
    "        # for k in [0]:    \n",
    "        print(name, demonstration, k, split)\n",
    "        result_row = [name, demonstration, k]\n",
    "        print(\"TESTING \" + str(k) + \"-SHOT:\")\n",
    "        mapping = \"./few_shot_preprocessing/\" + name + \"_\" + split + \"_\" + demonstration + \".csv\"\n",
    "        dem = \"sim\" if (demonstration == \"similarity\") else (\"rand\" if (demonstration == \"random\") else \"unk\")\n",
    "        output = \"../../fewshot-outputs/\" + name + \"/\" + str(k) + \".\" + dem + \".csv\"\n",
    "        bleu, sari = fewshot_eval(train_set, test_set, mapping, k=k, output_csv=output, checkpoint=output)\n",
    "        print(\"BLEU\", bleu)\n",
    "        print(\"SARI\", sari)\n",
    "        result_row.extend([bleu, sari])\n",
    "\n",
    "        bleu, sari = few_shot_backoff(train_set, test_set, mapping, k=k, output_csv=output, checkpoint=output)\n",
    "        print(\"BLEU\", bleu)\n",
    "        print(\"SARI\", sari)\n",
    "        result_row.extend([bleu, sari])\n",
    "        result_scores.append(result_row)\n",
    "        print(result_scores)\n",
    "        \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(result_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(result_scores[1:], columns=result_scores[0]).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(result_scores[1:], columns=result_scores[0]).round(2).to_csv(\"result_scores.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# few_shot_backoff(\"../data/Urdu/SimplifyUR_train.csv\", \"../data/Urdu/SimplifyUR_test.csv\", \"./few_shot_preprocessing/SimplifyUR_test_similarity.csv\", k=5, output_csv=\"../../fewshot-outputs/SimplifyUR/5.sim.csv\", checkpoint=\"../../fewshot-outputs/SimplifyUR/5.sim.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for input_path in [\n",
    "                # \"../../data/German/GEOLino Corpus_\", \n",
    "                # \"../../data/German/TextComplexityDE Parallel Corpus_\",\n",
    "                # \"../../data/German/GEOLino-full_\", \n",
    "                # \"../../data/German/TextComplexityDE-full_\", # only zero-shot\n",
    "                 # \"../../data/German/simple-german-corpus_\", \n",
    "                 #  \"../../data/German/DEplain-APA_\", \n",
    "                 #  \"../../data/German/DEplain-web_\", \n",
    "                 (\"BiSECT\", \"BiSECT\")\n",
    "                 (\"ABGB\", \"ABGB\") # only zero-shot\n",
    "                 # \"../../data/German/APA-LHA-or-a2_\",\n",
    "                 # \"../../data/German/APA-LHA-or-b1_\",\n",
    "                (\"TextComplexityDE-full\", \"TextComplexityDE\"),\n",
    "                (\"TextComplexityDE\", \"tcde-small\"),\n",
    "                (\"GEOLino-full\", \"geolino\"),\n",
    "                (\"GEOLino\", \"geolino-small\")\n",
    "                 ]:\n",
    "    # _,_, d, lang, name = input_path.split(\"/\")\n",
    "    name, output_name = input_path\n",
    "    if \"TextComplexityDE-full\" in name or \"ABGB\" in name:\n",
    "        t_list = [(\"0.rand\", \"0-random\")]\n",
    "    else:\n",
    "        t_list = [(\"0.rand\", \"0-random\"), (\"10.rand\", \"10-random\"), (\"10.sim\", \"10-similarity\")]\n",
    "    for t, t_name in t_list:\n",
    "        print(name)\n",
    "        data = pd.read_csv(\"../../fewshot-outputs/\"+name+\"/\"+t+\".csv\")\n",
    "        # print(data.head())\n",
    "        with open(\"/home/SSD1TB/easse-de-clean/easse-de/easse/resources/data/system_outputs/sentence_level/\"+output_name+\"/test/BLOOM-\"+t_name+\".txt\", \"w\") as f:\n",
    "            for i,row in data.iterrows():\n",
    "                text = row[\"fewshot output\"]\n",
    "                text = text.replace(\"\\n\",\" \")\n",
    "                f.write(text+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}