Cross Language Information Retrieval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Cross Language Information Retrieval\n",
    "\n",
    "Author of the Project is Kavin Jindel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "import math\n",
    "\n",
    "#Function to tokenise string/sentences.\n",
    "def tokenize(line, tokenizer=word_tokenize):\n",
    "    utf_line = line.lower()\n",
    "    return [token for token in tokenizer(utf_line)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['seit',\n",
       " 'damals',\n",
       " 'ist',\n",
       " 'er',\n",
       " 'auf',\n",
       " 'über',\n",
       " '10.000',\n",
       " 'punkte',\n",
       " 'gestiegen',\n",
       " '.']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenize(\"Seit damals ist er auf über 10.000 Punkte gestiegen.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEVELOPMENT_DOCS = 'data/clir/devel.docs' #Data file for IR engine development\n",
    "\n",
    "DEVELOPMENT_QUERIES = 'data/clir/devel.queries' #Data file containing queries in German\n",
    "\n",
    "DEVELOPMENT_QREL = 'data/clir/devel.qrel' #Data file containing a relevance score or query-doc pairs\n",
    "\n",
    "BITEXT_ENG = 'data/clir/bitext.en' #Bitext data file in English for translation engine and language model development\n",
    "\n",
    "BITEXT_DE = 'data/clir/bitext.de' #Bitext data file in German\n",
    "\n",
    "NEWSTEST_ENG = 'data/clir/newstest.en' #File for testing language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import re\n",
    "\n",
    "stopwords = set(nltk.corpus.stopwords.words('english')) #converting stopwords to a set for faster processing in the future.\n",
    "stemmer = nltk.stem.PorterStemmer() \n",
    "\n",
    "#Function to extract and tokenize terms from a document\n",
    "def extract_and_tokenize_terms(doc):\n",
    "    terms = []\n",
    "    for token in tokenize(doc):\n",
    "        if token not in stopwords: # 'in' and 'not in' operations are faster over sets than lists\n",
    "            if not re.search(r'\\d',token) and not re.search(r'[^A-Za-z-]',token): #Removing numbers and punctuations \n",
    "                #(excluding hyphenated words)\n",
    "                terms.append(stemmer.stem(token.lower()))\n",
    "    return terms\n",
    "\n",
    "documents = {} #Dictionary to store documents with ids as keys."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reading each line in the file and storing it documents dictionary\n",
    "f = open(DEVELOPMENT_DOCS)\n",
    "\n",
    "for line in f:\n",
    "    doc = line.split(\"\\t\")\n",
    "    terms = extract_and_tokenize_terms(doc[1])\n",
    "    documents[doc[0]] = terms\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['name',\n",
       " 'plural',\n",
       " 'ae',\n",
       " 'first',\n",
       " 'letter',\n",
       " 'vowel',\n",
       " 'iso',\n",
       " 'basic',\n",
       " 'latin',\n",
       " 'alphabet',\n",
       " 'similar',\n",
       " 'ancient',\n",
       " 'greek',\n",
       " 'letter',\n",
       " 'alpha',\n",
       " 'deriv',\n",
       " 'upper',\n",
       " 'case',\n",
       " 'version',\n",
       " 'consist']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents['290'][:20] #To keep things short, we're only going to check out 20 tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Building an inverted index for the documents\n",
    "\n",
    "from collections import defaultdict\n",
    "    \n",
    "inverted_index = defaultdict(set)\n",
    "\n",
    "for docid, terms in documents.items():\n",
    "    for term in terms:\n",
    "        inverted_index[term].add(docid)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'121569',\n",
       " '16553',\n",
       " '212541',\n",
       " '228211',\n",
       " '261023',\n",
       " '265975',\n",
       " '276433',\n",
       " '64083',\n",
       " '69930',\n",
       " '72701',\n",
       " '73441',\n",
       " '74323'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inverted_index['pizza']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Building a TF-IDF representation using BM25 \n",
    "\n",
    "NO_DOCS = len(documents) #Number of documents\n",
    "\n",
    "AVG_LEN_DOC = sum([len(doc) for doc in documents.values()])/len(documents) #Average length of documents\n",
    "\n",
    "def tf_idf_score(k1,b,term,docid):  \n",
    "    \n",
    "    ft = len(inverted_index[term]) \n",
    "    term = stemmer.stem(term.lower())\n",
    "    fdt =  documents[docid].count(term)\n",
    "    \n",
    "    idf_comp = math.log((NO_DOCS - ft + 0.5)/(ft+0.5))\n",
    "    \n",
    "    tf_comp = ((k1 + 1)*fdt)/(k1*((1-b) + b*(len(documents[docid])/AVG_LEN_DOC))+fdt)\n",
    "    \n",
    "    return idf_comp * tf_comp\n",
    "\n",
    "def create_tf_idf(k1,b):\n",
    "    tf_idf = defaultdict(dict)\n",
    "    for term in set(inverted_index.keys()):\n",
    "        for docid in inverted_index[term]:\n",
    "            tf_idf[term][docid] = tf_idf_score(k1,b,term,docid)\n",
    "    return tf_idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_idf = create_tf_idf(1.5,0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_qtf_comp(k3,term,fqt):\n",
    "    return ((k3+1)*fqt[term])/(k3 + fqt[term])\n",
    "\n",
    "\n",
    "def retr_docs(query,result_count):\n",
    "    q_terms = [stemmer.stem(term.lower()) for term in query.split() if term not in stopwords] \n",
    "    fqt = {}\n",
    "    for term in q_terms:\n",
    "        fqt[term] = fqt.get(term,0) + 1\n",
    "    \n",
    "    scores = {}\n",
    "    \n",
    "    for word in fqt.keys():\n",
    "        for document in inverted_index[word]:\n",
    "            scores[document] = scores.get(document,0) + (tf_idf[word][document]*get_qtf_comp(0,word,fqt)) \n",
    "    \n",
    "    return sorted(scores.items(),key = lambda x : x[1] , reverse=True)[:result_count]        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('19961', 12.570768983660066),\n",
       " ('83266', 12.500411244644424),\n",
       " ('266959', 12.464229013042164),\n",
       " ('20206', 12.324367209127187),\n",
       " ('253314', 12.008594517514213)]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retr_docs(\"Manchester United\",5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['manchest',\n",
       " 'unit',\n",
       " 'manchest',\n",
       " 'unit',\n",
       " 'footbal',\n",
       " 'club',\n",
       " 'english',\n",
       " 'profession',\n",
       " 'footbal',\n",
       " 'club',\n",
       " 'base',\n",
       " 'old',\n",
       " 'trafford',\n",
       " 'greater',\n",
       " 'manchest',\n",
       " 'play',\n",
       " 'premier',\n",
       " 'leagu',\n",
       " 'found',\n",
       " 'newton',\n",
       " 'heath',\n",
       " 'lyr',\n",
       " 'footbal',\n",
       " 'club',\n",
       " 'club',\n",
       " 'chang',\n",
       " 'name',\n",
       " 'manchest',\n",
       " 'unit',\n",
       " 'move']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents['19961'][:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Calculating the unigram, bigram and trigram counts. \n",
    "\n",
    "f = open(BITEXT_ENG)\n",
    "\n",
    "train_sentences = []\n",
    "\n",
    "for line in f:\n",
    "    train_sentences.append(tokenize(line))\n",
    "\n",
    "f.close()    \n",
    "\n",
    "#Function to mark the first occurence of words as unknown, for training.\n",
    "def check_for_unk_train(word,unigram_counts):\n",
    "    if word in unigram_counts:\n",
    "        return word\n",
    "    else:\n",
    "        unigram_counts[word] = 0\n",
    "        return \"UNK\"\n",
    "\n",
    "#Function to convert sentences for training the language model.    \n",
    "def convert_sentence_train(sentence,unigram_counts):\n",
    "    #<s1> and <s2> are sentinel tokens added to the start and end, for handling tri/bigrams at the start of a sentence.\n",
    "    return [\"<s1>\"] + [\"<s2>\"] + [check_for_unk_train(token.lower(),unigram_counts) for token in sentence] + [\"</s2>\"]+ [\"</s1>\"]\n",
    "\n",
    "#Function to obtain unigram, bigram and trigram counts.\n",
    "def get_counts(sentences):\n",
    "    trigram_counts = defaultdict(lambda: defaultdict(dict))\n",
    "    bigram_counts = defaultdict(dict)\n",
    "    unigram_counts = {}\n",
    "    for sentence in sentences:\n",
    "        sentence = convert_sentence_train(sentence, unigram_counts)\n",
    "        for i in range(len(sentence) - 2):\n",
    "            trigram_counts[sentence[i]][sentence[i+1]][sentence[i+2]] = trigram_counts[sentence[i]][sentence[i+1]].get(sentence[i+2],0) + 1\n",
    "            bigram_counts[sentence[i]][sentence[i+1]] = bigram_counts[sentence[i]].get(sentence[i+1],0) + 1\n",
    "            unigram_counts[sentence[i]] = unigram_counts.get(sentence[i],0) + 1\n",
    "    unigram_counts[\"</s1>\"] = unigram_counts[\"<s1>\"]\n",
    "    unigram_counts[\"</s2>\"] = unigram_counts[\"<s2>\"]\n",
    "    bigram_counts[\"</s2>\"][\"</s1>\"] = bigram_counts[\"<s1>\"][\"<s2>\"]\n",
    "    return unigram_counts, bigram_counts, trigram_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "unigram_counts, bigram_counts,trigram_counts = get_counts(train_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Constructing unigram model with 'add-k' smoothing\n",
    "token_count = sum(unigram_counts.values())\n",
    "\n",
    "#Function to convert unknown words for testing. \n",
    "#Words that don't appear in the training corpus (even if they are in the test corpus) are marked as UNK.\n",
    "def check_for_unk_test(word,unigram_counts):\n",
    "    if word in unigram_counts and unigram_counts[word] > 0:\n",
    "        return word\n",
    "    else:\n",
    "        return \"UNK\"\n",
    "\n",
    "\n",
    "def convert_sentence_test(sentence,unigram_counts):\n",
    "    return [\"<s1>\"] + [\"<s2>\"] + [check_for_unk_test(word.lower(),unigram_counts) for word in sentence] + [\"</s2>\"]  + [\"</s1>\"]\n",
    "\n",
    "#Returns the log probability of a unigram, with add-k smoothing. We're taking logs to avoid probability underflow.\n",
    "def get_log_prob_addk(word,unigram_counts,k):\n",
    "    return math.log((unigram_counts[word] + k)/ \\\n",
    "                    (token_count + k*len(unigram_counts)))\n",
    "\n",
    "#Returns the log probability of a sentence.\n",
    "def get_sent_log_prob_addk(sentence, unigram_counts,k):\n",
    "    sentence = convert_sentence_test(sentence, unigram_counts)\n",
    "    return sum([get_log_prob_addk(word, unigram_counts,k) for word in sentence])\n",
    "\n",
    "\n",
    "def calculate_perplexity_uni(sentences,unigram_counts, token_count, k):\n",
    "    total_log_prob = 0\n",
    "    test_token_count = 0\n",
    "    for sentence in sentences:\n",
    "        test_token_count += len(sentence) + 2 # have to consider the end token\n",
    "        total_log_prob += get_sent_log_prob_addk(sentence,unigram_counts,k)\n",
    "    return math.exp(-total_log_prob/test_token_count)\n",
    "\n",
    "\n",
    "f = open(NEWSTEST_ENG)\n",
    "\n",
    "test_sents = []\n",
    "for line in f:\n",
    "    test_sents.append(tokenize(line))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0001: 631.5624344258267\n",
      "0.01: 631.6372709016225\n",
      "0.1: 632.3728621611093\n",
      "1: 643.2578962510463\n",
      "10: 814.7925245672897\n"
     ]
    }
   ],
   "source": [
    "#Calculating the perplexity for different ks\n",
    "ks = [0.0001,0.01,0.1,1,10]\n",
    "\n",
    "for k in ks:\n",
    "    print(str(k) +\": \" + str(calculate_perplexity_uni(test_sents,unigram_counts,token_count,k)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Calculating the N1/N paramaters for Trigrams/Bigrams/Unigrams in Katz-Backoff Smoothing\n",
    "\n",
    "TRI_ONES = 0 #N1 for Trigrams\n",
    "TRI_TOTAL = 0 #N for Trigrams\n",
    "\n",
    "for twod in trigram_counts.values():\n",
    "    for oned in twod.values():\n",
    "        for val in oned.values():\n",
    "            if val==1:\n",
    "                TRI_ONES+=1 #Count of trigram seen once\n",
    "            TRI_TOTAL += 1 #Count of all trigrams seen\n",
    "\n",
    "BI_ONES = 0 #N1 for Bigrams\n",
    "BI_TOTAL = 0 #N for Bigrams\n",
    "\n",
    "for oned in bigram_counts.values():\n",
    "    for val in oned.values():\n",
    "        if val==1:\n",
    "            BI_ONES += 1 #Count of bigram seen once\n",
    "        BI_TOTAL += 1 #Count of all bigrams seen\n",
    "        \n",
    "UNI_ONES = list(unigram_counts.values()).count(1)\n",
    "UNI_TOTAL = len(unigram_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Constructing trigram model with backoff smoothing\n",
    "\n",
    "TRI_ALPHA = TRI_ONES/TRI_TOTAL #Alpha parameter for trigram counts\n",
    "    \n",
    "BI_ALPHA = BI_ONES/BI_TOTAL #Alpha parameter for bigram counts\n",
    "\n",
    "UNI_ALPHA = UNI_ONES/UNI_TOTAL\n",
    "    \n",
    "def get_log_prob_back(sentence,i,unigram_counts,bigram_counts,trigram_counts,token_count):\n",
    "    if trigram_counts[sentence[i-2]][sentence[i-1]].get(sentence[i],0) > 0:\n",
    "        return math.log((1-TRI_ALPHA)*trigram_counts[sentence[i-2]][sentence[i-1]].get(sentence[i])/bigram_counts[sentence[i-2]][sentence[i-1]])\n",
    "    else:\n",
    "        if bigram_counts[sentence[i-1]].get(sentence[i],0)>0:\n",
    "            return math.log(TRI_ALPHA*((1-BI_ALPHA)*bigram_counts[sentence[i-1]][sentence[i]]/unigram_counts[sentence[i-1]]))\n",
    "        else:\n",
    "            return math.log(TRI_ALPHA*BI_ALPHA*(1-UNI_ALPHA)*((unigram_counts[sentence[i]]+0.0001)/(token_count+(0.0001)*len(unigram_counts)))) \n",
    "        \n",
    "        \n",
    "def get_sent_log_prob_back(sentence, unigram_counts, bigram_counts,trigram_counts, token_count):\n",
    "    sentence = convert_sentence_test(sentence, unigram_counts)\n",
    "    return sum([get_log_prob_back(sentence,i, unigram_counts,bigram_counts,trigram_counts,token_count) for i in range(2,len(sentence))])\n",
    "\n",
    "\n",
    "def calculate_perplexity_tri(sentences,unigram_counts,bigram_counts,trigram_counts, token_count):\n",
    "    total_log_prob = 0\n",
    "    test_token_count = 0\n",
    "    for sentence in sentences:\n",
    "        test_token_count += len(sentence) + 2 # have to consider the end token\n",
    "        total_log_prob += get_sent_log_prob_back(sentence,unigram_counts,bigram_counts,trigram_counts,token_count)\n",
    "    return math.exp(-total_log_prob/test_token_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "463.6271760983476"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Calculating the perplexity \n",
    "calculate_perplexity_tri(test_sents,unigram_counts,bigram_counts,trigram_counts,token_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating lists of English and German sentences from bitext.\n",
    "\n",
    "from nltk.translate import IBMModel1\n",
    "from nltk.translate import AlignedSent, Alignment\n",
    "\n",
    "eng_sents = []\n",
    "de_sents = []\n",
    "\n",
    "f = open(BITEXT_ENG)\n",
    "for line in f:\n",
    "    terms = tokenize(line)\n",
    "    eng_sents.append(terms)\n",
    "f.close()\n",
    "\n",
    "f = open(BITEXT_DE)\n",
    "for line in f:\n",
    "    terms = tokenize(line)\n",
    "    de_sents.append(terms)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Zipping together the bitexts for easier access\n",
    "paral_sents = list(zip(eng_sents,de_sents))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Building English to German translation table for words (Backward alignment)\n",
    "eng_de_bt = [AlignedSent(E,G) for E,G in paral_sents]\n",
    "eng_de_m = IBMModel1(eng_de_bt, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Building German to English translation table for words (Backward alignment)\n",
    "de_eng_bt = [AlignedSent(G,E) for E,G in paral_sents]\n",
    "de_eng_m = IBMModel1(de_eng_bt, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Script below to combine alignments using set intersections\n",
    "combined_align = []\n",
    "\n",
    "for i in range(len(eng_de_bt)):\n",
    "\n",
    "    forward = {x for x in eng_de_bt[i].alignment}\n",
    "    back_reversed = {x[::-1] for x in de_eng_bt[i].alignment}\n",
    "    \n",
    "    combined_align.append(forward.intersection(back_reversed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating German to English dictionary with occurence count of word pairs\n",
    "de_eng_count = defaultdict(dict)\n",
    "\n",
    "for i in range(len(de_eng_bt)):\n",
    "    for item in combined_align[i]:\n",
    "        de_eng_count[de_eng_bt[i].words[item[1]]][de_eng_bt[i].mots[item[0]]] =  de_eng_count[de_eng_bt[i].words[item[1]]].get(de_eng_bt[i].mots[item[0]],0) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a English to German dict with occ count of word pais\n",
    "eng_de_count = defaultdict(dict)\n",
    "\n",
    "for i in range(len(eng_de_bt)):\n",
    "    for item in combined_align[i]:\n",
    "        eng_de_count[eng_de_bt[i].words[item[0]]][eng_de_bt[i].mots[item[1]]] =  eng_de_count[eng_de_bt[i].words[item[0]]].get(eng_de_bt[i].mots[item[1]],0) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating German to English table with word translation probabilities\n",
    "de_eng_prob = defaultdict(dict)\n",
    "\n",
    "for de in de_eng_count.keys():\n",
    "    for eng in de_eng_count[de].keys():\n",
    "        de_eng_prob[de][eng] = de_eng_count[de][eng]/sum(de_eng_count[de].values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating English to German dict with word translation probabilities \n",
    "eng_de_prob = defaultdict(dict)\n",
    "\n",
    "for eng in eng_de_count.keys():\n",
    "    for de in eng_de_count[eng].keys():\n",
    "        eng_de_prob[eng][de] = eng_de_count[eng][de]/sum(eng_de_count[eng].values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'question': 1.0}\n",
      "{'spans': 0.5, 'side': 0.5}\n",
      "{'house': 0.625, 'charity': 0.125, 'hospitalized': 0.125, 'offset': 0.125}\n"
     ]
    }
   ],
   "source": [
    "#Examples of translating individual words from German to English\n",
    "print(de_eng_prob['frage'])\n",
    "\n",
    "print(de_eng_prob['handlung'])\n",
    "\n",
    "print(de_eng_prob['haus'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Building noisy channel translation model\n",
    "def de_eng_noisy(german):\n",
    "    noisy={}\n",
    "    for eng in de_eng_prob[german].keys():\n",
    "        noisy[eng] = eng_de_prob[eng][german]+ get_log_prob_addk(eng,unigram_counts,0.0001)\n",
    "    return noisy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\n",
      "{'house': -8.153224281882322, 'charity': -11.032065847914977, 'hospitalized': -11.319739587276473, 'offset': -11.225188029412386}\n",
      "{'this': -4.991400947456017, 'that': -4.857303935975769, 'is': -4.278625574868814, 'the': -3.0973206970208214, 'for': -5.074260930518287, 'of': -3.88129895540149}\n",
      "{'pledging': -21.128725580698557}\n"
     ]
    }
   ],
   "source": [
    "#Test block to check alignments\n",
    "print(de_eng_noisy('vater'))\n",
    "print(de_eng_noisy('haus'))\n",
    "print(de_eng_noisy('das'))\n",
    "print(de_eng_noisy('entschuldigung'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bereue': 1.0}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eng_de_prob['sorry']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "82\n",
      "German: der ( von engl . action : tat , handlung , bewegung ) ist ein filmgenre des unterhaltungskinos , in welchem der fortgang der äußeren handlung von zumeist spektakulär inszenierten kampf - und gewaltszenen vorangetrieben und illustriert wird .\n",
      "\n",
      "English: the ( , leninism . action : rattling , side , movement ) is a filmgenre the unterhaltungskinos , in paulson the fortgang the trumpet side , zumeist spektakulär inszenierten fight - and gewaltszenen annan and illustriert is .\n",
      "\n",
      "\n",
      "116\n",
      "German: die ( einheitenzeichen : u für unified atomic mass unit , veraltet amu für atomic mass unit ) ist eine maßeinheit der masse .\n",
      "\n",
      "English: the ( einheitenzeichen : u for unified atomic manipulation unit , regime amu for atomic manipulation unit ) is a befuddled the masse .\n",
      "\n",
      "\n",
      "240\n",
      "German: der von lateinisch actualis , \" wirklich \" , auch aktualitätsprinzip , uniformitäts - oder gleichförmigkeitsprinzip , englisch uniformitarianism , ist die grundlegende wissenschaftliche methode in der .\n",
      "\n",
      "English: the , lateinisch actualis , `` really `` , not aktualitätsprinzip , uniformitäts - or gleichförmigkeitsprinzip , english uniformitarianism , is the basic intended method in the .\n",
      "\n",
      "\n",
      "320\n",
      "German: die ( griechisch el , von altgriechisch grc , - \" zusammen - \" , \" anbinden \" , gemeint ist \" die herzbeutel angehängte \" ) , ist ein blutgefäß , welches das blut vom herz wegführt .\n",
      "\n",
      "English: the ( griechisch el , , altgriechisch grc , - `` zusammen - `` , `` anbinden `` , meant is `` the herzbeutel angehängte `` ) , is a blutgefäß , welches the blood vom herz wegführt .\n",
      "\n",
      "\n",
      "540\n",
      "German: unter der bezeichnung fasst man die drei im nördlichen alpenvorland liegenden gewässereinheiten obersee , untersee und seerhein zusammen .\n",
      "\n",
      "English: under the eurasian fasst man the three the nördlichen alpenvorland underlying gewässereinheiten obersee , untersee and seerhein zusammen .\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Translating first 5 queries into English\n",
    "\n",
    "#Function for direct translation\n",
    "def de_eng_direct(query):\n",
    "    query_english = [] \n",
    "    query_tokens = tokenize(query)\n",
    "    \n",
    "    for token in query_tokens:\n",
    "        try:\n",
    "            query_english.append(max(de_eng_prob[token], key=de_eng_prob[token].get))\n",
    "        except:\n",
    "            query_english.append(token) #Returning the token itself when it cannot be found in the translation table.\n",
    "            #query_english.append(\"NA\") \n",
    "    \n",
    "    return \" \".join(query_english)\n",
    "\n",
    "#Function for noisy channel translation\n",
    "def de_eng_noisy_translate(query):  \n",
    "    query_english = [] \n",
    "    query_tokens = tokenize(query)\n",
    "    \n",
    "    for token in query_tokens:\n",
    "        try:\n",
    "            query_english.append(max(de_eng_noisy(token), key=de_eng_noisy(token).get))\n",
    "        except:\n",
    "            query_english.append(token) #Returning the token itself when it cannot be found in the translation table.\n",
    "            #query_english.append(\"NA\") \n",
    "    \n",
    "    return \" \".join(query_english)\n",
    "            \n",
    "f = open(DEVELOPMENT_QUERIES)\n",
    "\n",
    "lno = 0\n",
    "plno = 0\n",
    "\n",
    "#Also building a dictionary of query ids and query content (only for the first 100s)\n",
    "german_qs = {}\n",
    "\n",
    "test_query_trans_sents = [] #Building a list for perplexity checks.\n",
    "\n",
    "for line in f:\n",
    "    lno+=1\n",
    "    query_id = line.split('\\t')[0]\n",
    "    query_german = line.split('\\t')[1]  \n",
    "    \n",
    "    german_qs[query_id] = query_german.strip()\n",
    "    \n",
    "    translation = str(de_eng_noisy_translate(query_german))\n",
    " \n",
    "    if plno<5:\n",
    "        print(query_id + \"\\n\" + \"German: \" + str(query_german) + \"\\n\" + \"English: \" + translation +\"\\n\\n\")\n",
    "        plno+=1\n",
    "    test_query_trans_sents.append(translation)\n",
    "    if lno==100:\n",
    "        break\n",
    "\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Building a dictionary for queryids and relevant document ids\n",
    "qrel = defaultdict(list)\n",
    "\n",
    "f = open(DEVELOPMENT_QREL)\n",
    "\n",
    "for line in f:\n",
    "    item = line.split('\\t')\n",
    "    qrel[item[0]].append(item[2])\n",
    "    \n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Single function to retreive documents for a German query\n",
    "def trans_retr_docs(german_query,no_of_results,translation_function):\n",
    "    \n",
    "    trans_query = \" \".join(extract_and_tokenize_terms(translation_function(german_query)))\n",
    "    return [item[0] for item in retr_docs(trans_query,no_of_results)] #Retriving 100 documents\n",
    "\n",
    "#Calculating the map score\n",
    "def calc_map(no_of_results,translation_function):\n",
    "    \n",
    "    average_precision = []\n",
    "    \n",
    "    for gq in german_qs.keys():\n",
    "        \n",
    "        relevant_docs = qrel[gq]\n",
    "        incremental_precision = []\n",
    "        resulting_docs = trans_retr_docs(german_qs[gq],no_of_results,translation_function)\n",
    "        \n",
    "        total_counter = 0\n",
    "        true_positive_counter = 0\n",
    "        \n",
    "        for doc in resulting_docs:\n",
    "            total_counter+=1\n",
    "            if doc in relevant_docs:\n",
    "                true_positive_counter += 1\n",
    "                incremental_precision.append(true_positive_counter/total_counter)\n",
    "        \n",
    "        #For no relevant retreivals, the average precision will be considered 0.\n",
    "        try:\n",
    "            average_precision.append(sum(incremental_precision)/len(incremental_precision))\n",
    "        except:\n",
    "            average_precision.append(0)\n",
    "        \n",
    "    return (sum(average_precision)/len(average_precision))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.23816698096268418\n"
     ]
    }
   ],
   "source": [
    "#Printing the map score for direct translations\n",
    "print(calc_map(100,de_eng_direct))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2547451202335909\n"
     ]
    }
   ],
   "source": [
    "#Printing the map score for noisy channel translations\n",
    "print(calc_map(100,de_eng_noisy_translate))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}