ceviriler_data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "af91e3a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\VuralBayrakli\\anaconda3\\envs\\env\\lib\\site-packages\\thinc\\compat.py:36: UserWarning: 'has_mps' is deprecated, please use 'torch.backends.mps.is_built()'\n",
      "  hasattr(torch, \"has_mps\")\n",
      "C:\\Users\\VuralBayrakli\\anaconda3\\envs\\env\\lib\\site-packages\\thinc\\compat.py:37: UserWarning: 'has_mps' is deprecated, please use 'torch.backends.mps.is_built()'\n",
      "  and torch.has_mps  # type: ignore[attr-defined]\n"
     ]
    }
   ],
   "source": [
    "# Gerekli kütüphaneleri içe aktar\n",
    "from io import open\n",
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "import os\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torch.utils.data import DataLoader\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "from torchtext.vocab import Vocab, build_vocab_from_iterator\n",
    "from collections import Counter \n",
    "from tqdm.notebook import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import ticker\n",
    "import pickle\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d7c821e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# gRPC üzerinden Zemberek dil işleme servislerini kullanmak için gerekli modülleri ve paketleri içe aktar\n",
    "import sys\n",
    "import grpc\n",
    "import zemberek_grpc.language_id_pb2 as z_langid\n",
    "import zemberek_grpc.language_id_pb2_grpc as z_langid_g\n",
    "import zemberek_grpc.normalization_pb2 as z_normalization\n",
    "import zemberek_grpc.normalization_pb2_grpc as z_normalization_g\n",
    "import zemberek_grpc.preprocess_pb2 as z_preprocess\n",
    "import zemberek_grpc.preprocess_pb2_grpc as z_preprocess_g\n",
    "import zemberek_grpc.morphology_pb2 as z_morphology\n",
    "import zemberek_grpc.morphology_pb2_grpc as z_morphology_g\n",
    "\n",
    "# gRPC kanalını belirtilen adres ve port üzerinden oluştur\n",
    "channel = grpc.insecure_channel('localhost:6789')\n",
    "\n",
    "# Dil tespiti için servis istemcisini oluştur\n",
    "langid_stub = z_langid_g.LanguageIdServiceStub(channel)\n",
    "\n",
    "# Normalizasyon için servis istemcisini oluştur\n",
    "normalization_stub = z_normalization_g.NormalizationServiceStub(channel)\n",
    "\n",
    "# Metin ön işleme için servis istemcisini oluştur\n",
    "preprocess_stub = z_preprocess_g.PreprocessingServiceStub(channel)\n",
    "\n",
    "# Morfoloji analizi için servis istemcisini oluştur\n",
    "morphology_stub = z_morphology_g.MorphologyServiceStub(channel)\n",
    "\n",
    "# Dil tespiti fonksiyonu\n",
    "def find_lang_id(i):\n",
    "    response = langid_stub.Detect(z_langid.LanguageIdRequest(input=i))\n",
    "    return response.langId\n",
    "\n",
    "# Metni token'lara ayıran fonksiyon\n",
    "def tokenize(i):\n",
    "    response = preprocess_stub.Tokenize(z_preprocess.TokenizationRequest(input=i))\n",
    "    return response.tokens\n",
    "\n",
    "# Decode işlemini gerçekleştiren fonksiyon\n",
    "def fix_decode(text):\n",
    "    \"\"\"Pass decode.\"\"\"\n",
    "    if sys.version_info < (3, 0):\n",
    "        return text.decode('utf-8')\n",
    "    else:\n",
    "        return text\n",
    "    \n",
    "# Morfoloji analizi fonksiyonu\n",
    "def analyze(i):\n",
    "    response = morphology_stub.AnalyzeSentence(z_morphology.SentenceAnalysisRequest(input=i))\n",
    "    return response;\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "b3e32a84",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Metni normalize etmek için bir fonksiyon\n",
    "def normalizeString(s):\n",
    "    # Metni düşük harfe dönüştür, Türkçe karakterleri ve belirli noktalama işaretlerini koru\n",
    "    s = s.lower().strip()\n",
    "    # Sadece harfler, nokta (.), soru işareti (?), ünlem işareti (!) ve Türkçe karakterleri koru\n",
    "    s = re.sub(r\"[^a-zçğıöşü.!?,'']+\", \" \", s)\n",
    "    return s\n",
    "\n",
    "# Belirli kriterlere göre çiftleri filtrelemek için bir fonksiyon\n",
    "def filterPair(p, max_length, prefixes):\n",
    "    # Her iki cümle de belirtilen maksimum uzunluktan daha kısa mı kontrol et\n",
    "    good_length = (len(p[0].split(' ')) < max_length) and (len(p[1].split(' ')) < max_length)\n",
    "    # Eğer önekler belirtilmişse, cümlenin önek ile başlayıp başlamadığını kontrol et\n",
    "    if len(prefixes) == 0:\n",
    "        return good_length\n",
    "    else:\n",
    "        return good_length and p[0].startswith(prefixes)\n",
    "\n",
    "# Belirli kriterlere göre çiftleri filtrelemek için bir fonksiyon\n",
    "def filterPairs(pairs, max_length, prefixes=()):\n",
    "    return [pair for pair in pairs if filterPair(pair, max_length, prefixes)]\n",
    "\n",
    "# Veriyi hazırlamak için bir fonksiyon\n",
    "def prepareData(lines, filter=False, reverse=False, max_length=10, prefixes=()):\n",
    "    # Her bir satırı normalize et ve çiftlere ayır\n",
    "    pairs = [(normalizeString(pair[0]), normalizeString(pair[1])) for pair in ceviriler]\n",
    "\n",
    "    print(f\"Given {len(pairs):,} sentence pairs.\")\n",
    "\n",
    "    # Eğer filtreleme etkinse, çiftleri belirtilen kriterlere göre filtrele\n",
    "    if filter:\n",
    "        pairs = filterPairs(pairs, max_length=max_length, prefixes=prefixes)\n",
    "        print(f\"After filtering, {len(pairs):,} remain.\")\n",
    "\n",
    "    return pairs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "ff424670",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"mary'yi\""
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalizeString(\"Mary'yi\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "b77d6a4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"mary'yi\"]"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenize_tr(normalizeString(\"mary'yi\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "fb37e24e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ingilizce']"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "f52acb52",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"ceviriler/ceviriler.pkl\", \"rb\") as f:\n",
    "    ceviriler = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d91783e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "basic_prefixes = (\n",
    "    \"i am \", \"i m \",\n",
    "    \"he is\", \"he s \",\n",
    "    \"she is\", \"she s \",\n",
    "    \"you are\", \"you re \",\n",
    "    \"we are\", \"we re \",\n",
    "    \"they are\", \"they re \",\n",
    "    'are you', 'am i ', \n",
    "    'were you', 'was i ', \n",
    "    'where are', 'where is',\n",
    "    'what is', 'what are'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "a79beffe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Yasama organı, yasaları yapma sorumluluğuna sahiptir.'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ceviriler[5][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "d9e5d2ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "pairss = [(normalizeString(pair[0]), normalizeString(pair[1])) for pair in ceviriler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "8fefbf72",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"ceviriler/pairss.pkl\", \"wb\") as f2:\n",
    "    pickle.dump(pairss, f2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "2e252811",
   "metadata": {},
   "outputs": [],
   "source": [
    "# İngilizce dil modelini yükleme\n",
    "en_nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "# İngilizce cümleleri token'lara ayıran fonksiyon\n",
    "def tokenize_en(text):\n",
    "    return [tok.text for tok in en_nlp.tokenizer(text)]\n",
    "\n",
    "# Türkçe cümleleri token'lara ayıran fonksiyon\n",
    "def tokenize_tr(sentence):\n",
    "    liste = []\n",
    "    # Zemberek dil işleme servisini kullanarak Türkçe cümleyi analiz etme\n",
    "    analysis_result = analyze(sentence)\n",
    "    for a in analysis_result.results:\n",
    "        best = a.best\n",
    "        lemmas = \"\"\n",
    "        liste.append(a.token)\n",
    "    \n",
    "    return liste"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "52eab5b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English:  the camera detected motion and triggered the security system to alert the homeowners.\n",
      "['the', 'camera', 'detected', 'motion', 'and', 'triggered', 'the', 'security', 'system', 'to', 'alert', 'the', 'homeowners', '.']\n",
      "Turkish:  kamera, hareketi algıladı ve güvenlik sistemini ev sahiplerini uyaracak şekilde tetikledi.\n",
      "['kamera', ',', 'hareketi', 'algıladı', 've', 'güvenlik', 'sistemini', 'ev', 'sahiplerini', 'uyaracak', 'şekilde', 'tetikledi', '.']\n",
      "\n",
      "English:  the broadcaster delivered the news to a wide audience through the television network.\n",
      "['the', 'broadcaster', 'delivered', 'the', 'news', 'to', 'a', 'wide', 'audience', 'through', 'the', 'television', 'network', '.']\n",
      "Turkish:  yayıncı, televizyon ağı aracılığıyla geniş bir izleyici kitlesine haberleri iletti.\n",
      "['yayıncı', ',', 'televizyon', 'ağı', 'aracılığıyla', 'geniş', 'bir', 'izleyici', 'kitlesine', 'haberleri', 'iletti', '.']\n",
      "\n",
      "English:  i need to make a decision about which university to attend.\n",
      "['i', 'need', 'to', 'make', 'a', 'decision', 'about', 'which', 'university', 'to', 'attend', '.']\n",
      "Turkish:  hangi üniversiteye gideceğim konusunda bir karar vermem gerekiyor.\n",
      "['hangi', 'üniversiteye', 'gideceğim', 'konusunda', 'bir', 'karar', 'vermem', 'gerekiyor', '.']\n",
      "\n",
      "English:  it was merely a suggestion, not a formal proposal.\n",
      "['it', 'was', 'merely', 'a', 'suggestion', ',', 'not', 'a', 'formal', 'proposal', '.']\n",
      "Turkish:  bu sadece bir öneri idi, resmi bir teklif değil.\n",
      "['bu', 'sadece', 'bir', 'öneri', 'idi', ',', 'resmi', 'bir', 'teklif', 'değil', '.']\n",
      "\n",
      "English:  he has ambitious goals for his career.\n",
      "['he', 'has', 'ambitious', 'goals', 'for', 'his', 'career', '.']\n",
      "Turkish:  kariyeri için iddialı hedefleri var.\n",
      "['kariyeri', 'için', 'iddialı', 'hedefleri', 'var', '.']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# İngilizce ve Türkçe kelimeleri saymak için sayaçlar oluştur\n",
    "en_counter = Counter()\n",
    "tr_counter = Counter()\n",
    "\n",
    "# Rastgele seçilmiş 5 çift üzerinde işlem yap\n",
    "for eng, tur in random.choices(pairss, k=5):\n",
    "    # İngilizce cümleyi ekrana yazdır\n",
    "    print(f\"English:  {eng}\")\n",
    "    # İngilizce cümleyi token'lara ayır ve ekrana yazdır\n",
    "    print(tokenize_en(eng))\n",
    "    # Türkçe cümleyi ekrana yazdır\n",
    "    print(f\"Turkish:  {tur}\")\n",
    "    # Türkçe cümleyi token'lara ayır ve ekrana yazdır\n",
    "    aa = tokenize_tr(tur)\n",
    "    print(aa)\n",
    "    print()\n",
    "\n",
    "    # İngilizce kelimeleri say\n",
    "    en_counter.update(tokenize_en(eng))\n",
    "    # Türkçe kelimeleri say\n",
    "    tr_counter.update(aa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "58ac0078",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n"
     ]
    }
   ],
   "source": [
    "# Özel token'ları tanımla\n",
    "SPECIALS = ['<unk>', '<pad>', '<bos>', '<eos>']\n",
    "\n",
    "# İngilizce ve Türkçe cümle listeleri\n",
    "en_list = []\n",
    "tr_list = []\n",
    "\n",
    "# İngilizce ve Türkçe kelime sayaçları\n",
    "en_counter = Counter()\n",
    "tr_counter = Counter()\n",
    "\n",
    "# İngilizce ve Türkçe cümle uzunlukları\n",
    "en_lengths = []\n",
    "tr_lengths = []\n",
    "\n",
    "# Tokenleme işlemi\n",
    "sayac = 0\n",
    "for en, tr in pairss:\n",
    "    # İngilizce ve Türkçe cümleleri token'lara ayır\n",
    "    en_toks = tokenize_en(en)\n",
    "    tr_toks = tokenize_tr(tr)\n",
    "    \n",
    "    # Token'ları ilgili listelere ekle\n",
    "    en_list += [en_toks]\n",
    "    tr_list += [tr_toks]\n",
    "    \n",
    "    # Kelime sayılarını güncelle\n",
    "    en_counter.update(en_toks)\n",
    "    tr_counter.update(tr_toks)\n",
    "    \n",
    "    # Cümle uzunluklarını kaydet\n",
    "    en_lengths.append(len(en_toks))\n",
    "    tr_lengths.append(len(tr_toks))\n",
    "    \n",
    "    sayac += 1\n",
    "    \n",
    "    # Her 1000 çift için ilerlemeyi ekrana yazdır\n",
    "    if sayac % 1000 == 0:\n",
    "        print(sayac)\n",
    "\n",
    "# İngilizce ve Türkçe kelime dağarcıklarını oluştur\n",
    "en_vocab = build_vocab_from_iterator(en_list, specials=SPECIALS)\n",
    "tr_vocab = build_vocab_from_iterator(tr_list, specials=SPECIALS)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "3d6f3385",
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = \"ceviriler\"\n",
    "with open(os.path.join(datadir,'en_lengths.pkl'), 'wb') as f:\n",
    "    pickle.dump(en_lengths, f)\n",
    "    \n",
    "with open(os.path.join(datadir,'tr_lengths.pkl'), 'wb') as f:\n",
    "    pickle.dump(tr_lengths, f)\n",
    "    \n",
    "with open(os.path.join(datadir,'en_counter.pkl'), 'wb') as f:\n",
    "    pickle.dump(en_counter, f)\n",
    "    \n",
    "    \n",
    "with open(os.path.join(datadir,'tr_counter.pkl'), 'wb') as f:\n",
    "    pickle.dump(tr_counter, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "213e1759",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "\n",
      "      Training pairs: 1,568\n",
      "      Validation pairs: 155\n",
      "      Test pairs: 207\n"
     ]
    }
   ],
   "source": [
    "# Veri setini bölme oranları\n",
    "VALID_PCT = 0.1\n",
    "TEST_PCT = 0.1\n",
    "\n",
    "# Boş veri setleri oluştur\n",
    "train_data = []\n",
    "valid_data = []\n",
    "test_data = []\n",
    "\n",
    "# Rastgele tohum belirleme\n",
    "random.seed(6547)\n",
    "\n",
    "# Her bir çifti işleme al\n",
    "sayac = 0\n",
    "for (en, tr) in pairss:\n",
    "    # İngilizce ve Türkçe cümleleri tensor'a çevir\n",
    "    en_tensor_ = torch.tensor([en_vocab[token] for token in tokenize_en(en)])\n",
    "    tr_tensor_ = torch.tensor([tr_vocab[token] for token in tokenize_tr(tr)])\n",
    "    \n",
    "    # Rastgele bir sayı çek ve bölme oranlarına göre veri setlerine ekle\n",
    "    random_draw = random.random()\n",
    "    if random_draw <= VALID_PCT:\n",
    "        valid_data.append((en_tensor_, tr_tensor_))\n",
    "    elif random_draw <= VALID_PCT + TEST_PCT:\n",
    "        test_data.append((en_tensor_, tr_tensor_))\n",
    "    else:\n",
    "        train_data.append((en_tensor_, tr_tensor_))\n",
    "    \n",
    "    sayac += 1\n",
    "    \n",
    "    # Her 1000 çift için ilerlemeyi ekrana yazdır\n",
    "    if sayac % 1000 == 0:\n",
    "        print(sayac)\n",
    "\n",
    "# Bölünmüş veri seti boyutlarını ekrana yazdır\n",
    "print(f\"\"\"\n",
    "      Training pairs: {len(train_data):,}\n",
    "      Validation pairs: {len(valid_data):,}\n",
    "      Test pairs: {len(test_data):,}\"\"\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "1c49693e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Özel token indekslerini belirle\n",
    "PAD_IDX = en_vocab['<pad>']\n",
    "BOS_IDX = en_vocab['<bos>']\n",
    "EOS_IDX = en_vocab['<eos>']\n",
    "\n",
    "# İki dilin özel tokenlerinin indekslerini karşılaştır ve eşit olup olmadığını kontrol et\n",
    "for en_id, tr_id in zip(en_vocab.lookup_indices(SPECIALS), tr_vocab.lookup_indices(SPECIALS)):\n",
    "    assert en_id == tr_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "5ae1e237",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_batch(data_batch):\n",
    "    '''\n",
    "    Veri yığınlarını modelleme için hazırlar. Her bir örneğe BOS/EOS belirteçlerini ekler, tensörleri birleştirir\n",
    "    ve daha kısa cümlelerin sonundaki boşlukları <pad> belirteci ile doldurur. \n",
    "    English-to-Turkish DataLoader'ında collate_fn olarak kullanılması amaçlanmıştır.\n",
    "\n",
    "    Input:\n",
    "    - data_batch, yukarıda oluşturulan veri setlerinden alınan (İngilizce, Türkçe) tuple'larını içeren bir iterasyon\n",
    "\n",
    "    Output:\n",
    "    - en_batch: İngilizce token ID'leri içeren (maksimum uzunluk X yığın boyutu) bir tensör\n",
    "    - tr_batch: Türkçe token ID'leri içeren (maksimum uzunluk X yığın boyutu) bir tensör \n",
    "    '''\n",
    "    \n",
    "    en_batch, tr_batch = [], []\n",
    "    \n",
    "    for (en_item, tr_item) in data_batch:\n",
    "        en_batch.append(torch.cat([torch.tensor([BOS_IDX]), en_item, torch.tensor([EOS_IDX])], dim=0))\n",
    "        tr_batch.append(torch.cat([torch.tensor([BOS_IDX]), tr_item, torch.tensor([EOS_IDX])], dim=0))\n",
    "\n",
    "    en_batch = pad_sequence(en_batch, padding_value=PAD_IDX, batch_first=False)\n",
    "    tr_batch = pad_sequence(tr_batch, padding_value=PAD_IDX, batch_first=False)\n",
    "\n",
    "    return en_batch, tr_batch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "61ea8804",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mini-batch boyutunu belirle\n",
    "BATCH_SIZE = 16\n",
    "\n",
    "# DataLoader ile eğitim, doğrulama ve test veri iteratörlerini oluştur\n",
    "train_iter = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True, collate_fn=generate_batch)\n",
    "valid_iter = DataLoader(valid_data, batch_size=BATCH_SIZE, shuffle=False, collate_fn=generate_batch)\n",
    "test_iter = DataLoader(test_data, batch_size=BATCH_SIZE, shuffle=False, collate_fn=generate_batch)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "5de3f70b",
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = \"ceviriler\"\n",
    "\n",
    "# İngilizce kelime dağarcığını pickle formatında kaydetme\n",
    "with open(os.path.join(datadir,'en_vocab.pkl'), 'wb') as f:\n",
    "    pickle.dump(en_vocab, f)\n",
    "\n",
    "# Türkçe kelime dağarcığını pickle formatında kaydetme\n",
    "with open(os.path.join(datadir,'tr_vocab.pkl'), 'wb') as f:\n",
    "    pickle.dump(tr_vocab, f)\n",
    "\n",
    "# Eğitim veri setini pickle formatında kaydetme\n",
    "with open(os.path.join(datadir,'train_data.pkl'), 'wb') as f:\n",
    "    pickle.dump(train_data, f)\n",
    "    \n",
    "# Doğrulama veri setini pickle formatında kaydetme\n",
    "with open(os.path.join(datadir,'valid_data.pkl'), 'wb') as f:\n",
    "    pickle.dump(valid_data, f)\n",
    "\n",
    "# Test veri setini pickle formatında kaydetme\n",
    "with open(os.path.join(datadir,'test_data.pkl'), 'wb') as f:\n",
    "    pickle.dump(test_data, f)\n",
    "\n",
    "# Eğitim veri seti yükleyicisini pickle formatında kaydetme\n",
    "with open(os.path.join(datadir,'train_iter.pkl'), 'wb') as f:\n",
    "    pickle.dump(train_iter, f)\n",
    "\n",
    "# Test veri seti yükleyicisini pickle formatında kaydetme\n",
    "with open(os.path.join(datadir, 'test_iter.pkl'), 'wb') as f:\n",
    "    pickle.dump(test_iter, f)\n",
    "\n",
    "# Doğrulama veri seti yükleyicisini pickle formatında kaydetme\n",
    "with open(os.path.join(datadir, 'valid_iter.pkl'), 'wb') as f:\n",
    "    pickle.dump(valid_iter, f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d31c0822",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "719c8c02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English: <bos> please provide your input on the proposed changes to the project . <eos> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "Turkish: <bos> lütfen projedeki önerilen değişikliklerle ilgili görüşlerinizi belirtin . <eos> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\n",
      "English: <bos>   the cat is sleeping peacefully on the sofa . <eos> <pad> <pad> <pad> <pad>\n",
      "Turkish: <bos> kedi , koltukta huzurlu bir şekilde uyuyor . <eos> <pad> <pad> <pad> <pad> <pad> <pad>\n",
      "\n",
      "English: <bos> the national flag is symbolic of the country 's identity and values . <eos> <pad> <pad>\n",
      "Turkish: <bos> ulusal bayrak , ülkenin kimliği ve değerleri için semboliktir . <eos> <pad> <pad> <pad>\n",
      "\n",
      "English: <bos> the councilor played a key role in making decisions for the local community . <eos> <pad> <pad> <pad> <pad>\n",
      "Turkish: <bos> meclis üyesi , yerel toplum için kararlar almakta önemli bir rol oynadı . <eos> <pad> <pad>\n",
      "\n",
      "English: <bos> the researchers were able to derive meaningful conclusions from the data . <eos> <pad> <pad> <pad> <pad>\n",
      "Turkish: <bos> araştırmacılar , veriden anlamlı sonuçlar çıkarmayı başardı . <eos> <pad> <pad> <pad> <pad>\n"
     ]
    }
   ],
   "source": [
    "# Eğitim veri iteratöründen örnekler al ve ekrana yazdır\n",
    "for i, (en_id, tr_id) in enumerate(train_iter):\n",
    "    print('English:', ' '.join([en_vocab.lookup_token(idx) for idx in en_id[:, 0]]))\n",
    "    print('Turkish:', ' '.join([tr_vocab.lookup_token(idx) for idx in tr_id[:, 0]]))\n",
    "    \n",
    "    # İlk 5 mini-batch'i ekrana yazdıktan sonra döngüyü sonlandır\n",
    "    if i == 4: \n",
    "        break\n",
    "    else:\n",
    "        print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "2f703b00",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "    def __init__(self, d_model, dropout_p=0.1, max_len=100):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout_p)\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-np.log(10000.) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
    "        self.register_buffer('pe', pe)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x + self.pe[:x.size(0), :]\n",
    "        return self.dropout(x)\n",
    "\n",
    "\n",
    "class TransformerModel(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim, d_model, num_attention_heads, \n",
    "                 num_encoder_layers, num_decoder_layers, dim_feedforward, \n",
    "                 max_seq_length, pos_dropout, transformer_dropout):\n",
    "        super().__init__()\n",
    "        self.d_model = d_model\n",
    "        self.embed_src = nn.Embedding(input_dim, d_model)\n",
    "        self.embed_tgt = nn.Embedding(output_dim, d_model)\n",
    "        self.pos_enc = PositionalEncoding(d_model, pos_dropout, max_seq_length)\n",
    "        \n",
    "        self.transformer = nn.Transformer(d_model, num_attention_heads, num_encoder_layers, \n",
    "                                          num_decoder_layers, dim_feedforward, transformer_dropout)\n",
    "        self.output = nn.Linear(d_model, output_dim)\n",
    "        \n",
    "    def forward(self,\n",
    "                src=None, \n",
    "                tgt=None,\n",
    "                src_mask=None,\n",
    "                tgt_mask=None, \n",
    "                src_key_padding_mask=None, \n",
    "                tgt_key_padding_mask=None,\n",
    "                memory_key_padding_mask=None,\n",
    "                src_embeds=None, \n",
    "                tgt_embeds=None):\n",
    "        \n",
    "        if (src_embeds is None) and (src is not None):\n",
    "            if (tgt_embeds is None) and (tgt is not None):\n",
    "                src_embeds, tgt_embeds = self._embed_tokens(src, tgt)\n",
    "        elif (src_embeds is not None) and (src is not None):\n",
    "            raise ValueError(\"Must specify exactly one of src and src_embeds\")\n",
    "        elif (src_embeds is None) and (src is None):\n",
    "            raise ValueError(\"Must specify exactly one of src and src_embeds\")\n",
    "        elif (tgt_embeds is not None) and (tgt is not None):\n",
    "            raise ValueError(\"Must specify exactly one of tgt and tgt_embeds\")\n",
    "        elif (tgt_embeds is None) and (tgt is None):\n",
    "            raise ValueError(\"Must specify exactly one of tgt and tgt_embeds\")\n",
    "        \n",
    "        output = self.transformer(src_embeds, \n",
    "                                  tgt_embeds, \n",
    "                                  tgt_mask=tgt_mask, \n",
    "                                  src_key_padding_mask=src_key_padding_mask,\n",
    "                                  tgt_key_padding_mask=tgt_key_padding_mask,\n",
    "                                  memory_key_padding_mask=memory_key_padding_mask)\n",
    "        \n",
    "        return self.output(output)\n",
    "    \n",
    "    def _embed_tokens(self, src, tgt):\n",
    "        src_embeds = self.embed_src(src) * np.sqrt(self.d_model)\n",
    "        tgt_embeds = self.embed_tgt(tgt) * np.sqrt(self.d_model)\n",
    "        \n",
    "        src_embeds = self.pos_enc(src_embeds)\n",
    "        tgt_embeds = self.pos_enc(tgt_embeds)\n",
    "        return src_embeds, tgt_embeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "a805ac2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\VuralBayrakli\\anaconda3\\envs\\env\\lib\\site-packages\\torch\\nn\\modules\\transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance)\n",
      "  warnings.warn(f\"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}\")\n"
     ]
    }
   ],
   "source": [
    "transformer = TransformerModel(input_dim=len(en_vocab), \n",
    "                             output_dim=len(tr_vocab), \n",
    "                             d_model=256, \n",
    "                             num_attention_heads=8,\n",
    "                             num_encoder_layers=6, \n",
    "                             num_decoder_layers=6, \n",
    "                             dim_feedforward=2048,\n",
    "                             max_seq_length=32,\n",
    "                             pos_dropout=0.15,\n",
    "                             transformer_dropout=0.3)\n",
    "\n",
    "transformer = transformer.to(\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "f910f49c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_transformer(text, model, \n",
    "                        src_vocab=en_vocab, \n",
    "                        src_tokenizer=tokenize_en, \n",
    "                        tgt_vocab=tr_vocab, \n",
    "                        device=\"cpu\"):\n",
    "    \n",
    "    input_ids = [src_vocab[token.lower()] for token in src_tokenizer(text)]\n",
    "    input_ids = [BOS_IDX] + input_ids + [EOS_IDX]\n",
    "    \n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        input_tensor = torch.tensor(input_ids).to(device).unsqueeze(1) \n",
    "        \n",
    "        causal_out = torch.ones(MAX_SENTENCE_LENGTH, 1).long().to(device) * BOS_IDX\n",
    "        for t in range(1, MAX_SENTENCE_LENGTH):\n",
    "            decoder_output = transformer(input_tensor, causal_out[:t, :])[-1, :, :]\n",
    "            next_token = decoder_output.data.topk(1)[1].squeeze()\n",
    "            causal_out[t, :] = next_token\n",
    "            if next_token.item() == EOS_IDX:\n",
    "                break\n",
    "                \n",
    "        pred_words = [tgt_vocab.lookup_token(tok.item()) for tok in causal_out.squeeze(1)[1:(t)]]\n",
    "        return \" \".join(pred_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "c3c8aae5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_transformer(model, iterator, optimizer, loss_fn, device, clip=None):\n",
    "    model.train()\n",
    "        \n",
    "    epoch_loss = 0\n",
    "    with tqdm(total=len(iterator), leave=False) as t:\n",
    "        for i, (src, tgt) in enumerate(iterator):\n",
    "            src = src.to(device)\n",
    "            tgt = tgt.to(device)\n",
    "            \n",
    "            # Create tgt_inp and tgt_out (which is tgt_inp but shifted by 1)\n",
    "            tgt_inp, tgt_out = tgt[:-1, :], tgt[1:, :]\n",
    "\n",
    "            tgt_mask = model.transformer.generate_square_subsequent_mask(tgt_inp.size(0)).to(device)\n",
    "            src_key_padding_mask = (src == PAD_IDX).transpose(0, 1)\n",
    "            tgt_key_padding_mask = (tgt_inp == PAD_IDX).transpose(0, 1)\n",
    "            memory_key_padding_mask = src_key_padding_mask.clone()\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            output = model(src=src, tgt=tgt_inp, \n",
    "                           tgt_mask=tgt_mask,\n",
    "                           src_key_padding_mask = src_key_padding_mask,\n",
    "                           tgt_key_padding_mask = tgt_key_padding_mask,\n",
    "                           memory_key_padding_mask = memory_key_padding_mask)\n",
    "            \n",
    "            loss = loss_fn(output.view(-1, output.shape[2]),\n",
    "                           tgt_out.view(-1))\n",
    "            \n",
    "            loss.backward()\n",
    "            \n",
    "            if clip is not None:\n",
    "                nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "            \n",
    "            optimizer.step()\n",
    "            epoch_loss += loss.item()\n",
    "            \n",
    "            avg_loss = epoch_loss / (i+1)\n",
    "            t.set_postfix(loss='{:05.3f}'.format(avg_loss),\n",
    "                          ppl='{:05.3f}'.format(np.exp(avg_loss)))\n",
    "            t.update()\n",
    "            \n",
    "    return epoch_loss / len(iterator)\n",
    "    \n",
    "def evaluate_transformer(model, iterator, loss_fn, device):\n",
    "    model.eval()\n",
    "        \n",
    "    epoch_loss = 0\n",
    "    with torch.no_grad():\n",
    "        with tqdm(total=len(iterator), leave=False) as t:\n",
    "            for i, (src, tgt) in enumerate(iterator):\n",
    "                src = src.to(device)\n",
    "                tgt = tgt.to(device)\n",
    "                \n",
    "                # Create tgt_inp and tgt_out (which is tgt_inp but shifted by 1)\n",
    "                tgt_inp, tgt_out = tgt[:-1, :], tgt[1:, :]\n",
    "                \n",
    "                tgt_mask = model.transformer.generate_square_subsequent_mask(tgt_inp.size(0)).to(device)\n",
    "                src_key_padding_mask = (src == PAD_IDX).transpose(0, 1)\n",
    "                tgt_key_padding_mask = (tgt_inp == PAD_IDX).transpose(0, 1)\n",
    "                memory_key_padding_mask = src_key_padding_mask.clone()\n",
    "\n",
    "                output = model(src=src, tgt=tgt_inp, \n",
    "                               tgt_mask=tgt_mask,\n",
    "                               src_key_padding_mask = src_key_padding_mask,\n",
    "                               tgt_key_padding_mask = tgt_key_padding_mask,\n",
    "                               memory_key_padding_mask = memory_key_padding_mask)\n",
    "                \n",
    "                loss = loss_fn(output.view(-1, output.shape[2]),\n",
    "                               tgt_out.view(-1))\n",
    "                \n",
    "                epoch_loss += loss.item()\n",
    "                \n",
    "                avg_loss = epoch_loss / (i+1)\n",
    "                t.set_postfix(loss='{:05.3f}'.format(avg_loss),\n",
    "                              ppl='{:05.3f}'.format(np.exp(avg_loss)))\n",
    "                t.update()\n",
    "    \n",
    "    return epoch_loss / len(iterator)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "acb15eba",
   "metadata": {},
   "outputs": [],
   "source": [
    "xf_optim = torch.optim.AdamW(transformer.parameters(), lr=1e-4)\n",
    "loss_fn = nn.CrossEntropyLoss(ignore_index=PAD_IDX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31819b3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "300c555583c249348f7403837703854d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch:   0%|          | 0/50 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e5a6f1a32594c40af0f8e5dd62061c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/98 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\VuralBayrakli\\anaconda3\\envs\\env\\lib\\site-packages\\torch\\nn\\functional.py:5076: UserWarning: Support for mismatched key_padding_mask and attn_mask is deprecated. Use same type for both instead.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "N_EPOCHS = 50\n",
    "CLIP = 15 # clipping value, or None to prevent gradient clipping\n",
    "EARLY_STOPPING_EPOCHS = 5\n",
    "SAVE_DIR = os.getcwd() \n",
    "model_path = os.path.join(SAVE_DIR, 'transformer_en_tr.pt')\n",
    "transformer_metrics = {}\n",
    "best_valid_loss = float(\"inf\")\n",
    "early_stopping_count = 0\n",
    "for epoch in tqdm(range(N_EPOCHS), desc=\"Epoch\"):\n",
    "    train_loss = train_transformer(transformer, train_iter, xf_optim, loss_fn, device, clip=CLIP)\n",
    "    valid_loss = evaluate_transformer(transformer, valid_iter, loss_fn, device)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        tqdm.write(f\"Checkpointing at epoch {epoch + 1}\")\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(transformer.state_dict(), model_path)\n",
    "        early_stopping_count = 0\n",
    "    elif epoch > EARLY_STOPPING_EPOCHS:\n",
    "        early_stopping_count += 1\n",
    "    \n",
    "    transformer_metrics[epoch+1] = dict(\n",
    "        train_loss = train_loss,\n",
    "        train_ppl = np.exp(train_loss),\n",
    "        valid_loss = valid_loss,\n",
    "        valid_ppl = np.exp(valid_loss)\n",
    "    )\n",
    "    \n",
    "    if early_stopping_count == EARLY_STOPPING_EPOCHS:\n",
    "        tqdm.write(f\"Early stopping triggered in epoch {epoch + 1}\")\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c59a9b2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}