7-RL_and_GAN/7일차/2019-10-07_RL_Q_Learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19950631168807583848837421626835850838234968318861924548520089498529438830221946631919961684036194597899331129423209124271556491349413781117593785932096323957855730046793794526765246551266059895520550086918193311542508608460618104685509074866089624888090489894838009253941633257850621568309473902556912388065225096643874441046759871626985453222868538161694315775629640762836880760732228535091641476183956381458969463899410840960536267821064621427333394036525565649530603142680234969400335934316651459297773279665775606172582031407994198179607378245683762280037302885487251900834464581454650557929601414833921615734588139257095379769119277800826957735674444123062018757836325502728323789270710373802866393031428133241401624195671690574061419654342324638801248856147305207431992259611796250130992860241708340807605932320161268492288496255841312844061536738951487114256315111089745514203313820202931640957596464756010405845841566072044962867016515061920631004186422275908670900574606417856951911456055068251250406007519842261898059237118054444788072906395242548339221982707404473162376760846613033778706039803413197133493654622700563169937455508241780972810983291314403571877524768509857276937926433221599399876886660808368837838027643282775172273657572744784112294389733810861607423253291974813120197604178281965697475898164531258434135959862784130128185406283476649088690521047580882615823961985770122407044330583075869039319604603404973156583208672105913300903752823415539745394397715257455290510212310947321610753474825740775273986348298498340756937955646638621874569499279016572103701364433135817214311791398222983845847334440270964182851005072927748364550578634501100852987812389473928699540834346158807043959118985815145779177143619698728131459483783202081474982171858011389071228250905826817436220577475921417653715687725614904582904992461028630081535583308130101987675856234343538955409175623400844887526162643568648833519463720377293240094456246923254350400678027273837755376406726898636241037491410966718557050759098100246789880178271925953381282421954028302759408448955014676668389697996886241636313376393903373455801407636741877711055384225739499110186468219696581651485130494222369947714763069155468217682876200362777257723781365331611196811280792669481887201298643660768551639860534602297871557517947385246369446923087894265948217008051120322365496288169035739121368338393591756418733850510970271613915439590991598154654417336311656936031122249937969999226781732358023111862644575299135758175008199839236284615249881088960232244362173771618086357015468484058622329792853875623486556440536962622018963571028812361567512543338303270029097668650568557157505516727518899194129711337690149916181315171544007728650573189557450920330185304847113818315407324053319038462084036421763703911550639789000742853672196280903477974533320468368795868580237952218629120080742819551317948157624448298518461509704888027274721574688131594750409732115080498190455803416826949787141316063210686391511681774304792596709376"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pow(2,100*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "env=gym.make('FrozenLake-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q=np.zeros([env.observation_space.n, env.action_space.n])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate=.85\n",
    "dis=.99\n",
    "num_episodes=2000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score : 0.0175\n",
      "[[0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.99 0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.  ]\n",
      " [0.   0.   1.   0.  ]\n",
      " [0.   0.   0.   0.  ]]\n"
     ]
    }
   ],
   "source": [
    "rList=[]\n",
    "for i in range(num_episodes):\n",
    "    state=env.reset()\n",
    "    done=False\n",
    "    rAll=0\n",
    "    while not done:\n",
    "        action=np.argmax(Q[state,:]+np.random.randn(1, env.action_space.n)/(i+1))  #상/하/좌/우\n",
    "        new_state, reward, done, _ =env.step(action)\n",
    "        #q_table 갱신\n",
    "        Q[state, action]=reward+dis*np.max(Q[new_state, :])\n",
    "        state=new_state\n",
    "        rAll+=reward\n",
    "    rList.append(rAll)\n",
    "print(\"score : \"+str(sum(rList)/num_episodes))\n",
    "print(Q)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#q-learning + nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "env=gym.make('FrozenLake-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_size=env.observation_space.n\n",
    "output_size=env.action_space.n\n",
    "learning_rate=0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=tf.placeholder(shape=[1,input_size],dtype=tf.float32) #(1,16)\n",
    "W=tf.Variable(tf.random_uniform([input_size,output_size],0,0.01))\n",
    "\n",
    "Qpred=tf.matmul(X,W) #(1,4)\n",
    "Y=tf.placeholder(shape=[1,output_size], dtype=tf.float32)\n",
    "\n",
    "loss=tf.reduce_sum(tf.square(Y-Qpred))\n",
    "train=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "dis=.99\n",
    "num_episodes=2000\n",
    "\n",
    "rList=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot(x):\n",
    "    return np.identity(16)[x:x+1]\n",
    "#one_hot(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4725%\n"
     ]
    }
   ],
   "source": [
    "init=tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for i in range(num_episodes):\n",
    "        s=env.reset() # s=0\n",
    "        e=1. / ((i/50)+10) #e-greedy 0.1 -> 0.102 -> .... 0근사\n",
    "        rAll=0\n",
    "        done=False\n",
    "    \n",
    "        while not done:\n",
    "            Qs=sess.run(Qpred, feed_dict={X:one_hot(s)})\n",
    "            #exploit & exploration\n",
    "            if np.random.rand(1) < e : #10%확률로 탐험\n",
    "                a=env.action_space.sample()\n",
    "            else :\n",
    "                a=np.argmax(Qs)\n",
    "\n",
    "            s1, reward, done, _ =env.step(a)    \n",
    "\n",
    "            if done: #terminal state임\n",
    "                Qs[0,a]=reward            \n",
    "            else:#terminal state가 아님\n",
    "                Qs1=sess.run(Qpred,feed_dict={X:one_hot(s1)})\n",
    "                Qs[0,a]=reward+dis*np.max(Qs1)\n",
    "            sess.run(train, feed_dict={X:one_hot(s), Y:Qs})\n",
    "\n",
    "            rAll+=reward\n",
    "            s=s1\n",
    "        rList.append(rAll)\n",
    "print(str(sum(rList)/num_episodes)+\"%\")\n",
    "            \n",
    "            \n",
    "            \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.identity(16)\n",
    "#np.identity(4)[0:0+1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}