Windy01_DP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ennvironment setup\n",
    "# Variables\n",
    "#   sizeX, sizeY : size of the gridworld\n",
    "# \n",
    "# Functions\n",
    "#   stateNum : input(x,y)= coordinate, output= state number\n",
    "#   stateNumToCoord : input(st)= state number, output[x,y]= coordinate\n",
    "#   keepInside : input(x,y)= coordinate, output[x,y]= coordinate\n",
    "#   moveAgent : input(x,y,a)= coordinate, action, output[x,y]=coordinate\n",
    "#   wind : input(x,y)=coordinate, output[x,y]=coordinate\n",
    "\n",
    "sizeX = 6\n",
    "sizeY = 5\n",
    "def stateNum(x, y):\n",
    "    return(x + (y-1) *sizeX -1)\n",
    "\n",
    "dicAction = {'up':1, 'left':2, 'down':3, 'right':4}\n",
    "\n",
    "def stateNumToCoord(st):\n",
    "    return([st % sizeX +1, 1+st // sizeX])  # //, %  division and the remainder\n",
    "\n",
    "def keepInside(x, y):    \n",
    "    if (x<1): \n",
    "        x=1\n",
    "    elif (x>sizeX):\n",
    "        x=sizeX\n",
    "        \n",
    "    if (y<1):\n",
    "        y=1\n",
    "    elif (y>sizeY):\n",
    "        y=sizeY\n",
    "        \n",
    "    return([x,y])\n",
    "\n",
    "goalState = stateNum(4,3)\n",
    "\n",
    "\n",
    "def moveAgent(x, y, a):\n",
    "   # if (stateNum(x, y) == goalState):\n",
    "   #     return([x,y])\n",
    "    if a in dicAction:\n",
    "        a = dicAction[a]\n",
    "    \n",
    "    if (a == 1):\n",
    "        y = y - 1\n",
    "    if (a == 2):\n",
    "        x = x - 1\n",
    "    if (a == 3):\n",
    "        y = y + 1\n",
    "    if (a == 4):\n",
    "        x = x + 1\n",
    "    \n",
    "    return(keepInside(x,y))\n",
    "\n",
    "def wind(x,y):    \n",
    "    if (x in [2,5]):\n",
    "        y=y-1\n",
    "    if (x in [3,4]):\n",
    "        y=y-2\n",
    "    return(keepInside(x,y))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 1 0 [1, 1]\n",
      "2 1 1 [2, 1]\n",
      "3 1 2 [3, 1]\n",
      "4 1 3 [4, 1]\n",
      "5 1 4 [5, 1]\n",
      "6 1 5 [6, 1]\n",
      "1 2 6 [1, 2]\n",
      "2 2 7 [2, 2]\n",
      "3 2 8 [3, 2]\n",
      "4 2 9 [4, 2]\n",
      "5 2 10 [5, 2]\n",
      "6 2 11 [6, 2]\n",
      "1 3 12 [1, 3]\n",
      "2 3 13 [2, 3]\n",
      "3 3 14 [3, 3]\n",
      "4 3 15 [4, 3]\n",
      "5 3 16 [5, 3]\n",
      "6 3 17 [6, 3]\n",
      "1 4 18 [1, 4]\n",
      "2 4 19 [2, 4]\n",
      "3 4 20 [3, 4]\n",
      "4 4 21 [4, 4]\n",
      "5 4 22 [5, 4]\n",
      "6 4 23 [6, 4]\n",
      "1 5 24 [1, 5]\n",
      "2 5 25 [2, 5]\n",
      "3 5 26 [3, 5]\n",
      "4 5 27 [4, 5]\n",
      "5 5 28 [5, 5]\n",
      "6 5 29 [6, 5]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Check to see if the functions work fine\n",
    "for y in np.arange(1,sizeY+1):    \n",
    "    for x in np.arange(1,sizeX+1):\n",
    "        print(x,y, stateNum(x,y), stateNumToCoord(stateNum(x,y)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[2, 1]\n",
      "[3, 2]\n"
     ]
    }
   ],
   "source": [
    "# Test\n",
    "print(stateNum(1,1))\n",
    "print(moveAgent(3,1,'left'))\n",
    "print(wind(3,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 5\n",
      "4 3\n",
      "4 2\n",
      "[ 9 14 21 16]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "istate = 1\n",
    "\n",
    "# Environment : Transition = wind + action\n",
    "# sNext[sCurrent, action] : next state when the current state is sCurrent and action is taken.\n",
    "sNext = np.zeros((sizeX*sizeY, 4), dtype=np.int)\n",
    "\n",
    "for istate in range(sizeX*sizeY):\n",
    "    for iaction in [1,2,3,4]:\n",
    "        x, y=stateNumToCoord(istate)\n",
    "        x, y=wind(x,y)\n",
    "        x, y=moveAgent(x, y, iaction)        \n",
    "        sNext[istate, iaction-1] = int(stateNum(x,y))\n",
    "sNext[goalState,:]=goalState\n",
    "\n",
    "# Testing\n",
    "istate=27;\n",
    "iaction = 1\n",
    "x, y=stateNumToCoord(istate)\n",
    "print(x,y)\n",
    "x, y=wind(x,y)\n",
    "print(x,y)\n",
    "x, y=moveAgent(x, y, iaction)        \n",
    "print(x,y)\n",
    "sNext[istate, iaction-1] = int(stateNum(x,y))\n",
    "print(sNext[27,:])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4, 3]\n",
      "15\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 9, 14, 21, 16])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# goal state\n",
    "print(stateNumToCoord(goalState))\n",
    "print(goalState)\n",
    "sNext[27,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculating P(s' | s, a) -> Tr[s,a,s']\n",
    "nstate = sizeX*sizeY\n",
    "naction = 4\n",
    "Tr = np.zeros((nstate, naction, nstate), dtype=np.int)\n",
    "\n",
    "for istate in range(nstate):\n",
    "    for iaction in [1,2,3,4]:        \n",
    "        for jstate in range(nstate):\n",
    "            if jstate == sNext[istate, iaction-1]:\n",
    "                Tr[istate,iaction-1, jstate] = 1\n",
    "            else:\n",
    "                Tr[istate,iaction-1, jstate] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# arrR[s, a, s'] = R(s,a,s')\n",
    "arrR = np.zeros((nstate, naction, nstate), dtype=np.float)\n",
    "arrR[:,:,:] = -1\n",
    "arrR[goalState, :, :] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ER [s,a'] = Expected rewards of R(s,a,s') over s'\n",
    "ER = np.zeros((nstate, naction), dtype=np.float)\n",
    "for istate in range(nstate):\n",
    "    for iaction in [1,2,3,4]:\n",
    "        ER[istate, iaction-1] = sum(arrR[istate, iaction-1, :] * Tr[istate, iaction-1, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pi : policy\n",
    "# pi0 : random policy with pi(a|s)=1/4\n",
    "pi0 = np.zeros((nstate, naction), dtype=np.float)\n",
    "pi0[:, :] = 1/4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# matT[(s,a), (s',a')] = P((s',a')|(s,a)) given policy pi0\n",
    "matT = np.zeros((nstate*naction, nstate*naction), dtype=np.float)\n",
    "for istate in range(nstate):\n",
    "    for iaction in range(naction):\n",
    "        for jstate in range(nstate):\n",
    "            for jaction in range(naction):\n",
    "                matT[istate*naction +iaction, jstate*naction+jaction] = Tr[istate, iaction, jstate] * pi0[jstate, jaction]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.25"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test : given (1,1), 1:'up', what's the probablity of (1,1),2:'left'\n",
    "matT[stateNum(1,1)*naction + 0, stateNum(1,1)*naction + 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "I=np.diag(np.ones(nstate*naction))  # identity matrix I"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 17.08340901  16.08340901  16.08340901 ...,   0.5196498    0.5196498\n",
      "    0.5196498 ]\n",
      " [ 16.08340901  17.08340901  16.08340901 ...,   0.5196498    0.5196498\n",
      "    0.5196498 ]\n",
      " [ 15.67715001  15.67715001  16.67715001 ...,   0.5196498    0.5196498\n",
      "    0.5196498 ]\n",
      " ..., \n",
      " [ 12.17396425  12.17396425  12.17396425 ...,   1.54749855   0.54749855\n",
      "    0.54749855]\n",
      " [ 11.71464056  11.71464056  11.71464056 ...,   1.12179734   2.12179734\n",
      "    1.12179734]\n",
      " [ 11.71464056  11.71464056  11.71464056 ...,   1.12179734   1.12179734\n",
      "    2.12179734]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ -6.10522607e+02,  -6.10522607e+02,  -6.10522607e+02,\n",
       "        -6.08963138e+02,  -6.08963138e+02,  -6.10522607e+02,\n",
       "        -6.08963138e+02,  -6.05837956e+02,  -6.05837956e+02,\n",
       "        -6.08963138e+02,  -6.05837956e+02,  -6.01134547e+02,\n",
       "        -6.01134547e+02,  -6.05837956e+02,  -6.01134547e+02,\n",
       "        -5.94834080e+02,  -5.94834080e+02,  -6.01134547e+02,\n",
       "        -5.94834080e+02,  -5.86911326e+02,  -5.86911326e+02,\n",
       "        -5.94834080e+02,  -5.77334564e+02,  -5.86911326e+02,\n",
       "        -6.10522607e+02,  -6.10522607e+02,  -6.10522607e+02,\n",
       "        -6.08963138e+02,  -6.08963138e+02,  -6.10522607e+02,\n",
       "        -6.08963138e+02,  -6.05837956e+02,  -6.05837956e+02,\n",
       "        -6.08963138e+02,  -6.05837956e+02,  -6.01134547e+02,\n",
       "        -6.01134547e+02,  -6.05837956e+02,  -6.01134547e+02,\n",
       "        -5.94834080e+02,  -5.94834080e+02,  -6.01134547e+02,\n",
       "        -5.94834080e+02,  -5.86911326e+02,  -5.86911326e+02,\n",
       "        -5.94834080e+02,  -5.48565932e+02,  -5.77334564e+02,\n",
       "        -6.10522607e+02,  -6.10522607e+02,  -6.10522607e+02,\n",
       "        -6.08963138e+02,  -6.08963138e+02,  -6.10522607e+02,\n",
       "        -6.08963138e+02,  -6.05837956e+02,  -6.05837956e+02,\n",
       "        -6.08963138e+02,  -6.05837956e+02,  -6.01134547e+02,\n",
       "        -1.98558475e-14,  -1.98558475e-14,  -1.98558475e-14,\n",
       "        -1.99353504e-14,  -5.94834080e+02,  -6.01134547e+02,\n",
       "        -5.91646081e+02,  -5.77334564e+02,  -5.77334564e+02,\n",
       "        -5.91646081e+02,  -4.74909608e+02,  -5.48565932e+02,\n",
       "        -6.10522607e+02,  -6.10522607e+02,  -6.10522607e+02,\n",
       "        -6.08963138e+02,  -6.08963138e+02,  -6.10522607e+02,\n",
       "        -6.08963138e+02,  -6.05837956e+02,  -6.05837956e+02,\n",
       "        -6.08963138e+02,  -6.05837956e+02,  -6.01134547e+02,\n",
       "        -6.01134547e+02,  -6.05837956e+02,  -1.00000000e+00,\n",
       "        -5.94834080e+02,  -5.94834080e+02,  -1.00000000e+00,\n",
       "        -3.82291107e+02,  -5.48565932e+02,  -5.48565932e+02,\n",
       "        -3.82291107e+02,  -4.91769322e+02,  -4.74909608e+02,\n",
       "        -6.10522607e+02,  -6.10522607e+02,  -6.10522607e+02,\n",
       "        -6.08963138e+02,  -6.08963138e+02,  -6.10522607e+02,\n",
       "        -6.08963138e+02,  -6.05837956e+02,  -6.05837956e+02,\n",
       "        -6.08963138e+02,  -6.05837956e+02,  -1.00000000e+00,\n",
       "        -6.01134547e+02,  -6.05837956e+02,  -4.51250944e+02,\n",
       "        -5.91646081e+02,  -5.91646081e+02,  -4.51250944e+02,\n",
       "        -5.06594077e+02,  -4.74909608e+02,  -4.74909608e+02,\n",
       "        -5.06594077e+02,  -4.91769322e+02,  -4.91769322e+02])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gamma=0.999\n",
    "inv=np.linalg.inv(I- gamma*matT)   # can't solve the equation with gamma=1\n",
    "print(inv)\n",
    "vER = np.ndarray.flatten(ER)\n",
    "np.dot(inv, vER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-743.3071585 ]\n",
      " [-743.3071585 ]\n",
      " [-743.3071585 ]\n",
      " [-741.39205735]\n",
      " [-741.39205735]\n",
      " [-743.3071585 ]\n",
      " [-741.39205735]\n",
      " [-737.55624645]\n",
      " [-737.55624645]\n",
      " [-741.39205735]\n",
      " [-737.55624645]\n",
      " [-731.78852371]\n",
      " [-731.78852371]\n",
      " [-737.55624645]\n",
      " [-731.78852371]\n",
      " [-724.07212369]\n",
      " [-724.07212369]\n",
      " [-731.78852371]\n",
      " [-724.07212369]\n",
      " [-714.38476266]\n",
      " [-714.38476266]\n",
      " [-724.07212369]\n",
      " [-702.69869856]\n",
      " [-714.38476266]\n",
      " [-743.3071585 ]\n",
      " [-743.3071585 ]\n",
      " [-743.3071585 ]\n",
      " [-741.39205735]\n",
      " [-741.39205735]\n",
      " [-743.3071585 ]\n",
      " [-741.39205735]\n",
      " [-737.55624645]\n",
      " [-737.55624645]\n",
      " [-741.39205735]\n",
      " [-737.55624645]\n",
      " [-731.78852371]\n",
      " [-731.78852371]\n",
      " [-737.55624645]\n",
      " [-731.78852371]\n",
      " [-724.07212369]\n",
      " [-724.07212369]\n",
      " [-731.78852371]\n",
      " [-724.07212369]\n",
      " [-714.38476266]\n",
      " [-714.38476266]\n",
      " [-724.07212369]\n",
      " [-667.60738056]\n",
      " [-702.69869856]\n",
      " [-743.3071585 ]\n",
      " [-743.3071585 ]\n",
      " [-743.3071585 ]\n",
      " [-741.39205735]\n",
      " [-741.39205735]\n",
      " [-743.3071585 ]\n",
      " [-741.39205735]\n",
      " [-737.55624645]\n",
      " [-737.55624645]\n",
      " [-741.39205735]\n",
      " [-737.55624645]\n",
      " [-731.78852371]\n",
      " [   0.        ]\n",
      " [   0.        ]\n",
      " [   0.        ]\n",
      " [   0.        ]\n",
      " [-724.07212369]\n",
      " [-731.78852371]\n",
      " [-720.18045294]\n",
      " [-702.69869856]\n",
      " [-702.69869856]\n",
      " [-720.18045294]\n",
      " [-577.81187329]\n",
      " [-667.60738056]\n",
      " [-743.3071585 ]\n",
      " [-743.3071585 ]\n",
      " [-743.3071585 ]\n",
      " [-741.39205735]\n",
      " [-741.39205735]\n",
      " [-743.3071585 ]\n",
      " [-741.39205735]\n",
      " [-737.55624645]\n",
      " [-737.55624645]\n",
      " [-741.39205735]\n",
      " [-737.55624645]\n",
      " [-731.78852371]\n",
      " [-731.78852371]\n",
      " [-737.55624645]\n",
      " [  -1.        ]\n",
      " [-724.07212369]\n",
      " [-724.07212369]\n",
      " [  -1.        ]\n",
      " [-465.12634358]\n",
      " [-667.60738056]\n",
      " [-667.60738056]\n",
      " [-465.12634358]\n",
      " [-598.31735924]\n",
      " [-577.81187329]\n",
      " [-743.3071585 ]\n",
      " [-743.3071585 ]\n",
      " [-743.3071585 ]\n",
      " [-741.39205735]\n",
      " [-741.39205735]\n",
      " [-743.3071585 ]\n",
      " [-741.39205735]\n",
      " [-737.55624645]\n",
      " [-737.55624645]\n",
      " [-741.39205735]\n",
      " [-737.55624645]\n",
      " [  -1.        ]\n",
      " [-731.78852371]\n",
      " [-737.55624645]\n",
      " [-549.21968674]\n",
      " [-720.18045294]\n",
      " [-720.18045294]\n",
      " [-549.21968674]\n",
      " [-616.49553331]\n",
      " [-577.81187329]\n",
      " [-577.81187329]\n",
      " [-616.49553331]\n",
      " [-598.31735924]\n",
      " [-598.31735924]]\n"
     ]
    }
   ],
   "source": [
    "# vectorized Expected reward(s,a) \n",
    "vER = np.ndarray.flatten(ER)\n",
    "vER = vER.reshape((nstate*naction,1))\n",
    "\n",
    "# vectorized Q(s,a)\n",
    "vQ = np.zeros((nstate*naction,1))\n",
    "#print(vQ.shape)\n",
    "for _ in range(1000):\n",
    "    vQ = vER + np.dot(matT, vQ)\n",
    "\n",
    "print(vQ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEIAAAD8CAYAAADDlHLtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACNBJREFUeJztnV+oHGcZh5/fnpzEYA9ik5qUtNpW\ncpEoWE0IxhYaKUgpwapVaS+kF0ItGlDITamgIgheqL1UIgZ7obb1T7VINZYq6I01yaF/OVaTcExj\nQpOqpC2BnJyc14v9bVg2O7Oz//ecvA8MZ3bmm2++fc43883MvvuuIoIEauNuwKSQIkyKMCnCpAiT\nIkyKMCnCpIgGEdHzBNwBvAIcAR6sUD7GMJ2p9F76kDAFHAVuAlYDzwNbJ1DEoSrvp59DYwdwJCKO\nRcQC8ChwVx/1jZV+RGwCXm16fcLLliWr+thWbZZddisr6X7g/j72MxL6EXECuL7p9XXAydZCEbEP\n2AcgaWLv+fs5NA4CmyXdKGk1cA/w5GCaNXp67hERsShpD3CA+giyPyJeHljLukBqd5TWqfrgSaN8\nQjWsQ6ODiMMRsb1THXllaVKESREmRZgUYVKE6efKsieKhrqyYbxseASo1Yr/nxcvXqzUruwRJkWY\nFGFShEkRJkWYkQ+fRUPd0tJS19s0mJqaKlyXw2eXpAiTIkyKMCnCpAgz0uFTUulQV0SnbVatKn4b\nCwsLlfaRPcKkCJMiTIowKcKkCDPy4bNsqCti7dq1pesvXLjQa5Mu0ZcISfPAm8BFYLHKh62TyiB6\nxEcj4vUB1DNW8hxh+hURwB8kHXas1GVIul/SIUmHJvnbQv0eGrdExElJ7wKelvT3iPhzc4HmGKqp\nqamJNdFXj4iIk/57GniCeuzlsqRnEZLeLmmmMQ98DHhpUA0bNf0cGhuAJ/wB7SrgpxHx+7INlpaW\nOHfuXNc7qnor3Q/9RNUdAz4wwLaMlRw+TYowKcKkCJMizMg/BB4Gg4jFzh5hUoRJESZFmBRhUoTJ\nEORGHZVKXQGkCJMiTIowKcKkCJMhyI19VCp1BZAiTIowKcKkCJMiTIowHa8jJO0HdgOnI+L9XnY1\n8BhwAzAPfDYi/lehrmUdi/1j6hnImnkQeCYiNgPP+PWypqMIR8D8t2XxXcAjnn8E+MSA2zVyer3E\n3hARpwAi4pRDh9pyJeShqkRzDFWtVltxMVSvSboWwH9PD65J46FXEU8C93n+PuA3g2nO+KgyfP4M\n2AWsl3QC+DrwbeBxSZ8HjgOfqbKzXmOxN27cWLr+7NmzXdfZSsdWRcS9Batu73vvE0ReWZoUYVKE\nSREmRZgVkcaxA5nGsRtShEkRJkWYFGFShMkQZJM9wqQIkyJMijApwqQIkyJMxmI36qhU6gogRZgU\nYVKESREmRZiOIiTtl3Ra0ktNy74h6d+SnvN0Z5Wdbdu2jaWlpbZT2e/sFW3TmBYXFwungYmgfQwV\nwMMRcbOnpyrvcULpNYZqxdHPOWKPpBd86LxzYC0aE72K+D7wXuBm4BTw3aKCzQm5zpw50+Puhk9P\nIiLitYi4GBFLwA8pyT8VEfsiYntEbL/mmmt6befQ6UlEI5DMfJJlnH+qQa8xVLsk3Uw9V9088IUq\nO5udnWXNmjVt15XdJU5PT5fWW1RnN/QaQ/Wjvvc8YeSVpUkRJkWYFGFShEkRZqRPsWu1WmGO6/Pn\nzxdud9ttt5XWOzc3V7iuapx29giTIkyKMCnCpAiTIkzGYpvsESZFmBRhUoRJESZFmIzFNtkjTIow\nKcKkCJMiTIowVWKorpf0J0lzkl6W9GUvv1rS05L+6b+VgkUktZ162aYx1Wq1wqkqVUouAnsjYgvw\nYeBLkraywnJRVYmhOhURs55/E5gDNrHCclF1dY6QdAPwQeBZWnJRAYW5qJYDlS+xJV0F/BL4SkS8\n0em4btpuWSTkqtQjJE1Tl/CTiPiVF1fKRdUcQzWIBg+LKqOGqEfIzEXE95pWrahcVB0f3kq6FfgL\n8CLQyNn8EPXzxOPAu3EuqogoDUyVFGP4Bk+lh7cjf4pdlJuyLC922TooDyZbWFjIp9jdkCJMijAp\nwqQIkyLMSJ9il6WDLvva0d69e0vrLct3efz48Uptyx5hUoRJESZFmBRhUoQZ6fA5PT1dONSVDZEH\nDx4srXfLli2F63L47JIUYVKESREmRZgUYTIE2WSPMCnCpAiTIkyKMCnCpAjTTzBZT0m5hkFZoFlV\nqjyPaASTzUqaAQ5LetrrHo6I7/TQ9omjSmqVU9RTLBERb0pqBJOtKPoJJoMKSbma81D11dJhU5Ys\nr3kCrgIOA5/y6w3AFHWZ3wL2V6gjhjE5EqftBByq8v56DibrJinXcqDnYLKVlpSryqhxC/A54EVJ\nz3nZQ8C9vSTlGkYwWVmMVeV40FE/jxiDiHwe0Q0pwqQIkyJMijAj/yZwUdx02Zn/7rvvLq2zKLdV\nN2SPMCnCpAiTIkyKMCnCjPSmq1arRdG3bXbv3l243fz8fGm969atK1x34MCBvOnqhhRhUoRJESZF\nmBRhUoQZ6W34zMwMO3fubLuu7Fqh0232uXPn+mkWkD3iEinCpAiTIkyKMCnCjPqzzzPAv5oWrQde\nH0DVZfW8JyI6/kDYSEVctnPp0CCS8Ayinjw0TIow4xaxb1LqGes5YpIYd4+YGIYuQtIdkl6RdETS\nZakeJa2R9JjXP+tYztYybcOgW8rsknS2KST6a101tGqcZS8T9TjMo8BNwGrgeWBrS5kvAj/w/D3A\nY23quRb4kOdngH+0qWcX8Nte2zrsHrEDOBIRxyJiAXiUeh7MZprzYv4CuF0t0WMlOTUHxrBFbAJe\nbXp9gsvfwKUyEbEInAUKP7FpEwbdzE5Jz0v6naT3ddPQYT+hahcX2DpMVSlTL9iSU7Nl9Sz1y+m3\n/JWJXwObqzZ02D3iBHB90+vrgJNFZSStAt5Bm5/nLsipeYmIeCMi3vL8U8C0pPVVGzpsEQeBzZJu\nlLSa+snwyZYyzXkxPw38MVoubkpyajaX2dg4t0jaQf29/adyS4c5avj93En9LH8U+KqXfRP4uOff\nBvwcOAL8DbipTR23Uj9cXgCe83Qn8ADwgMvsAV6mPjL9FfhIN+3MK0uTV5YmRZgUYVKESREmRZgU\nYVKE+T+y8IdsLbeDeQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2cf883790f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAD8CAYAAADe49kaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACe5JREFUeJzt3c2LXYUdxvHncUwwvgQXDSVkhsaF\nCOJCS8gm0EWwJb6gdSEo6EqYhRUiLYgu/QfETTeDSlsUg6AFsbY2oEEDviUxWpPREsSXQWEUCRoC\nlZini7mFkM7MPdPeM+d3cr4fGJybHC4PMvnOuede7nUSAUAFF3U9AAD+gyABKIMgASiDIAEogyAB\nKIMgASiDIAEogyABKIMgASjj4jbu1DYv/0bv2O56QmN92ipJZ8+e/SbJlnHHtRIktKdPP4gXXdSv\nE/CpqamuJzR28cX9+qd7+vTpz5oc16+fGAAXNIIEoAyCBKAMggSgDIIEoAyCBKAMggSgDIIEoAyC\nBKAMggSgDIIEoAyCBKAMggSgDIIEoAyCBKAMggSgjEZBsr3H9se2T9h+uO1RAIZpbJBsT0n6vaSb\nJF0r6W7b17Y9DMDwNDlD2inpRJJPkvwgaZ+k29udBWCImgRpm6Qvzrm9MPozAJioJu8Uvty7yv/X\np4rYnpU0+38vAjBYTYK0IGnmnNvTkr48/6Akc5LmJD4GCcD/pslDtnclXW37KtsbJd0l6cV2ZwEY\norFnSEnO2H5A0iuSpiQ9leRY68sADE6jT5tL8rKkl1veAmDgeKU2gDIIEoAyCBKAMggSgDIIEoAy\nCBKAMggSgDIIEoAyCBKAMggSgDIIEoAyCBKAMggSgDIIEoAyCBKAMggSgDIIEoAyGr1jJIBaNm7c\n2PWENTl9+nSj4zhDAlAGQQJQBkECUAZBAlAGQQJQBkECUAZBAlAGQQJQBkECUAZBAlAGQQJQBkEC\nUAZBAlAGQQJQBkECUAZBAlAGQQJQxtgg2X7K9qLtD9djEIDhanKG9AdJe1reAQDjg5TkdUnfrsMW\nAAPHNSQAZUzsU0dsz0qandT9ARieiQUpyZykOUmynUndL4Dh4CEbgDKaPO3/rKQ3JV1je8H2fe3P\nAjBEYx+yJbl7PYYAAA/ZAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACU\nQZAAlEGQAJRBkACUQZAAlEGQAJQxsffUBs535syZriesyaZNm7qe0NjmzZu7nrAmJ0+ebHQcZ0gA\nyiBIAMogSADKIEgAyiBIAMogSADKIEgAyiBIAMogSADKIEgAyiBIAMogSADKIEgAyiBIAMogSADK\nIEgAyiBIAMoYGyTbM7Zfsz1v+5jtvesxDMDwNHkL2zOSfpfkiO0rJB22vT/J8Za3ARiYsWdISb5K\ncmT0/feS5iVta3sYgOFZ0zUk29sl3SDp7TbGABi2xp86YvtySc9LejDJd8v8/ayk2QluAzAwjYJk\ne4OWYvRMkheWOybJnKS50fGZ2EIAg9HkWTZLelLSfJLH2p8EYKiaXEPaJeleSbttHx193dzyLgAD\nNPYhW5KDkrwOWwAMHK/UBlAGQQJQBkECUAZBAlAGQQJQBkECUAZBAlAGQQJQBkECUAZBAlAGQQJQ\nBkECUAZBAlAGQQJQBkECUAZBAlAGQQJQRuNPHQHW6s477+x6wprcf//9XU9o7Pjxfn1O6+eff97o\nOM6QAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACU\nQZAAlEGQAJRBkACUMTZIti+x/Y7t920fs/3oegwDMDxN3sL2X5J2Jzlle4Okg7b/muStlrcBGJix\nQUoSSadGNzeMvtLmKADD1Ogaku0p20clLUran+TtdmcBGKJGQUryY5LrJU1L2mn7uvOPsT1r+5Dt\nQ5MeCWAY1vQsW5KTkg5I2rPM380l2ZFkx4S2ARiYJs+ybbF95ej7TZJulPRR28MADE+TZ9m2Svqj\n7SktBey5JC+1OwvAEDV5lu0DSTeswxYAA8crtQGUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAA\nlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACU0eQdI1HIHXfc0fWExt54442uJ6zJzMxM\n1xMau/TSS7ue0ArOkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQ\nAJRBkACUQZAAlEGQAJRBkACUQZAAlNE4SLanbL9n+6U2BwEYrrWcIe2VNN/WEABoFCTb05JukfRE\nu3MADFnTM6THJT0k6WyLWwAM3Ngg2b5V0mKSw2OOm7V9yPahia0DMChNzpB2SbrN9qeS9knabfvp\n8w9KMpdkR5IdE94IYCDGBinJI0mmk2yXdJekV5Pc0/oyAIPD65AAlLGmT65NckDSgVaWABg8zpAA\nlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACU\nQZAAlEGQAJThJJO/U/trSZ9N+G5/IumbCd9nm/q0t09bpX7t7dNWqb29P0uyZdxBrQSpDbYP9ekT\nTfq0t09bpX7t7dNWqfu9PGQDUAZBAlBGn4I01/WANerT3j5tlfq1t09bpY739uYaEoALX5/OkABc\n4HoRJNt7bH9s+4Tth7vesxrbT9letP1h11vGsT1j+zXb87aP2d7b9aaV2L7E9ju23x9tfbTrTU3Y\nnrL9nu2Xut6yGtuf2v6H7aO2D3W2o/pDNttTkv4p6ZeSFiS9K+nuJMc7HbYC27+QdErSn5Jc1/We\n1djeKmlrkiO2r5B0WNKvK/6/tW1JlyU5ZXuDpIOS9iZ5q+Npq7L9W0k7JG1OcmvXe1Zi+1NJO5J0\n+pqpPpwh7ZR0IsknSX6QtE/S7R1vWlGS1yV92/WOJpJ8leTI6PvvJc1L2tbtquVlyanRzQ2jr9K/\nTW1PS7pF0hNdb+mLPgRpm6Qvzrm9oKL/aPrM9nZJN0h6u9slKxs9/DkqaVHS/iRlt448LukhSWe7\nHtJAJP3d9mHbs12N6EOQvMyflf7N2De2L5f0vKQHk3zX9Z6VJPkxyfWSpiXttF32IbHtWyUtJjnc\n9ZaGdiX5uaSbJP1mdOlh3fUhSAuSZs65PS3py462XHBG12Oel/RMkhe63tNEkpOSDkja0/GU1eyS\ndNvo2sw+SbttP93tpJUl+XL030VJf9bSpZJ114cgvSvpattX2d4o6S5JL3a86YIwulD8pKT5JI91\nvWc1trfYvnL0/SZJN0r6qNtVK0vySJLpJNu19DP7apJ7Op61LNuXjZ7UkO3LJP1KUifPEpcPUpIz\nkh6Q9IqWLro+l+RYt6tWZvtZSW9Kusb2gu37ut60il2S7tXSb++jo6+bux61gq2SXrP9gZZ+Se1P\nUvqp9B75qaSDtt+X9I6kvyT5WxdDyj/tD2A4yp8hARgOggSgDIIEoAyCBKAMggSgDIIEoAyCBKAM\nggSgjH8DGLkHNdVWm3gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2cf8af294e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "\n",
    "matQ =  np.reshape(vQ , (nstate, naction))\n",
    "V = np.zeros(nstate)\n",
    "for istate in range(nstate):\n",
    "    for iaction in range(naction):\n",
    "        V[istate] = sum(matQ[istate,:]*pi0[istate,:])\n",
    "\n",
    "plt.imshow(matQ, cmap=cm.Greys_r)\n",
    "plt.show()\n",
    "\n",
    "matV = np.reshape(V, (sizeY, sizeX))\n",
    "plt.imshow(matV, cmap=cm.Greys_r)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "pi1 = np.zeros((nstate, naction), dtype=np.float)\n",
    "\n",
    "matQ = np.reshape(vQ , (nstate, naction))\n",
    "\n",
    "\n",
    "for istate in range(nstate):\n",
    "    nQ = sum(matQ[istate,:]==max(matQ[istate,:]))\n",
    "    pi1[istate,matQ[istate,:]==max(matQ[istate,:])] = 1/nQ\n",
    "    \n",
    "matT = np.zeros((nstate*naction, nstate*naction), dtype=np.float)\n",
    "for istate in range(nstate):\n",
    "    for iaction in range(naction):\n",
    "        for jstate in range(nstate):\n",
    "            for jaction in range(naction):\n",
    "                matT[istate*naction +iaction, jstate*naction+jaction] = Tr[istate, iaction, jstate] * pi1[jstate, jaction]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "vER = np.ndarray.flatten(ER)\n",
    "vER = vER.reshape((nstate*naction,1))\n",
    "\n",
    "vQ = np.zeros((nstate*naction,1))\n",
    "for _ in range(1000):\n",
    "    vQ = vER + np.dot(matT, vQ)\n",
    "\n",
    "#print(vQ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEIAAAD8CAYAAADDlHLtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACLNJREFUeJztnV+MVGcZh58XOqsECKhQu+wCLYYL\nK4kIG6LWC0yjIWBEEzVtgumFERslKPHGVKKNiUkv1NILolmFWBO1qPiHGIyWamK8oPIntBZRQgl1\nl9kUULKlhKTQfb2Y325OZmfOnDln/m7eJ5nsmXO+833fPHzn+w4z77xj7k4A87rdgV4hRIgQIUKE\nCBEiRIgQIUKECBHiriInm9kW4ClgPvAjd3+iQfm23MaWSqW6x27fvn3N3Zc3qiO3CDObD+wHPgKM\nAyfM7Ii7/zNvnXlZvrz+6yyXy69kqaPIpbEJuODuF939DeAZYHuB+rpKERFDwFji+bj29SVF5gir\nsW/WHGBmO4GdBdrpCEVEjAMrE8+HgXJ1IXcfBUahfZNlKyhyaZwA1prZfWY2ADwEHGlNtzpP7hHh\n7nfMbBfwRyrL50F3P5t2TqlUqjvDl8uzBtMMK1asSO3LqlWr6h5LqzdJofsIdz8KHC1SR68Qd5Yi\nRIgQIUKECBEiRIhCy2ezDAwMpK759Wh0zurVq+seO378eKY2YkSIECFChAgRIkSIECE6vnymLXX1\naHTO4OBg3i7NECNChAgRIkSIECFChAjR0eWzVCrlWuoanRPLZwsJESJEiBAhQoQIEaIvls+RkZHU\n45cvX87bpRmKBpNdAm4AbwJ33D29xz1MK0bEh939Wgvq6SoxR4iiIhz4k5mdUqzULMxsp5mdNLOT\nN2/eLNhc+yh6aTzg7mUzuxt41sz+5e5/TRZIxlANDw/PyRgq3L2sv1eA31CJvexLcosws4Vmtnh6\nG/go8FKrOtZpLO+3/MxsDZVRAJVL7Gfu/u0G57Tl0kgLNiuXy6eyLOtFououAu/Ne36vEcunCBEi\nRIgQIUKE6Ph/w3s1FjtGhAgRIkSIECFChAgRIkKQRYwIESJEiBAhQoQIESJEhCCLGBEiRIgQIUKE\nCBEiRIgQIRreR5jZQeBjwBV3X6d9bwcOAfcCl4DPuPv1RnX1eyz2j4EtVfu+Bjzn7muB5/S8r2ko\nQhEw/6vavR14WttPA59ocb86Tt454p3uPgGgv3fXK5iMobp161bO5tpP2ydLdx919xF3H1mwYEG7\nm8tNXhGvmtkggP5eaV2XukNeEUeAR7T9CPC71nSne2RZPn8ObAaWmdk48E3gCeAXZvY54D/Ap7M0\nlnf53L49PSliR2Kx3f3hOoceLNx6DxF3liJEiBAhQoQIESJ3CHKuxno4BDlGhAgRIkSIECFChAgR\nIkKQRYwIESJEiBAhQoQIESJEiBARiy1iRIgQIUKECBEiRIgQMY27pz6Ag1Q+9n8pse9x4DJwRo+t\njepxdzZu3OidBjiZpW95Y6gAnnT39Xr0/S8z5Y2hmnMUmSN2mdmLZnbQzN7Wsh51ibwivg+8C1gP\nTADfrVcwGUx29erVnM21n1wi3P1Vd3/T3aeAH5KSf8oTwWRpP1DabXKJmA4kE5+kj/NPTZM3hmqz\nma2nkqvuEvCFLI2NjY2xZ8+emscmJibqnrdhw4bUeoeGiv/eat4YqgOFW+4x4s5ShAgRIkSIECFC\nhAjR0XexFy5cWDfHdVo89bZt21LrbcWte4wIESJEiBAhQoQIESJExGKLGBEiRIgQIUKECBEiRIiI\nxRYxIkSIECFChAgRIkSIEFk+BF4J/AS4B5gCRt39qTy5qPo9BPkO8FV3fzfwfuBLZnY/cywXVZYY\nqgl3P63tG8A5YIg5louqqTnCzO4F3gc8TxO5qPqBzLfYZrYIOAx8xd1fM7Os5+0EdkJljuhVMo0I\nMytRkfBTd/+1dmfKRZWMoSqVSq3oc1toKMIq//QHgHPu/r3EoTmViyrLpfEA8FngH2Z2RvseI2cu\nql4lSwzV34B6E0JTuajy5sU+duxY6vG9e/c2XWc1cWcpQoQIESJEiBAhQoTo+LvY9fJZHjhQP6p5\ncnIytd60fJf79u3L1LcYESJEiBAhQoQIESJEiI4un0uXLq271KUtkefPn0+ttxVfmosRIUKECBEi\nRIgQIUKEiBBkESNChAgRIkSIECFChAgRIkSRYLLHgc8D01/DfaxRPqpejsXO8n7EdDDZaTNbDJwy\ns2d17El3/06mlnqcLGEBE1RSLOHuN8xsOphsTlEkmAwyJOVK5qGampoq1Nl2kllEdTAZGZNyJWOo\n5s3r3bk5dzBZM0m5+oHcwWRzLSlXkWCyh5tNytWuWOy0GKtFixZlaqNIMFnf57BM0ruzV4cJESJE\niBAhQoTo+K8p5AlBPnz4cOrx/fv35+3SDDEiRIgQIUKECBEiRIgQHf0QeHh42Hfv3l3z2I4dO+qe\nd/166heMU1m3bl18CNwMIUKECBEiRIgQIUKECNHR/4YvWbKkbmrntHuFTvxSS4wIESJEiBAhQoQI\nESJEp2OxrwKvJHYtA661oOq0ela7e8PvOnVUxKzGzU5mea+gE/XEpSFChOi2iNFeqaerc0Qv0e0R\n0TO0XYSZbTGzf5vZBTOblerRzN5iZod0/HnFclaXWWlmfzGzc2Z21sy+XKPMZjObNLMzenyjqY5m\n+eHgvA9gPvAysAYYAF4A7q8q80XgB9p+CDhUo55BYIO2FwPna9SzGfh93r62e0RsAi64+0V3fwN4\nhkoezCTJvJi/Ah60qtSIKTk1W0a7RQwBY4nn48x+ATNl3P0OMAm8o16FNcKgk3zAzF4wsz+Y2Xua\n6Wi736GqFZZYvUxlKVMpODsMOslpKrfTr5vZVuC3wNqsHW33iBgHViaeDwPVX6CYKWNmdwFLqPHz\n3HVyas7g7q+5++vaPgqUzGxZ1o62W8QJYK2Z3WdmA1QmwyNVZZJ5MT8F/Nmrbm5Scmomy9wzPbeY\n2SYqr+2/mXvazlVDr2crlVn+ZeDr2vct4OPafivwS+AC8HdgTY06PkTlcnkROKPHVuBR4FGV2QWc\npbIyHQc+2Ew/485SxJ2lCBEiRIgQIUKECBEiRIgQIf4PGemtD+WpizsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2cf8afb4860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASQAAAD8CAYAAADe49kaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACfFJREFUeJzt3c2LXYUdxvHn6XSiohYXDXXIxERE\nBHGhJWQT6CLYEl/QLhXMSpiFFSItiC79B8RNEQaVpigGQQtqbW1AgwR8S2K0xtESxJcQIQ1GdDYV\n9elirhDSmbln6D1zfifn+4HBuZPL5SHE75xz7uVeJxEAVPCTrgcAwI8IEoAyCBKAMggSgDIIEoAy\nCBKAMggSgDIIEoAyCBKAMn7axoPa7s3Lv6enp7uesCYXXHBB1xMa69NWqV//FjZs2ND1hDX57LPP\nTifZOO5+rQSpTy6//PKuJ6zJVVdd1fWExq644oquJ6zJ7Oxs1xMa69NWSbrnnns+bXI/TtkAlEGQ\nAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAA\nlNEoSLZ32f7I9nHbD7Q9CsAwjQ2S7SlJf5R0k6RrJd1p+9q2hwEYniZHSNslHU/ycZJvJe2TdHu7\nswAMUZMgbZL0+Vm3T4x+BgAT1eRN/r3Mz/7nU0Vsz0ma+78XARisJkE6IWnzWbdnJZ08905J5iXN\nS/36GCQAdTQ5ZXtb0tW2r7S9QdIdkp5vdxaAIRp7hJTkO9v3SnpZ0pSkJ5Ica30ZgMFp9EGRSV6S\n9FLLWwAMHK/UBlAGQQJQBkECUAZBAlAGQQJQBkECUAZBAlAGQQJQBkECUAZBAlAGQQJQBkECUAZB\nAlAGQQJQBkECUAZBAlAGQQJQRqN3jARQy5YtW7qe0AqOkACUQZAAlEGQAJRBkACUQZAAlEGQAJRB\nkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlEGQAJRBkACUQZAAlDE2SLafsH3K9vvrMQjA\ncDU5QvqTpF0t7wCA8UFK8pqkL9dhC4CB4xoSgDIm9qkjtuckzU3q8QAMz8SClGRe0rwk2c6kHhfA\ncHDKBqCMJk/7Py3pdUnX2D5h++72ZwEYorGnbEnuXI8hAMApG4AyCBKAMggSgDIIEoAyCBKAMggS\ngDIIEoAyCBKAMggSgDIIEoAyCBKAMggSgDIIEoAyCBKAMggSgDIIEoAyJvae2sC59u7d2/WENXn0\n0Ue7ntDYzMxM1xNawRESgDIIEoAyCBKAMggSgDIIEoAyCBKAMggSgDIIEoAyCBKAMggSgDIIEoAy\nCBKAMggSgDIIEoAyCBKAMggSgDIIEoAyxgbJ9mbbr9pesH3M9p71GAZgeJq8he13kv6Q5IjtSyUd\ntr0/yQctbwMwMGOPkJJ8keTI6PtvJC1I2tT2MADDs6ZrSLa3SrpB0pttjAEwbI0/dcT2JZKelXRf\nkq+X+fM5SXMT3AZgYBoFyfa0lmL0VJLnlrtPknlJ86P7Z2ILAQxGk2fZLOlxSQtJHm5/EoChanIN\naYek3ZJ22j46+rq55V0ABmjsKVuSg5K8DlsADByv1AZQBkECUAZBAlAGQQJQBkECUAZBAlAGQQJQ\nBkECUAZBAlAGQQJQBkECUAZBAlAGQQJQBkECUAZBAlAGQQJQBkECUEbjTx0B1mpxcbHrCWuye/fu\nric0dubMma4ntIIjJABlECQAZRAkAGUQJABlECQAZRAkAGUQJABlECQAZRAkAGUQJABlECQAZRAk\nAGUQJABlECQAZRAkAGUQJABlECQAZYwNku0Lbb9l+13bx2w/tB7DAAxPk7ew/Y+knUkWbU9LOmj7\nb0neaHkbgIEZG6QkkfTjmyNPj77S5igAw9ToGpLtKdtHJZ2StD/Jm+3OAjBEjYKU5Psk10ualbTd\n9nXn3sf2nO1Dtg9NeiSAYVjTs2xJvpJ0QNKuZf5sPsm2JNsmtA3AwDR5lm2j7ctG318k6UZJH7Y9\nDMDwNHmWbUbSXttTWgrYM0lebHcWgCFq8izbe5JuWIctAAaOV2oDKIMgASiDIAEogyABKIMgASiD\nIAEogyABKIMgASiDIAEogyABKIMgASiDIAEogyABKIMgASiDIAEogyABKKPJO0aikBdeeKHrCY2d\nOXOm6wnnrdOnT3c9oRUcIQEogyABKIMgASiDIAEogyABKIMgASiDIAEogyABKIMgASiDIAEogyAB\nKIMgASiDIAEogyABKIMgASiDIAEogyABKKNxkGxP2X7H9ottDgIwXGs5QtojaaGtIQDQKEi2ZyXd\nIumxducAGLKmR0iPSLpf0g8tbgEwcGODZPtWSaeSHB5zvznbh2wfmtg6AIPS5Ahph6TbbH8iaZ+k\nnbafPPdOSeaTbEuybcIbAQzE2CAleTDJbJKtku6Q9EqSu1pfBmBweB0SgDLW9Mm1SQ5IOtDKEgCD\nxxESgDIIEoAyCBKAMggSgDIIEoAyCBKAMggSgDIIEoAyCBKAMggSgDIIEoAyCBKAMggSgDIIEoAy\nCBKAMggSgDIIEoAynGTyD2r/W9KnE37Yn0s6PeHHbFOf9vZpq9SvvX3aKrW3d0uSjePu1EqQ2mD7\nUJ8+0aRPe/u0VerX3j5tlbrfyykbgDIIEoAy+hSk+a4HrFGf9vZpq9SvvX3aKnW8tzfXkACc//p0\nhATgPNeLINneZfsj28dtP9D1ntXYfsL2Kdvvd71lHNubbb9qe8H2Mdt7ut60EtsX2n7L9rujrQ91\nvakJ21O237H9YtdbVmP7E9v/tH3U9qHOdlQ/ZbM9Jelfkn4t6YSktyXdmeSDToetwPavJC1K+nOS\n67resxrbM5Jmkhyxfamkw5J+W/Hv1rYlXZxk0fa0pIOS9iR5o+Npq7L9e0nbJP0sya1d71mJ7U8k\nbUvS6Wum+nCEtF3S8SQfJ/lW0j5Jt3e8aUVJXpP0Zdc7mkjyRZIjo++/kbQgaVO3q5aXJYujm9Oj\nr9K/TW3PSrpF0mNdb+mLPgRpk6TPz7p9QkX/p+kz21sl3SDpzW6XrGx0+nNU0ilJ+5OU3TryiKT7\nJf3Q9ZAGIukftg/bnutqRB+C5GV+Vvo3Y9/YvkTSs5LuS/J113tWkuT7JNdLmpW03XbZU2Lbt0o6\nleRw11sa2pHkl5JukvS70aWHddeHIJ2QtPms27OSTna05bwzuh7zrKSnkjzX9Z4mknwl6YCkXR1P\nWc0OSbeNrs3sk7TT9pPdTlpZkpOj/56S9BctXSpZd30I0tuSrrZ9pe0Nku6Q9HzHm84LowvFj0ta\nSPJw13tWY3uj7ctG318k6UZJH3a7amVJHkwym2Srlv7NvpLkro5nLcv2xaMnNWT7Ykm/kdTJs8Tl\ng5TkO0n3SnpZSxddn0lyrNtVK7P9tKTXJV1j+4Ttu7vetIodknZr6bf30dHXzV2PWsGMpFdtv6el\nX1L7k5R+Kr1HfiHpoO13Jb0l6a9J/t7FkPJP+wMYjvJHSACGgyABKIMgASiDIAEogyABKIMgASiD\nIAEogyABKOO/uFQFzbtYMZIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2cf8b05eb38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "matQ =  np.reshape(vQ , (nstate, naction))\n",
    "V = np.zeros(nstate)\n",
    "for istate in range(nstate):\n",
    "    for iaction in range(naction):\n",
    "        V[istate] = sum(matQ[istate,:]*pi1[istate,:])\n",
    "\n",
    "plt.imshow(matQ, cmap=cm.Greys_r)\n",
    "plt.show()\n",
    "\n",
    "matV = np.reshape(V, (sizeY, sizeX))\n",
    "plt.imshow(matV, cmap=cm.Greys_r)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-11.]\n",
      " [-11.]\n",
      " [-11.]\n",
      " [-10.]\n",
      " [-10.]\n",
      " [-11.]\n",
      " [-10.]\n",
      " [ -9.]\n",
      " [ -9.]\n",
      " [-10.]\n",
      " [ -9.]\n",
      " [ -8.]\n",
      " [ -8.]\n",
      " [ -9.]\n",
      " [ -8.]\n",
      " [ -7.]\n",
      " [ -7.]\n",
      " [ -8.]\n",
      " [ -7.]\n",
      " [ -6.]\n",
      " [ -6.]\n",
      " [ -7.]\n",
      " [ -5.]\n",
      " [ -6.]\n",
      " [-11.]\n",
      " [-11.]\n",
      " [-11.]\n",
      " [-10.]\n",
      " [-10.]\n",
      " [-11.]\n",
      " [-10.]\n",
      " [ -9.]\n",
      " [ -9.]\n",
      " [-10.]\n",
      " [ -9.]\n",
      " [ -8.]\n",
      " [ -8.]\n",
      " [ -9.]\n",
      " [ -8.]\n",
      " [ -7.]\n",
      " [ -7.]\n",
      " [ -8.]\n",
      " [ -7.]\n",
      " [ -6.]\n",
      " [ -6.]\n",
      " [ -7.]\n",
      " [ -4.]\n",
      " [ -5.]\n",
      " [-11.]\n",
      " [-11.]\n",
      " [-11.]\n",
      " [-10.]\n",
      " [-10.]\n",
      " [-11.]\n",
      " [-10.]\n",
      " [ -9.]\n",
      " [ -9.]\n",
      " [-10.]\n",
      " [ -9.]\n",
      " [ -8.]\n",
      " [  0.]\n",
      " [  0.]\n",
      " [  0.]\n",
      " [  0.]\n",
      " [ -7.]\n",
      " [ -8.]\n",
      " [ -6.]\n",
      " [ -5.]\n",
      " [ -5.]\n",
      " [ -6.]\n",
      " [ -3.]\n",
      " [ -4.]\n",
      " [-11.]\n",
      " [-11.]\n",
      " [-11.]\n",
      " [-10.]\n",
      " [-10.]\n",
      " [-11.]\n",
      " [-10.]\n",
      " [ -9.]\n",
      " [ -9.]\n",
      " [-10.]\n",
      " [ -9.]\n",
      " [ -8.]\n",
      " [ -8.]\n",
      " [ -9.]\n",
      " [ -1.]\n",
      " [ -7.]\n",
      " [ -7.]\n",
      " [ -1.]\n",
      " [ -2.]\n",
      " [ -4.]\n",
      " [ -4.]\n",
      " [ -2.]\n",
      " [ -4.]\n",
      " [ -3.]\n",
      " [-11.]\n",
      " [-11.]\n",
      " [-11.]\n",
      " [-10.]\n",
      " [-10.]\n",
      " [-11.]\n",
      " [-10.]\n",
      " [ -9.]\n",
      " [ -9.]\n",
      " [-10.]\n",
      " [ -9.]\n",
      " [ -1.]\n",
      " [ -8.]\n",
      " [ -9.]\n",
      " [ -2.]\n",
      " [ -6.]\n",
      " [ -6.]\n",
      " [ -2.]\n",
      " [ -3.]\n",
      " [ -3.]\n",
      " [ -3.]\n",
      " [ -3.]\n",
      " [ -4.]\n",
      " [ -4.]]\n"
     ]
    }
   ],
   "source": [
    "print(vQ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}