Windy02_DP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-2"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.choice([0,-1,-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ennvironment setup\n",
    "# Variables\n",
    "#   sizeX, sizeY : size of the gridworld\n",
    "# \n",
    "# Functions\n",
    "#   stateNum : input(x,y)= coordinate, output= state number\n",
    "#   stateNumToCoord : input(st)= state number, output[x,y]= coordinate\n",
    "#   keepInside : input(x,y)= coordinate, output[x,y]= coordinate\n",
    "#   moveAgent : input(x,y,a)= coordinate, action, output[x,y]=coordinate\n",
    "#   wind : input(x,y)=coordinate, output[x,y]=coordinate\n",
    "\n",
    "sizeX = 6\n",
    "sizeY = 5\n",
    "nstate = sizeX*sizeY\n",
    "naction =4\n",
    "def stateNum(x, y):\n",
    "    return(int(x + (y-1) *sizeX -1))\n",
    "\n",
    "dicAction = {'up':1, 'left':2, 'down':3, 'right':4}\n",
    "\n",
    "def stateNumToCoord(st):\n",
    "    return([st % sizeX +1, 1+st // sizeX])  # //, %  division and the remainder\n",
    "\n",
    "def keepInside(x, y):    \n",
    "    if (x<1): \n",
    "        x=1\n",
    "    elif (x>sizeX):\n",
    "        x=sizeX\n",
    "        \n",
    "    if (y<1):\n",
    "        y=1\n",
    "    elif (y>sizeY):\n",
    "        y=sizeY\n",
    "        \n",
    "    return([x,y])\n",
    "\n",
    "goalState = stateNum(4,3)\n",
    "\n",
    "\n",
    "\n",
    "def moveAgent(x, y, a):\n",
    "   # if (stateNum(x, y) == goalState):\n",
    "   #     return([x,y])\n",
    "    if a in dicAction:\n",
    "        a = dicAction[a]\n",
    "    \n",
    "    if (a == 1):\n",
    "        y = y - 1\n",
    "    if (a == 2):\n",
    "        x = x - 1\n",
    "    if (a == 3):\n",
    "        y = y + 1\n",
    "    if (a == 4):\n",
    "        x = x + 1\n",
    "    \n",
    "    return(keepInside(x,y))\n",
    "\n",
    "def wind(x,y):    \n",
    "    if (x in [2,5]):\n",
    "        \n",
    "        y=y-np.random.choice([0,1,2])\n",
    "    if (x in [3,4]):\n",
    "        y=y-np.random.choice([1,2,3])\n",
    "    return(keepInside(x,y))\n",
    "\n",
    "def prMoveAgent(prIstate, a):\n",
    "   # if (stateNum(x, y) == goalState):\n",
    "   #     return([x,y])\n",
    "    cumPr = np.zeros(nstate)\n",
    "    cumPr[goalState] = prIstate[goalState]\n",
    "    for istate in range(nstate):\n",
    "        if (istate != goalState and prIstate[istate] > 0):\n",
    "            x, y = stateNumToCoord(istate)\n",
    "            pr = np.zeros(nstate)\n",
    "               \n",
    "            if a in dicAction:\n",
    "                a = dicAction[a]\n",
    "    \n",
    "            if (a == 1):\n",
    "                y = y - 1\n",
    "            if (a == 2):\n",
    "                x = x - 1\n",
    "            if (a == 3):\n",
    "                y = y + 1\n",
    "            if (a == 4):\n",
    "                x = x + 1\n",
    "            x, y = keepInside(x,y)\n",
    "            pr[stateNum(x,y)]=1\n",
    "            cumPr = cumPr + prIstate[istate]*pr\n",
    "            \n",
    "    return(cumPr)\n",
    "    \n",
    "\n",
    "def prWind(prIstate):\n",
    "    cumPr = np.zeros(nstate)    \n",
    "    cumPr[goalState] = prIstate[goalState]\n",
    "    for istate in range(nstate):\n",
    "        if (istate != goalState and prIstate[istate] > 0):\n",
    "            x, y = stateNumToCoord(istate)                \n",
    "            pr = np.zeros(nstate)   \n",
    "            if (x in [2,5]):\n",
    "                x = np.ones(3)*x\n",
    "                y = np.array([y, y-1, y-2])     \n",
    "            elif (x in [3,4]):\n",
    "                x = np.ones(3)*x\n",
    "                y = np.array([y-1, y-2, y-3])     \n",
    "            else:\n",
    "                x = np.array([x])\n",
    "                y = np.array([y])\n",
    "            prob = 1/x.shape[0]            \n",
    "            for i in range(x.shape[0]):                \n",
    "                xTemp, yTemp = keepInside(x[i], y[i])\n",
    "                pr[stateNum(xTemp, yTemp)]=pr[stateNum(xTemp, yTemp)]+prob\n",
    "            \n",
    "            cumPr = cumPr + prIstate[istate]*pr   \n",
    "    \n",
    "    \n",
    "    \n",
    "    return(cumPr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    if i==1:\n",
    "        break\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.zeros(nstate)\n",
    "x.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prWind([0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0])\n",
    "prMoveAgent([0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "prIstate = [0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
    "\n",
    "cumPr = np.zeros(nstate)\n",
    "istate =4\n",
    "if (prIstate[istate] > 0):\n",
    "    x, y = stateNumToCoord(istate)                \n",
    "    pr = np.zeros(nstate)   \n",
    "    if (x in [2,5]):\n",
    "        x = np.ones(3)*x\n",
    "        y = np.array([y, y-1, y-2])     \n",
    "    elif (x in [3,4]):\n",
    "        x = np.ones(3)*x\n",
    "        y = np.array([y-1, y-2, y-3])  \n",
    "    else:\n",
    "        x = np.array([x])\n",
    "        y = np.array([y])\n",
    "    prob = 1/x.shape[0]\n",
    "    for i in range(x.shape[0]):                \n",
    "        xTemp, yTemp = keepInside(x[i], y[i])\n",
    "        pr[stateNum(xTemp, yTemp)]=pr[stateNum(xTemp,yTemp)]+prob\n",
    "     \n",
    "    cumPr = cumPr + prIstate[istate]*pr  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cumPr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "cumPr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape([0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0])\n",
    "nstate\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[2, 1]\n",
      "[3, 1]\n"
     ]
    }
   ],
   "source": [
    "# Test\n",
    "print(stateNum(1,1))\n",
    "print(moveAgent(3,1,'left'))\n",
    "print(wind(3,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4, 3]\n",
      "15\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4, 5]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# goal state\n",
    "print(stateNumToCoord(goalState))\n",
    "print(goalState)\n",
    "sNext[27,:]\n",
    "stateNumToCoord(27)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculating P(s' | s, a) -> Tr[s,a,s']\n",
    "nstate = sizeX*sizeY\n",
    "naction = 4\n",
    "Tr = np.zeros((nstate, naction, nstate), dtype=np.float) # float!\n",
    "\n",
    "for istate in range(nstate):\n",
    "    for iaction in [1,2,3,4]:     \n",
    "        pr = np.zeros(nstate)\n",
    "        pr[istate] = 1\n",
    "        pr = prWind(pr)\n",
    "        pr = prMoveAgent(pr, iaction)  \n",
    "        Tr[istate,iaction-1,:] = pr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.          0.66666667  0.          0.          0.          0.          0.\n",
      "  0.33333333  0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.        ]\n",
      "[ 0.          0.66666667  0.          0.          0.          0.          0.\n",
      "  0.33333333  0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.        ]\n",
      "[ 0.          0.66666667  0.          0.          0.          0.          0.\n",
      "  0.33333333  0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.          0.          0.          0.\n",
      "  0.          0.        ]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(Tr[stateNum(3,3),1,:])\n",
    "\n",
    "pr = np.zeros(nstate)\n",
    "pr[14]=1\n",
    "prWind(pr)\n",
    "print(prMoveAgent(prWind(pr), 2))\n",
    "        \n",
    "print(Tr[14, 1,:])   \n",
    "stateNum(3,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# arrR[s, a, s'] = R(s,a,s')\n",
    "arrR = np.zeros((nstate, naction, nstate), dtype=np.float)\n",
    "arrR[:,:,:] = -1\n",
    "arrR[goalState, :, :] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ER [s,a'] = Expected rewards of R(s,a,s') over s'\n",
    "ER = np.zeros((nstate, naction), dtype=np.float)\n",
    "for istate in range(nstate):\n",
    "    for iaction in [1,2,3,4]:\n",
    "        ER[istate, iaction-1] = sum(arrR[istate, iaction-1, :] * Tr[istate, iaction-1, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pi : policy\n",
    "# pi0 : random policy with pi(a|s)=1/4\n",
    "pi0 = np.zeros((nstate, naction), dtype=np.float)\n",
    "pi0[:, :] = 1/4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# matT[(s,a), (s',a')] = P((s',a')|(s,a)) given policy pi0\n",
    "matT = np.zeros((nstate*naction, nstate*naction), dtype=np.float)\n",
    "for istate in range(nstate):\n",
    "    for iaction in range(naction):\n",
    "        for jstate in range(nstate):\n",
    "            for jaction in range(naction):\n",
    "                matT[istate*naction +iaction, jstate*naction+jaction] = Tr[istate, iaction, jstate] * pi0[jstate, jaction]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.25"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test : given (1,1), 1:'up', what's the probablity of (1,1),2:'left'\n",
    "matT[stateNum(1,1)*naction + 0, stateNum(1,1)*naction + 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "I= np.diag(np.ones(nstate*naction))  # identity matrix I"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEIAAAD8CAYAAADDlHLtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACVlJREFUeJztnVuMVVcZx38fMx0uAiIwtA0lthgS\nRBNRCFHrA6aJaYixmihpH6QkRmyURMEEmpJo45MPauMDSDCF1kQt3qrEtGqDJEQSa7lWCNbABGQE\nCgUpl3DJDJ8P5z+4nTl7n332PreZfr/kZM7stfda6/xm7bPWOfubb5u7E8C4dnegUwgRIkSIECFC\nhAgRIkSIECFCxBDuXvgBPAy8ARwDnsyxv7fhcT7XaykhoQs4DswFeoBDwIIOFLE3z+spc2osAY65\ne5+73wJeAB4pUV9bKSNiNnAq8Xu/to1Kuksca1W2jfgoa2argFUl2mkJZUT0A3MSv98HnB6+k7tv\nAbYAmFnHfuYvc2q8BswzswfMrAd4FNjRmG61nsIjwt0HzGw18EcqM8hWdz/SsJ4lMKt2Fv6PcePS\n/56Dg4P52mjlN1RFT42SIva5++JabcTKUoQIESJEiBAhQoQIUWZlWYiurq6q27Pm+6zpMavOWvX+\nXxu59noHECJEiBAhQoQIESJES6dPM8uc6tKodUxPT09q2a1bt3K1ESNChAgRIkSIECFChAjR8ukz\nbarLmuYmTZqUWW+tT6d5iBEhQoQIESJEiBAhQoTomOkzi4kTJ2aW37x5s2iX7lBKhJmdAK4Ag8BA\nnoutnUojRsQn3f2tBtTTVuI9QpQV4cCfzGyfYqVGYGarzGyvme3t5P8WKntqPOjup81sFvCKmf3D\n3Xcnd0jGUHV3d3esiVIjwt1P6+c54EUqsZejksIizOxdZjZl6DnwKeBwozrWasqcGncDLyq+qRv4\nmbv/IeuAwcFBLl68WHdD165dK9TBeigTVdcHfKiBfWkrMX2KECFChAgRIkSIlsdQNYNGxGLHiBAh\nQoQIESJEiBAhQkQI8lAbufZ6BxAiRIgQIUKECBEiRIQgixgRIkSIECFChAgRIkSIECFqriPMbCvw\naeCcu39Q26YD24H7gRPAcnf/T466CsViz5w5M7PerGOvXr1aq1tAvhHxHJUMZEmeBHa6+zxgp34f\n1dQUoQiY4dfyHwGe1/Pngc82uF8tp+gS+253PwPg7mcUOlSVZB6qWrli2knTP2skY6i6urrGXAzV\nm2Z2L4B+nmtcl9pDURE7gMf1/HHgd43pTvvIM33+HFgKzDSzfuDbwHeBX5jZl4B/AV/I01jW9Nnd\nnd6V6dOnZ9Z74cKFPM1nUlOEuz+WUvRQ6dY7iFhZihAhQoQIESJEiJZ+i50Vgpy1/L58+XJmvY34\n94cYESJEiBAhQoQIESJEiI4JQc6aAluRDjpGhAgRIkSIECFChAgRIkSIURGLXWsdEbHYDSREiBAh\nQoQIESJEDJHjnn1bqVz2P5zY9jTwb+CgHsvy3C9v0aJF3mpo4L38nmNkDBXAM+6+UI+Xyv9J2kvR\nGKoxR5n3iNVm9rqZbTWz9zSsR22iqIgfAe8DFgJngO+n7ZhMyHX+/PmCzTWfQiLc/U13H3T328CP\nycg/5e5b3H2xuy/u7e0t2s+mU0jEUCCZ+ByjOP/UEEVjqJaa2UIquepOAF/J09iBAweYOnVq1bIV\nK1akHrd79+7UMoBLly7laT6TojFUz5ZuucOIlaUIESJEiBAhQoQIEaKl32J3dXUxefLkqmV79uxJ\nPa7WitQzrqSfOnUqtSxJjAgRIkSIECFChAgRIoRlTT0Nb8zM0y7oZl3oLXPXpYGBgX2e4y4PMSJE\niBAhQoQIESJEiBCjIha7FuPHj08tGxgYyFVHjAgRIkSIECFChAgRIkSei8BzgJ8A9wC3gS3u/sOi\nuajSPknevn079Zis6RFg5cqVqWUbN26s1aVKv3LsMwB8093fD3wU+JqZLWCM5aLKE0N1xt336/kV\n4CgwmzGWi6qu9wgzux/4MPAqw3JRAam5qEYDuZfYZjYZ+DXwDXe/nDe5VjIhVyeTa0SY2V1UJPzU\n3X+jzblyUSVjqBrR4WZRU4RV/vTPAkfd/QeJojGViyrPqfEg8EXg72Z2UNueomAuqk4lTwzVX4C0\nN4S6clH19vayfPnyqmVZ64isQDOA7du319ONqsTKUoQIESJEiBAhQoQI0dKLwD09PZ6W2nndunWp\nx+3atSuz3r6+vtSyw4cPx0XgeggRIkSIECFChAgRouUhyDNmzKhatnPnztTjJkyYkFnvrFnlvyWM\nESFChAgRIkSIECFChGh5CHLaReAyYcZZd2K4ceNGfPqshxAhQoQIESJEiBAhQoQoE0z2NPBlYCiD\nzlNl8lFlXQTOWicATJs2LbXs7NmzudrP833EUDDZfjObAuwzs1dU9oy7fy9XSx1OnrCAM1RSLOHu\nV8xsKJhsTFEmmAxyJOVK5qEq1dMmk1vE8GAyciblGjMxVFA9mKyepFyjgcLBZGMtKVeZYLLHiiTl\nSovPzPo6oNZNDdevX59atmbNmjzdKhVMNupzWCaJlaUIESJEiBAhQoRo6UXg+fPns23btqplWWHE\nGzZsyKx306ZNpfoFMSLuECJEiBAhQoQIESJES6fPkydPsmpV9X/427x5c+pxa9euzay3v7+/VL8g\nRsQdQoQIESJEiBAhQoQIEaKl64ju7u7U1M5pH88Brl+/nllvWnx3PcSIECFChAgRIkSIECFCtDoW\n+zxwMrFpJvBWA6rOque97l7zBmEtFTGicbO9jQggaUQ9cWqIECHaLWJLp9TT1veITqLdI6JjaLoI\nM3vYzN4ws2NmNiLVo5mNN7PtKn9VsZzD95ljZrvM7KiZHTGzr1fZZ6mZvW1mB/X4Vl0dzXPj4KIP\noAs4DswFeoBDwIJh+3wV2KznjwLbq9RzL/ARPZ8C/LNKPUuB3xfta7NHxBLgmLv3ufst4AUqeTCT\nJPNi/gp4yIaF3mXk1GwYzRYxG0jeKKufkS/gzj7uPgC8DaR+01IlDDrJx8zskJm9bGYfqKejzf6G\nqlpY4vBpKs8+lR1HhkEn2U9lOX3VzJYBvwXm5e1os0dEPzAn8ft9wOm0fcysG3g3VW7PnZJT8w7u\nftndr+r5S8BdZlY9xVEVmi3iNWCemT1gZj1U3gx3DNsnmRfz88CffdjiJiOnZnKfe4beW8xsCZXX\ndiF3T5s5a+j1LKPyLn8c2KBt3wE+o+cTgF8Cx4C/AXOr1PEJKqfL68BBPZYBTwBPaJ/VwBEqM9Nf\ngY/X089YWYpYWYoQIUKECBEiRIgQIUKECBHiv9cToe8QDsLJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1f6d0278908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAD8CAYAAAAylrwMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAE7tJREFUeJzt3X+sXGd95/H3J07iIkpJJdONsa0m\nKxykJMuG4LrZRt1tU9M4AcX9oe06ElB+aL2tzAq2W9G6ltoFZKmiLf2hbdF6IdqNSnGzCxQrZKGx\nCkVIhGCHkMT5AVckbG6d3ewVNBShGNn+9o853kzMnTtnnHtm5tjvlzTKzDkzz3xlJR9/88xzzpOq\nQpK0+i6YdQGSdK4yYCWpIwasJHXEgJWkjhiwktQRA1aSOmLASjrvJNme5LEkC0l+s7PvcR2spPNJ\nkjXAV4HXAovAl4Bbq+rh1f4uO1hJ55utwEJVfb2qvgccAHZ08UUXdjFoEttikWTWJUykT/X2qVaA\nkydPLlXVy87289u3b6+lpaVW7z1y5MhR4NmhQ/urav/Q6w3Ak0OvF4EfP9vaVtJJwKo7a9asmXUJ\nrfWpVoCLL7541iW01qdaAb75zW9+44V8fmlpicOHD7d6b5Jnq2rLSm9Z5lgnTaFTBJLON4vApqHX\nG4FjXXyRASupF6qq1aOFLwGbk1ye5GJgJ3Cwi5qdIpDUC6u14qmqTiR5O/BpYA1wW1UdXZXBz2DA\nSjrvVNVdwF1df48BK6kX+rhm3zlYSeqIHaykuTfBD1hzxYCV1At9DFinCCSpI3awknrBDlaS9P/Z\nwUrqhT52sAaspF7oY8A6RSBJHTFgJakjThFImnt9vdDADlaSOtIqYKe1A6MkjbKK94OdmrEB2+zA\n+KfATcCVwK1Jruy6MEkadk4GLFPcgVGSziVtAna5HRg3nPmmJLuSHE7SbmcySZrANDrYJP86ydEk\np5JsOePcnmaa9LEkN7YZr80qglY7MDbb4u5vCpmvPl2S2nkI+AXgvwwfbKZFdwJXAS8HDiW5oqpO\nrjRYm4Cd2g6MkjTKNOZXq+oRgOT7+sodwIGqOg48nmSBwfTpF1Yar80UwdR2YJSk5bSdHmhCeN3p\n6crmsWsVSmg1VXqmsR3sNHdglKRVsFRVW0adTHIIuHSZU3ur6hOjPrbMsbEtdasruaa1A6MkjbKK\n23ZvO4uPndVUqVdySdJ4B4GdSdYmuRzYDNw77kMGrKRemNIyrZ9Psgj8C+CTST7dfPdR4A7gYeBT\nwO5xKwjAm71I6okprSL4OPDxEef2AfsmGc+AlTT35vEy2DYMWEm90MeAdQ5WkjpiByupF/rYwRqw\nknrBgJWkjhiwktQBVxFIUof6GLCuIpCkjtjBSuqFPnawBqykXjBgJakjBqwkdcBVBD21zN47Ok9d\neGF//nNYu3btrEuYOgNWkjrSx4B1mZYkNZL8XpJHkzyQ5ONJLhk6tyfJQpLHktzYZjwDVlIvTGNH\nA+Bu4OqqehXwVWAPQJIrGeyofRWwHfizJGvGDWbASuqFaQRsVf11VZ1oXt7DYHNDgB3Agao6XlWP\nAwvA1nHjGbCS5l7bcG0Cdl2Sw0OPXWf5tW8F/lfzfAPw5NC5xebYivyRS1IvTNCdLlXVllEnkxwC\nLl3m1N6q+kTznr3ACeDDpz+2XEnjCjFgJfXCaq0iqKptK51P8svA64Gfqee+dBHYNPS2jcCxcd/l\nFIEkNZJsB34DuKWqvjt06iCwM8naJJcDm4F7x41nByupF6a0DvY/A2uBu5uLkO6pql+pqqNJ7gAe\nZjB1sLuqTo4bzICV1AvTCNiqesUK5/YB+yYZz4CVNPe8F4EkdciAlaSOGLCS1BEDVpI6UFWcOnVq\n1mVMzHWwktQRO1hJvdDHKYKxHWyS25I8neShaRQkScuZ0u0KV1WbKYL/xuD+h5I0M30M2LFTBFX1\nuSSXdV+KJI02b+HZhnOwkuZeX1cRrFrANje1Pdsb20rSis7rDraq9gP7AZL0709C0lzrY8C6DlaS\nOtJmmdZHgC8Ar0yymORt3ZclSc93rq4iuHUahUjSKNMKzyTvZbCD7CngaeDNVXUsg7tv/zFwM/Dd\n5vh948ZzikBSL5w6darV4wX6vap6VVVdA9wJ/HZz/CYG28RsZvBj/gfaDGbASuqFaUwRVNW3h16+\nmOd2jt0B3F4D9wCXJFk/bjzXwUrqhWnNrybZB7wJeAb46ebwBuDJobctNseeWmksO1hJc69t99qE\n8Lokh4cez1ufn+RQkoeWeexovmtvVW0CPgy8/fTHlitrXN12sJLONUtVtWXUyara1nKcvwA+CfwO\ng45109C5jcCxcQPYwUrqhWn8yJVk89DLW4BHm+cHgTdl4DrgmapacXoA7GAl9cSU5mB/N8krGSzT\n+gbwK83xuxgs0VpgsEzrLW0GM2Al9cI0AraqfnHE8QJ2TzqeAStp7s3jVVptGLCSesGAlaSOGLCS\n1BEDVpI6cN7vaCBJXbKDlaSOGLCS1BEDdsjg/rQ6nx0/fnzWJUzk2muvnXUJrZ04cWLWJUzkqafG\nXlW6oqri5MmTq1TN9NjBSuoFO1hJ6ogBK0kdMWAlqQOug5WkDtnBSlJH7GAlqQN9nSJwyxhJvTCN\nbbtPS/LrSSrJuuZ1kvxJkoUkDyRptWjaDlZSL0xx2+5NwGuB/z10+CZgc/P4ceADzT9XZAcrqRem\n2MH+IfAunr8t9w7g9hq4B7gkyfpxA9nBSpp7E4bnuiSHh17vr6r9bT6Y5Bbg76rqK2dc7r8BeHLo\n9WJzbMVrgA1YSb0wwb0Ilqpqy6iTSQ4Bly5zai/wW8DPLvexZY6NTXwDVlIvrNYcbFVtW+54kn8G\nXA6c7l43Avcl2cqgY9009PaNwLFx3+UcrKS513b+9YWEcFU9WFU/UlWXVdVlDEL12qr6P8BB4E3N\naoLrgGeqauwtwuxgJfXCjNfB3gXcDCwA3wXe0uZDBqykXpj2pbJNF3v6eQG7Jx3DgJU09/p6JZcB\nK6kX+nizl7E/ciXZlOQzSR5JcjTJO6ZRmCQNO3XqVKvHPGnTwZ4A/mNV3ZfkJcCRJHdX1cMd1yZJ\nwMQXGsyNsQHbLEV4qnn+D0keYXAFgwEraWrOyYAdluQy4NXAF5c5twvYtSpVSdIZzumATfKDwEeB\nd1bVt88831zru795b//+JCTNrXN62+4kFzEI1w9X1ce6LUmSvt852cFmcFHuh4BHqur93ZckSd+v\njwHb5l4E1wNvBG5Icn/zuLnjuiTpeaa5o8FqabOK4PMsf6suSZqaeQvPNrySS9Lcm8futA0DVlIv\nzNtVWm0YsJJ6wQ5WkjrQ17tpuaOBpF6YxiqCJP8pyd8tt2IqyZ4kC0keS3Jjm/HsYCX1whSnCP6w\nqn5/+ECSK4GdwFXAy4FDSa6oqhUvL7ODldQLM75d4Q7gQFUdr6rHGWwds3XchwxYSXNvwk0P1yU5\nPPSY9CZUb0/yQJLbkvxwc2wD8OTQexabYytyikBSL0wwRbBUVVtGnUxyCLh0mVN7gQ8A7wWq+ecf\nAG9l+YutxhZkwErqhdWag62qbW3el+S/Anc2LxeBTUOnNwLHxo3hFIGkXpjSKoL1Qy9/HnioeX4Q\n2JlkbZLLgc3AvePGs4OV1AtTWkXwviTXMPjf/yeAf9d899EkdzDYyeUEsHvcCgIwYCX1wLRuuF1V\nb1zh3D5g3yTjGbCSesFLZSWpIwasJHXA2xWeYbDTzPy74IJ+LaRYv379+DfNife85z2zLmEiGzaM\nXTc+N5599tlZlzCRBx988AWPYcBKUkcMWEnqSB9vV2jASpp7zsFKUocMWEnqiFMEktQRO1hJ6kBf\n9+QyYCX1gh2sJHXEgJWkjhiwktQB18FKUof6GLD9utOJpPPWtLbtTvLvkzyW5GiS9w0d35NkoTl3\nY5ux7GAl9cI0OtgkPw3sAF5VVceT/Ehz/EpgJ3AV8HLgUJIrxm0bYwcrae613fBwFUL4V4Hfrarj\nzfc+3RzfARyoquNV9TiwAGwdN5gBK6kXJgjYdUkODz12TfA1VwA/meSLSf42yY81xzcATw69b7E5\ntiKnCCT1wgTd6VJVbRl1Mskh4NJlTu1lkIk/DFwH/BhwR5J/Ciy3g8DYgsYGbJIfAD4HrG3e/z+r\n6nfGfU6SVtNqXSpbVdtGnUvyq8DHapDm9yY5Baxj0LFuGnrrRuDYuO9qM0VwHLihqv45cA2wPcl1\nLT4nSatiinOwfwXcAJDkCuBiYAk4COxMsjbJ5cBm4N5xg43tYJsk/07z8qLm0b8FaZJ6bUrrYG8D\nbkvyEPA94JebDDya5A7gYeAEsHvcCgJoOQebZA1wBHgF8KdV9cWzrV6SzsY0Araqvge8YcS5fcC+\nScZrtYqgqk5W1TUM5h22Jrn6zPck2XX6V7tJCpCkNqY0RbCqJlqmVVV/D3wW2L7Muf1VtWWlX+8k\n6WydkwGb5GVJLmmevwjYBjzadWGSdNrpG25P41LZ1dRmDnY98N+bedgLgDuq6s5uy5Kk55u37rSN\nNqsIHgBePYVaJGmkczJgJWnW3JNLkjpkBytJHTFgJakjThFIUgfmcY1rGwaspF4wYCWpIwasJHXE\ngJWkDlQVJ0+OvTvg3DFgJfVCH1cRuOmhpLk3rR0NkvxlkvubxxNJ7h86tyfJQpLHktzYZjw7WEm9\nMI0Otqr+zennSf4AeKZ5fiWwE7gKeDlwKMkV43Y1sIOV1AvTvB9skgC/BHykObQDOFBVx6vqcWAB\n2DpuHDtYSb0wQXiuO2Nnlf1VtX/Cr/tJ4P9W1dea1xuAe4bOLzbHVmTASpp7E64iWFppZ5Ukh4BL\nlzm1t6o+0Ty/lee6V4AsV9a4QgxYSb2wWnOwVbVtpfNJLgR+AXjN0OFFYNPQ643AsXHf1VnAXnBB\nP6Z3L7roolmXMJGXvvSlsy6htauv/r69Mefa4uLirEto7Vvf+tasS5iqKd+LYBvwaFUN/wtxEPiL\nJO9n8CPXZuDecQPZwUrqhSkG7E6ePz1AVR1NcgfwMHAC2D1uBQEYsJJ6YloXGlTVm0cc3wfsm2Qs\nA1ZSL3gvAknqgPcikKQO2cFKUgfcVVaSOmQHK0kdsYOVpA74I5ckdcgpAknqiAErSR1wFYEkdcgO\nVpI6YgcrSR1wikCSOmTASlJH+jgH249tBySd96axq2ySa5Lck+T+JIeTbG2OJ8mfJFlI8kCSa9uM\n1zpgk6xJ8uUkd55t8ZJ0NtqG6yp0ue8D3l1V1wC/3bwGuInBNjGbgV3AB9oMNkkH+w7gkQneL0mr\nZkoBW8APNc9fynMbG+4Abq+Be4BLkqwfN1irOdgkG4HXMdgu4dcmLlmSXqAJ7kWwLsnhodf7q2p/\ny8++E/h0kt9n0ID+RHN8A/Dk0PsWm2NPrTRY2x+5/gh4F/CSUW9IsotB6yxJq2rC7nSpqraMOpnk\nEHDpMqf2Aj8D/Ieq+miSXwI+xGCX2SxX1rhCxgZsktcDT1fVkSQ/Nep9zd8Q+5vP9O/nPklzbbVW\nEVTVtlHnktzOYDoU4H8AH2yeLwKbht66keemD0ZqMwd7PXBLkieAA8ANSf68xeckadVMaQ72GPCv\nmuc3AF9rnh8E3tSsJrgOeKaqVpwegBYdbFXtAfYANB3sr1fVG86icEk6a1NaB/tvgT9OciHwLM9N\ne94F3AwsAN8F3tJmMC80kDT3pnXD7ar6PPCaZY4XsHvS8SYK2Kr6LPDZSb9Ekl6oPl7JZQcrqRcM\nWEnqiAErSR1YpRUCU2fASuoFb1coSR0xYCWpA04RSFKHDFhJ6ogBK0kdcQ5WkjrgHKwkdcgOVpI6\nYgcrSR0xYCWpA1XlFIEkdcUOVpI60scONl38rZDk/wHfWOVh1wFLqzxml/pUb59qhX7V26daobt6\nf7SqXna2H07yKQa1tbFUVdvP9rtWUycB24Ukh1fainfe9KnePtUK/aq3T7VC/+qdd212lZUknQUD\nVpI60qeA3T/rAibUp3r7VCv0q94+1Qr9q3eu9WYOVpL6pk8drCT1igErSR3pRcAm2Z7ksSQLSX5z\n1vWsJMltSZ5O8tCsaxknyaYkn0nySJKjSd4x65pGSfIDSe5N8pWm1nfPuqY2kqxJ8uUkd866lpUk\neSLJg0nuT3J41vWcK+Z+DjbJGuCrwGuBReBLwK1V9fBMCxshyb8EvgPcXlVXz7qelSRZD6yvqvuS\nvAQ4AvzcPP7ZJgnw4qr6TpKLgM8D76iqe2Zc2oqS/BqwBfihqnr9rOsZJckTwJaq6tNFEXOvDx3s\nVmChqr5eVd8DDgA7ZlzTSFX1OeCbs66jjap6qqrua57/A/AIsGG2VS2vBr7TvLyoecx1d5BkI/A6\n4IOzrkWz0YeA3QA8OfR6kTkNgT5LchnwauCLs61ktOZ/t+8Hngburqq5rbXxR8C7gD5cRF/AXyc5\nkmTXrIs5V/QhYLPMsbnuXPomyQ8CHwXeWVXfnnU9o1TVyaq6BtgIbE0yt1MwSV4PPF1VR2ZdS0vX\nV9W1wE3A7maqSy9QHwJ2Edg09HojcGxGtZxzmvnMjwIfrqqPzbqeNqrq74HPAnNxQ48RrgduaeY2\nDwA3JPnz2ZY0WlUda/75NPBxBlNzeoH6ELBfAjYnuTzJxcBO4OCMazonND8cfQh4pKreP+t6VpLk\nZUkuaZ6/CNgGPDrbqkarqj1VtbGqLmPw7+zfVNUbZlzWspK8uPmRkyQvBn4WmPtVMH0w9wFbVSeA\ntwOfZvAjzB1VdXS2VY2W5CPAF4BXJllM8rZZ17SC64E3Muiu7m8eN8+6qBHWA59J8gCDv3Tvrqq5\nXvrUI/8E+HySrwD3Ap+sqk/NuKZzwtwv05Kkvpr7DlaS+sqAlaSOGLCS1BEDVpI6YsBKUkcMWEnq\niAErSR35R8TdrJgZg2OhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1f6cffbdb00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gamma=0.99\n",
    "inv=np.linalg.inv(I- gamma* matT)   # can't solve the equation with gamma=1\n",
    "vER = np.ndarray.flatten(ER)\n",
    "vQ = np.dot(inv, vER)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "\n",
    "matQ =  np.reshape(vQ , (nstate, naction))\n",
    "V = np.zeros(nstate)\n",
    "for istate in range(nstate):\n",
    "    for iaction in range(naction):\n",
    "        V[istate] = sum(matQ[istate,:]*pi0[istate,:])\n",
    "\n",
    "plt.imshow(matQ, cmap=cm.Greys_r)\n",
    "plt.show()\n",
    "\n",
    "matV = np.reshape(V, (sizeY, sizeX))\n",
    "plt.imshow(matV, cmap=cm.Greys_r)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.25  0.25  0.25  0.25  0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.  ]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEIAAAD8CAYAAADDlHLtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACYtJREFUeJztnW2MVFcZx38Pu8AGAcEFtg0QWciS\nWAyCEILWD2saCSHEYqKm/SANMWCjJJr4xdREGxOTflBLPxANKhETtfiKxCC1KSbGECsv2XYLqOGt\nlpcAK4TSZKFZ9vHD/Hcdl7l37tw7Mzu7eX7JZO/cc8+5Z3575p47d555rrk7AUwZ7w60CiFChAgR\nIkSIECFChAgRIkSIEO1FKpvZRuAFoA34kbs/V2X7hpzGtrcnv4yhoaEBd59ftY28OzezNmA38Ang\nEnDMzA66++m8beZlzpw5iWUDAwNvZmmjyFtjHXDW3c+7+7vAi8DjBdobV4qIWAi8Vfb8ktZNSIoc\nI6zCugeOAWa2A9hRYD9NoYiIS8DisueLgCtjN3L3PcAeaNzBsh4UeWscA3rMrNvMpgFPAAfr063m\nk3tEuPuQme0EXqI0fe5191OpO2tvTzzCDwwMJNabN29eal8WLFiQWJbW7v/1LdNWCbj7IeBQkTZa\nhTizFCFChAgRIkSIECFCFJo+a95Ze3vqnJ9EtToLFyZ/xDl9OtuH4RgRIkSIECFChAgRIkSIpk6f\nU6dOTZ3qkqhWJ8+UPJYYESJEiBAhQoQIESJEiKZPn3mmukWLFqWWz5o1K2+XRokRIUKECBEiRIgQ\nIUKEaPrF287OzprrdXd3p5Zn/aI3jaLBZBeBO8B9YMjd1xbu0ThRjxHxcXcv/i8ZZ+IYIYqKcOBP\nZnZCsVIPYGY7zOy4mR0fHBwsuLvGUfSt8ai7XzGzBcDLZvYPd/9L+QblMVRdXV2TMoYKd7+iv9eB\n31GKvZyQ5BZhZu8xs1kjy8AG4I16dazZWN5f+ZnZUkqjAEpvsZ+7+7er1Mm1s7lz56aWt7W1JZYN\nDAycyDKtF4mqOw98KG/9ViOmTxEiRIgQIUKECNH0j+F5YrHTpkeoTyx2jAgRIkSIECFChAgRIkSE\nIIsYESJEiBAhQoQIESJEiAhBFjEiRIgQIUKECBEiRIgQIUJUPY8ws73AZuC6u39Q694H7AeWABeB\nz7r7rWpt5Y3FXrZsWWr5zJkza25zLFlGxE+AjWPWfQ14xd17gFf0fEJTVYQiYG6OWf04sE/L+4At\nde5X08l7jOhy96sA+ps43stjqO7evZtzd42n4QdLd9/j7mvdfW1HR0ejd5ebvCKumdnDAPp7vX5d\nGh/yijgIPKXlp4Df16c740eW6fMXQC8wz8wuAd8EngN+aWafB/4NfCbTzlJisadNm5ZYb+XKlant\nNiUW292fTCh6rPDeW4g4sxQhQoQIESJEiBC5Q5Bz7SwlBDktzHj+/PRszjdvjv0o9D+yhiDHiBAh\nQoQIESJEiBAhQrRMCPLt27cT61W7OBshyHUkRIgQIUKECBEiRIgQISZELHa1hFsRi11HQoQIESJE\niBAhQsQI7p76APZS+tr/jbJ1zwKXgT49NlVrx91Zs2aNNxvgeJa+5Y2hAnje3VfpMeHvzJQ3hmrS\nUeQYsdPMXjezvWaWngRmApBXxPeBZcAq4Crw3aQNy4PJbty4kXN3jSeXCHe/5u733X0Y+CEp+ae8\nLJis2ld340kuESOBZOJTTOD8UyPkjaHqNbNVlHLVXQS+kGVnFy5cYOvWrRXLVqxYkVjvzp07qe1O\nnz49y+5TyRtD9ePCe24x4sxShAgRIkSIECFChAjR1KvYHR0d9PT0VCxL+9Z6/fr1qe1ev178xwEx\nIkSIECFChAgRIkSIEE2NxW5vb/fZs2dXLEu7aFMto0ja1Nvf3x+x2LUQIkSIECFChAgRIkRTP32a\nWWJq57QpsEgMVX9/f6a+xYgQIUKECBEiRIgQIUJk+RJ4MfBT4CFgGNjj7i/kyUWVFoKcdgG2WjhB\nb29vYtnhw4dT646QZUQMAV919w8A64EvmdkjTLJcVFliqK66+0kt3wHOAAuZZLmoajpGmNkSYDXw\nKjXkopoIZBZhZjOB3wBfcfe3a6g3GkN1//79PH1sCplEmNlUShJ+5u6/1epMuajKY6iq3UJmPKkq\nwsyMUoTMGXf/XlnRpMpFleXT56PA54B+M+vTumfImYuqVckSQ/VXwBKKa8pF1dnZmRhMlhYwtmVL\n+oR08uTJWrpRkTizFCFChAgRIkSIECFCNPUq9r179zh37lzFss2bNyfWq/Zr3nqcuseIECFChAgR\nIkSIECFCNHX6nDFjBqtXr65YlnYVe/ny5ant1uPXgzEiRIgQIUKECBEiRIgQTZ0+r127xq5duyqW\n5bnZ4QhdXV25644QI0KECBEiRIgQIUKECBEiRJFgsmeB7cDIZ+BnquWjGh4eZnBwsGLZ5cuXE+sl\n/Xp4hKQ7PdVClhOqkWCyk2Y2CzhhZi+r7Hl3/07hXrQAWcICrlJKsYS73zGzkWCySUWRYDLIkJSr\nPIZqeHi4UGcbSZFgskxJucpjqKZMad1jc+5gslqSck0EcgeTTbakXEWCyZ6sNSlXW1tb4s+S0mKo\nlixZktruhg0bEst2795drVtAsWCyCZ/DspzWPXo1mRAhQoQIESJEiKZexe7u7mbfvn0Vy44ePZpY\nb9u2bantHjlypFC/IEbEKCFChAgRIkSIECFCNHX6vHXrFgcOHKhYtn379sR6fX19iWUAQ0NDhfoF\nMSJGCREiRIgQIUKECBEiRIiWicVOuzlytZyb9biBSYwIESJEiBAhQoQIESJEU9NBm9kN4M2yVfOA\n5ESW2Ulr5/3uXnV+baqIB3ZudjxLzupmtBNvDREixHiL2NMq7YzrMaKVGO8R0TI0XISZbTSzf5rZ\nWTN7INWjmU03s/0qf1WxnGO3WWxmfzazM2Z2ysy+XGGbXjO7bWZ9enyjpo5muXFw3gfQBpwDlgLT\ngNeAR8Zs80XgB1p+AthfoZ2HgQ9reRbwrwrt9AJ/yNvXRo+IdcBZdz/v7u8CL1LKg1lOeV7MXwOP\nKaRxlJScmnWj0SIWAm+VPb/Egy9gdBt3HwJuA4nh9hXCoMv5iJm9ZmZ/NLPkG4hWoNFXqCqFJY6d\nprJsU9owPafmSUqn0++Y2SbgAJD++4YyGj0iLgGLy54vAq4kbWNm7cB7qXB77oScmqO4+9vu/o6W\nDwFTzWxe1o42WsQxoMfMus1sGqWD4cEx25Tnxfw0cMTHnNyk5NQs3+ahkWOLma2j9Nr+k7mnjZw1\n9Ho2UTrKnwO+rnXfAj6p5Q7gV8BZ4O/A0gptfIzS2+V1oE+PTcDTwNPaZidwitLM9Dfgo7X0M84s\nRZxZihAhQoQIESJEiBAhQoQIEeK/5vS/J5K1FwwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1f6d02bfb00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAD8CAYAAAAylrwMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAELdJREFUeJzt3X+MHdV5xvHn8XptgmNKkRfb2FZN\nFYhEIIJ24zZFbVTHUIcQaFKlAok0UqOuGoXK9IfcUKutIiVSm1ZpKhWJrAiqoqBQGnBBIcWxFbuI\nCoPXYH4YA3VT0mxMZa9KlKAKI3bf/rFjcjF79879cWbmXH8/0hU7d/aeeW3sx++eOTPjiBAAYPCW\n1F0AAAwrAhYAEiFgASARAhYAEiFgASARAhYAEiFgAZxxbG+1/YLto7Y/m+w4rIMFcCaxPSLpRUlX\nSZqWdEDSjRHx3KCPRQcL4EyzSdLRiPheRLwu6W5J16c40NIUg9rOpi0eGRmpu4SuLF2a5H9ZEqOj\no3WX0JVly5bVXUJpuf25PXHixExEjPX6+a1bt8bMzEyp7z148OBhSa+1vDUZEZMt2+sk/aBle1rS\nL/Va22Ly+duayMqVK+suoStjYz3/Ga3cunXr6i6hK2vXrq27hNLOO++8ukvoym233fb9fj4/MzOj\nqampUt9r+7WIGF/sWxZ4L0lTyBQBgDPNtKQNLdvrJR1LcSACFkAWIqLUq4QDki6yfaHtZZJukPRA\niprP+CkCAHkY1IqniHjD9s2SdkkakXRnRBweyOCnIWABnHEi4tuSvp36OAQsgCzkuGafOVgASIQO\nFkDjdXECq1EIWABZyDFgmSIAgEToYAFkgQ4WAPAmOlgAWcixgyVgAWQhx4BligAAEiFgASARpggA\nNF6uFxrQwQJAIqUCtqonMAJAOwO8H2xlOgZs8QTG2yR9SNIlkm60fUnqwgCg1VAGrCp8AiMADJMy\nJ7lKPYHR9oSkiQHVBQBv0bTutIwyAVvqCYzFY3Enpbwe2w0AqZQJ2MqewAgA7eTYwZaZg63sCYwA\nsJCyJ7iaFsIdO9gqn8AIAMOk1JVcVT2BEQDaaVp3WgZXcgFAIgQsgCxUMQdr+29sP2/7ads7bZ/b\nz3gELIAsVHSSa7ekSyPivZJelHRrP4MRsAAar6pVBBHxnYh4o9jcr/llqT3jdoUAstBFeK6yPdWy\nPVlcCNWt35X0Tz187k0ELIBhMxMR4+122t4jac0Cu3ZExP3F9+yQ9Iaku/ophIAFkIVBLdOKiC2L\n7bf9SUnXSvpg9HlQAhZAFqpYB2t7q6Q/lfSBiPi/fscjYAFkoaILDf5B0nJJu21L0v6I+P1eByNg\nATReVfcZiIh3DXI8AhZAFrhUFgDwJjpYAFnIsYMlYAFkgYAFgEQIWABIoIlPKygjWcCOjIykGhpI\nIqc/sytWrKi7hMoRsACQSI4ByzItAEiEDhZAFnLsYAlYAFkgYAEgAVYRAEBCBCwAJJJjwLKKAAAS\noYMFkIUcO1gCFkAWCFgASIBVBACQEAELAIkQsACQCAELAAlEhObm5uouo2usgwWARAhYAFk4tZKg\n02sQbP+J7bC9qp9xOgas7TttH7f9bD8HAoB+VBWwtjdIukrSf/c7VpkO9h8lbe33QADQjwo72L+T\ntF1S34N1PMkVEQ/b3tjvgQCgH12E5yrbUy3bkxExWeaDtq+T9MOIeMp2tyW+DasIADRel6sIZiJi\nvN1O23skrVlg1w5Jfybp6u4rXNjAAtb2hKSJQY0HAK0GdQIrIrYs9L7tyyRdKOlU97pe0hO2N0XE\n//RyrIEFbNGCTxaF5rciGECjpb7QICKekXT+qW3bL0kaj4iZXsdkmRYAJFJmmdY3JD0q6d22p21/\nKn1ZAPBWVa6DLY63sZ/uVSq3iuDGfg4AAP3idoUAkFCO9yIgYAFkgQ4WABIhYAEggVznYFmmBQCJ\n0MECyAInuQAgkRynCAhYAFkgYAEggVxPchGwALJAwAJAIgQsACRCwAJAAl0+0aAxCFgAWaCDBYBE\nCFgASISALSxZskRnn312iqEHbmRkpO4ShtbevXvrLqEr27dvr7uE0sbGxuouoVIRodnZ2brL6Bod\nLIAs0MECQCIELAAkQsACQAKsgwWAhHLsYHmiAYAszM3NlXr1y/Yf2H7B9mHbX+xnLDpYAI1X1RSB\n7V+XdL2k90bESdvn9zMeAQsgCxVNEXxa0l9FxMnimMf7GYwpAgBZOHXT7U4vSatsT7W8Jro4zMWS\nftX2Y7b/zfb7+qmZDhZAFrroYGciYrzdTtt7JK1ZYNcOzWfiz0r6ZUnvk3SP7Z+PHttnAhZA4w3y\nkTERsaXdPtuflnRfEaiP256TtErSiV6ORcACyEJF9yL4F0mbJe2zfbGkZZJmeh2MgAWQhYpOct0p\n6U7bz0p6XdIne50ekAhYABmo6qmyEfG6pJsGNR4BCyALXCoLAInkeKksAQug8bjZCwAklGMH2/FK\nLtsbbO+1faS4+cG2KgoDgFZV3exlkMp0sG9I+uOIeML2SkkHbe+OiOcS1wYAkqpbRTBoHQM2Il6W\n9HLx9U9sH5G0ThIBC6AyQxmwrWxvlHSFpMcW2DchaaL4egClAcBPDXXA2n6npHsl3RIRPz59f0RM\nSpqUpJGRkfx+JwA01lA/ttv2qObD9a6IuC9tSQDwdkPZwXr+5/2vSjoSEV9KXxIAvF2OAVvmhttX\nSvqEpM22DxWvaxLXBQBv0cUNtxujzCqCRyRx1gpArZoWnmVwJReAxmtid1oGAQsgC027SqsMAhZA\nFuhgASAB7qYFAAnRwQJAIgQsACTCFAEAJMAyLQBIKMeALXOpLADUropLZW1fbnt/cUuAKdub+hmP\ngAWQhYruRfBFSZ+LiMsl/UWx3TOmCABkoaIpgpB0TvH1z0g61s9gBCyAxuvyhturbE+1bE8WDwQo\n4xZJu2z/reZ/wv+VLsp8GwIWQBa66GBnImK83U7beyStWWDXDkkflPSHEXGv7d/W/L2wt3Rb6ykE\nLIAsDGqKICLaBqbtr0naVmz+s6Q7+jkWJ7kANF7ZE1wDCOFjkj5QfL1Z0n/0M1iSDnbp0qUaGxtL\nMfTA5ba2bsOGDXWXUNrtt99edwldueCCC+ouobTVq1fXXULlKvq7+nuS/t72UkmvqXhSdq+YIgCQ\nhSoCtniCyy8OajwCFkAWuBcBACTAvQgAICECFgASYYoAABKhgwWABHgmFwAkRAcLAIkQsACQCAEL\nAAmwDhYAEiJgASARVhEAQCJ0sACQAHOwAJAQAQsAiQxlwNo+S9LDkpYX3//NiPjL1IUBQKthPcl1\nUtLmiHjV9qikR2z/a0TsT1wbAEga4jnYmP9VvVpsjhav/H6lALKWY8CWeqqs7RHbhyQdl7Q7Ih5L\nWxYAvFVFT5UdqFIBGxGzEXG5pPWSNtm+9PTvsT1he8r2VI5zJQCabWgD9pSI+JGkfZK2LrBvMiLG\nI2J8yZKuhgWAjoYyYG2P2T63+PodkrZIej51YQBwyqkbbpd5NUmZVnOtpL22n5Z0QPNzsN9KWxYA\nvFUVHaztj9s+bHvO9vhp+261fdT2C7Z/o8x4ZVYRPC3pih7rBYCBqOjH/2clfUzSV1rftH2JpBsk\nvUfSBZL22L44ImYXG4wruQA0XlXP5IqII5Jk+/Rd10u6OyJOSvov20clbZL06GLjEbAAstBFB7vK\n9lTL9mRETPZ5+HWSWi+umi7eWxQBCyALXQTsTESMt9tpe4+kNQvs2hER97f72EIldSqEgAWQhUFN\nEUTElh4+Ni1pQ8v2eknHOn2IBasAGq/sCoKEJ8IekHSD7eW2L5R0kaTHO32IgAWQhYqWaX3U9rSk\n90t60Pau4tiHJd0j6TlJD0n6TKcVBBJTBAAyUcUyrYjYKWlnm31fkPSFbsYjYAFkoWmXwZZBwAJo\nvIjQ7GzHn8gbh4AFkIWm3WegDAIWQOM18U5ZZRCwALJABwsAidDBAkAiBCwAJMAqAgBIiDnYwujo\nqNauXZti6DPe6tWr6y6htMsuu6zuErryyiuv1F1CaStXrqy7hEqxigAAEiJgASARpggAIBE6WABI\ngFUEAJAQHSwAJFDVU2UHjYAFkAU6WABIhA4WABLgJBcAJMQUAQAkQsACQAKsIgCAhHLsYJfUXQAA\nlDE3N1fq1Q/bH7d92Pac7fGW96+yfdD2M8V/N5cZjw4WQONVOEXwrKSPSfrKae/PSPpIRByzfamk\nXZLWdRqMgAWQhSoCNiKOSJLt099/smXzsKSzbC+PiJOLjUfAAshCF3Owq2xPtWxPRsTkAEv5LUlP\ndgpXiYAFkIkuAnYmIsbb7bS9R9KaBXbtiIj7FxvY9nsk/bWkq8sUUjpgbY9ImpL0w4i4tuznAKBf\ng3xkTERs6eVzttdL2inpdyLiP8t8ppsOdpukI5LO6aE2AOhLncu0bJ8r6UFJt0bEv5f9XKllWkVy\nf1jSHb2VBwD9mZ2dLfXqh+2P2p6W9H5JD9reVey6WdK7JP257UPF6/xO45XtYL8sabukto+ytD0h\naUKSli9fXnJYAOisqqfKRsROzU8DnP7+5yV9vtvxOnawtq+VdDwiDnYobDIixiNifOlSzp0BGKxT\nIdvp1SRlkvBKSdfZvkbSWZLOsf31iLgpbWkA8FNNC88yOnawEXFrRKyPiI2SbpD0XcIVQNWGtYMF\ngFqdETfcjoh9kvYlqQQAFtG07rQMOlgAWSBgASARAhYAEmjiCawyCFgAWeCRMQCQCAELAAkwRQAA\nCRGwAJAIAQsAiTAHCwAJMAcLAAnRwQJAInSwAJAIAQsACUQEUwQAkAodLAAkkmMH6xT/Ktg+Ien7\nAx52laSZAY+ZUk715lSrlFe9OdUqpav35yJirNcP235I87WVMRMRW3s91iAlCdgUbE9FxHjddZSV\nU7051SrlVW9OtUr51dt0HR96CADoDQELAInkFLCTdRfQpZzqzalWKa96c6pVyq/eRstmDhYAcpNT\nBwsAWSFgASCRLALW9lbbL9g+avuzddezGNt32j5u+9m6a+nE9gbbe20fsX3Y9ra6a2rH9lm2H7f9\nVFHr5+quqQzbI7aftP2tumtZjO2XbD9j+5DtqbrrGRaNn4O1PSLpRUlXSZqWdEDSjRHxXK2FtWH7\n1yS9KulrEXFp3fUsxvZaSWsj4gnbKyUdlPSbTfy9tW1JKyLiVdujkh6RtC0i9tdc2qJs/5GkcUnn\nRMS1ddfTju2XJI1HRE4XRTReDh3sJklHI+J7EfG6pLslXV9zTW1FxMOS/rfuOsqIiJcj4oni659I\nOiJpXb1VLSzmvVpsjhavRncHttdL+rCkO+quBfXIIWDXSfpBy/a0GhoCObO9UdIVkh6rt5L2ih+3\nD0k6Lml3RDS21sKXJW2XlMNF9CHpO7YP2p6ou5hhkUPAeoH3Gt255Mb2OyXdK+mWiPhx3fW0ExGz\nEXG5pPWSNtlu7BSM7WslHY+Ig3XXUtKVEfELkj4k6TPFVBf6lEPATkva0LK9XtKxmmoZOsV85r2S\n7oqI++qup4yI+JGkfZIacUOPNq6UdF0xt3m3pM22v15vSe1FxLHiv8cl7dT81Bz6lEPAHpB0ke0L\nbS+TdIOkB2quaSgUJ46+KulIRHyp7noWY3vM9rnF1++QtEXS8/VW1V5E3BoR6yNio+b/zH43Im6q\nuawF2V5RnOSU7RWSrpbU+FUwOWh8wEbEG5JulrRL8ydh7omIw/VW1Z7tb0h6VNK7bU/b/lTdNS3i\nSkmf0Hx3dah4XVN3UW2slbTX9tOa/0d3d0Q0eulTRlZLesT2U5Iel/RgRDxUc01DofHLtAAgV43v\nYAEgVwQsACRCwAJAIgQsACRCwAJAIgQsACRCwAJAIv8PQhObSvKQ8+UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1f6d01d9e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# vectorized Expected reward(s,a) \n",
    "vER = np.ndarray.flatten(ER)\n",
    "vER = vER.reshape((nstate*naction,1))\n",
    "\n",
    "# vectorized Q(s,a)\n",
    "matQ = np.zeros((nstate,naction)) # you can set matQ as random numbers\n",
    "\n",
    "vQ = np.zeros((nstate*naction,1))\n",
    "#print(vQ.shape)\n",
    "for _ in range(100):\n",
    "    for istate in range(nstate):\n",
    "        for iaction in range(naction):\n",
    "            tempQ = 0\n",
    "            for jstate in range(nstate):                \n",
    "                tempQ = tempQ + Tr[istate, iaction, jstate]*max(matQ[jstate,:])\n",
    "            matQ[istate,iaction] = ER[istate,iaction] + tempQ\n",
    "            \n",
    "    \n",
    "\n",
    "#print(vQ)\n",
    "print(matT[stateNum(4,3),:])\n",
    "\n",
    "for istate in range(nstate):\n",
    "    for iaction in range(naction):\n",
    "        V[istate] = sum(matQ[istate,:]*pi0[istate,:])\n",
    "\n",
    "plt.imshow(matQ, cmap=cm.Greys_r)\n",
    "plt.show()\n",
    "\n",
    "matV = np.reshape(V, (sizeY, sizeX))\n",
    "plt.imshow(matV, cmap=cm.Greys_r)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.25  0.25  0.25  0.25  0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.    0.\n",
      "  0.    0.    0.  ]\n"
     ]
    }
   ],
   "source": [
    "# vectorized Expected reward(s,a) \n",
    "vER = np.ndarray.flatten(ER)\n",
    "vER = vER.reshape((nstate*naction,1))\n",
    "\n",
    "# vectorized Q(s,a)\n",
    "vQ = np.zeros((nstate*naction,1))\n",
    "#print(vQ.shape)\n",
    "for _ in range(100000):\n",
    "    vQ = vER + np.dot(matT, vQ)\n",
    "\n",
    "#print(vQ)\n",
    "print(matT[stateNum(4,3),:])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEIAAAD8CAYAAADDlHLtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACUpJREFUeJztnX+MFGcZxz/PcbcQhJgiXCmU2GJI\nEE1EIUStf2CamIYYq4lKmyANMWJRiCYSUkqijX/5h9oY/sBgSiiJWvBHlZhWbSrEkGAtR45agjX8\nOCwChapQSC5Hlnv8Y79LNnc7s7Mz++PufD7J5mbnnXnn3c+9M++7O0+eMXcngJ5uN2CiECJEiBAh\nQoQIESJEiBAhQoSIKu6e+wU8BLwBnAaeyLC9d+F1NdNnKSBhGnAGWAyUgBPAsm6IMLPEF3Asy+cp\ncmqsAk67+1l3vwU8BzxcoL6uUkTEQuDNmvcXtG5S0ltgX6uzbtxXWTPbCGwscJyOUETEBWBRzft7\ngYtjN3L33cBuAJ2zE5Iip8arwBIzu9/MSsAjwMHWNKvz5O4R7l42s83AH6iMIHvc/WSj/czqnVFU\nR5Wm9qkybdq0xLJyudyoSZVjdPIXKg1pdcvaKGLA3Vc2alvMLEWIECFChAgRIkSIEEVmlrlIGupu\n377d9D5V+vr6EsuyziOiR4gQIUKECBEiRIgQITo6fJpZ6lCXRKN9pk+fnlg2PDyc6RjRI0SIECFC\nhAgRIkSIEB0dPnt6elKHuiRmzpyZWt7o22kWokeIECFChAgRIkSIECE6/u2zVCo1vV+j4XNkZCRv\nk+5QSISZDQE3gNtAOcvN1olKK3rEJ9397RbU01XiGiGKinDgj2Y2oFipcZjZRjM7ZmbHRkdHCx6u\nfRQ9NR5w94tm1g+8ZGZ/d/c/125QG0PV19c3JWOocPeL+nsFeJ5K7OWkJLcIM3uXmc2uLgOfAl5v\nVcM6TZFT427gecU39QI/c/ffp+1QLpe5cuVK0we6fv16rgY2Q5GourPAh1rYlq4Sw6cIESJEiBAh\nQoToeAxVnhDkRrQiFjt6hAgRIkSIECFChAgRIkKQRfQIESJEiBAhQoQIESJEhCCL6BEiRIgQIUKE\nCBEiRIgQIRrOI8xsD/Bp4Iq7f1Dr5gD7gfuAIeCL7v7fRnXljcWeN29eavmtW7earnMsWXrEXioZ\nyGp5AnjZ3ZcAL+v9pKahCEXA/GfM6oeBZ7X8LPDZFrer4+SdYt/t7pcA3P2SQofqUpuHqlGumG7S\n9u8atTFUvb29Uy6G6i0zuwdAf5sPg5lg5BVxEHhMy48Bv21Nc7pHluHz58BqYK6ZXQC+A3wPOGBm\nXwb+CXwhy8HSYrHTfqluNHzmicsaS0MR7v5oQtGDhY8+gYiZpQgRIkSIECFChOh4GseUsrT9UutN\n+wzuHmkcmyFEiBAhQoQIESJEiEkRgtyJdNDRI0SIECFChAgRIkSIECFCTIpY7J6e9P9XxGK3kBAh\nQoQIESJEiBBRJcMz+/ZQue3/es26p4B/AYN6rcnyvLwVK1Z4p6GFz/Lby/gYKoCn3X25Xi8U/5d0\nl7wxVFOOIteIzWb2mpntMbO7WtaiLpFXxC7gfcBy4BLwg6QNaxNyXb16Nefh2k8uEe7+lrvfdvdR\n4Cek5J9y993uvtLdVzaKfOkmuURUA8nE55jE+aeq5I2hWm1my6nkqhsCvprlYIODg8yZM6du2bp1\n6xL3O3LkSGq9165dy3L4VPLGUD1T+MgTjJhZihAhQoQIESJEiBAhOv5YiVmzZtUtO3r0aOJ+CxYs\naFhvEufOncvWtkxb/R8QIkSIECFChAgRIkTHY7GTAsPSAsYa3QROo1wuRyx2M4QIESJEiBAhQoQI\n0fEYqiSKDOMzZsxILLt582amOqJHiBAhQoQIESJEiBAhstwEXgTsA+YDo8Bud/9R3lxUeUKQ04ZH\ngA0bNiSW7dy5s1GTgGw9ogx8y93fD3wU+LqZLWOK5aLKEkN1yd2Pa/kGcApYyBTLRdXUNcLM7gM+\nDLzCmFxUQGIuqslA5im2mc0CfgV8093fyZpcqzYh10QmU48wsz4qEn7q7r/W6ky5qGpjqFrR4HbR\nUIRV/vXPAKfc/Yc1RVMqF1WWU+MB4EvA38xsUOueJGcuqolKlhiqI0DSBaGpXFT9/f2sXbu2blna\nPGL9+vWp9R44cKCZZtQlZpYiRIgQIUKECBEiRIiO3gQulUo+f/78umXbtm1L3O/w4cOp9Q4NDSWW\nDQwMxE3gZggRIkSIECFChAgRoqM3gfv6+ujvr/+L3qFDhxL3a/QrdtKQ3AzRI0SIECFChAgRIkSI\nEB0PQU4KJy4SZpyWh2p4eDi+fTZDiBAhQoQIESJEiBAhQhQJJnsK+ApQzaDzZJF8VGnzmUYPR77r\nruSkR1kfjpzl94hqMNlxM5sNDJjZSyp72t2/n+lIE5wsYQGXqKRYwt1vmFk1mGxKUSSYDDIk5arN\nQ1WopW0ms4ixwWRkTMo1ZWKooH4wWTNJuSYDuYPJplpSriLBZI/mScqV9HV7dHQ0cZ+kJF5Vtm/f\nnli2ZcuWLM0qFEw26XNY1hIzSxEiRIgQIUKECNHRm8BLly5l7969dcvSwoh37NiRWu+uXbuKNAuI\nHnGHECFChAgRIkSIECE6OnyeP3+eTZs21S1LGwK3bt2aWu/ly5cLtQuiR9whRIgQIUKECBEiRIgQ\nITo6jyiVSompnfft25e438jISGq9aTeBsxI9QoQIESJEiBAhQoQI0elY7KvA+ZpVc4G3W1B1Wj3v\ndfeGDwjrqIhxBzc71ooAklbUE6eGCBGi2yJ2T5R6unqNmEh0u0dMGNouwsweMrM3zOy0mY1L9Whm\n081sv8pfUSzn2G0WmdkhMztlZifN7Bt1tlltZtfNbFCvbzfV0CwPDs77AqYBZ4DFQAk4ASwbs83X\ngB9r+RFgf5167gE+ouXZwD/q1LMa+F3etra7R6wCTrv7WXe/BTxHJQ9mLbV5MX8JPGhjUiOm5NRs\nGe0WsRB4s+b9BcZ/gDvbuHsZuA68J6nCOmHQtXzMzE6Y2Ytm9oFmGtruX6jqhSWOHaaybFPZcHwY\ndC3HqUynb5rZGuA3wJKsDW13j7gALKp5fy9wMWkbM+sF3k2dx3Mn5NS8g7u/4+43tfwC0Gdmc7M2\ntN0iXgWWmNn9ZlaicjE8OGab2ryYnwf+5GMmNyk5NWu3mV+9tpjZKiqf7d+ZW9rOUUOfZw2Vq/wZ\nYIfWfRf4jJZnAL8ATgN/BRbXqeMTVE6X14BBvdYAjwOPa5vNwEkqI9NfgI83086YWYqYWYoQIUKE\nCBEiRIgQIUKECBHify0GomPFnSBeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1f6d0278780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV4AAAD8CAYAAAA/iMxLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAE6hJREFUeJzt3X/oXfd93/Hny5qklDSJXeTOtiRm\njSkFJzNJUOWA2Ugz/5BdY60bAxmWmHagtdgjZRuZPcG81RGUdmu7bl6Y1ogtNKswS7KIRK0j0WYh\nMMeSUseN7Dj94iSTag9NuE5rTBUkvffHPaI3yvd7v+fK3+8599zv8wEX3/s5597zljAvv/0553xO\nqgpJUneu6bsASVprDF5J6pjBK0kdM3glqWMGryR1zOCVpI4ZvJLWnCS7kryYZCHJI50f3+t4Ja0l\nSdYB3wLuBM4Ax4EHqur5rmqw45W01uwEFqrqpar6PnAI2N1lAX9lNX40iW20SNJ3CVO55prh9CFD\n+7u9cOHCuaq6/mq/v2vXrjp37lyrfU+ePHkK+IuxoQNVdWDs82bg9NjnM8BtV1vb1ViV4B2Sof0L\nvG7dur5LaG39+vV9lzCVjRs39l1Caxs2bOi7hKmcPXv2u2/m++fOnePEiROt9k3yF1W1Y9Iui4x1\n2iwO5z/xkrQyzgBbxz5vAV7usgCDV9IgVFWrVwvHge1JtiXZAOwBDq9q8VdY81MNkoZhpa7AqqoL\nSR4GngLWAQer6tSK/HhLBq+kNaeqjgBH+jq+wStpEObpngPneCWpY3a8kmbeFCfOBsHglTQI8xS8\nTjVIUsfseCUNgh2vJOmq2fFKGoR56ngNXkmDME/B61SDJHXM4JWkjjnVIGnmzdsNFHa8ktSxVsHb\n9xM5JWkF1+Pt3bLB2zyR8wngHuAW4IEkt6x2YZI0bk0FLzPwRE5JmidtTq61eiJnkr3A3hWqS5J+\nwFC62TbaBG+rJ3I2j08+AD7eXZImaRO8vT+RU5LmqeNtM8fb+xM5Ja1tbU+sDSWcl+14Z+GJnJI0\nT1rdudb3EzklaSjdbBveuSZJHXOtBkmDME8dr8EraRDmKXidapA087q6qiHJP0hyKsmlJDuu2PZo\ns17Ni0nuHhufei0bO15Jg9BRx/sN4O8B/3l8sFmfZg/wLuAm4FiSdzabnwDuZHTPw/Ekh6vq+UkH\nMXglqVFVLwAkP3TD7m7gUFWdB76dZIHROjbQrGXTfO/yWjYGr6Thm6Lj3ZTkxNjnA82SBm/GZuDp\nsc9nmjFosZbNlQxeSYMwRfCeq6odS21Mcgy4YZFN+6rqc0t9bbGSWPw82bKFGrySBmGl5nir6o6r\n+NqkNWumXsvGqxokzbwZWKvhMLAnycYk24DtwDNc5Vo2drySBqGLqxqS/AzwH4DrgS8kebaq7q6q\nU0meZHTS7ALwUFVdbL4z9Vo2Bq8kNarqs8Bnl9i2H9i/yPjUa9kYvJIGYZ7uXDN4JQ2CwStJHTN4\nJalDQ3q6RBsGr9RYt25d3yW0tnHjxr5L6JzBK0kdm6fg9QYKSeqYHa+kQZinjtfglTQIBq8kdcir\nGiSpBwavJHVsnoLXqxokqWN2vJIGYZ46XoNX0iAYvJLUIa9qkKQeGLyS1DGDV5I6ZvBKUoeqikuX\nLvVdxorxOl5J6pgdr6RBmKephmU73iQHk5xN8o0uCpKkxVy+pGy51xC0mWr4r8CuVa5Dkiaap+Bd\ndqqhqr6c5ObVL0WSljaUUG3Dk2uSZt7lqxravN6MJL+a5JtJnkvy2STXjm17NMlCkheT3D02vqsZ\nW0jySJvjrFjwJtmb5ESSEyv1m5J0WUdTDUeBd1fVrcC3gEcBktwC7AHexWjq9T8lWZdkHfAEcA9w\nC/BAs+9EKxa8VXWgqnZU1Y6V+k1JuqyL4K2qL1bVhebj08CW5v1u4FBVna+qbwMLwM7mtVBVL1XV\n94FDzb4TOdUgad5suvx/381r71X+zs8Bv9u83wycHtt2phlbanyiZU+uJfkd4AOM/jBngMeq6hOt\nypakFTJFN3tu0v95JzkG3LDIpn1V9blmn33ABeBTl7+2WEks3rwuW2ibqxoeWG4fSVpNK3mpWFXd\nMWl7kgeB+4C/U3950DPA1rHdtgAvN++XGl+SUw2SBqGjqxp2Af8CuL+q3hjbdBjYk2Rjkm3AduAZ\n4DiwPcm2JBsYnYA7vNxxvGVY0iB0dB3vfwQ2AkeTADxdVT9fVaeSPAk8z2gK4qGqugiQ5GHgKWAd\ncLCqTi13EINX0iB0EbxV9TcmbNsP7F9k/AhwZJrjGLySZt6QbgduwzleSeqYHa+kQZinhdANXkmD\nME9TDQavpEEweCWpQ/N2cs3glTQIBq8kdczglaSOGbyS1KHLT6CYFwavpEGw45Wkjhm8ktQxg7eF\nZkk1rWFvvPHG8jvNkNtuu63vElq7ePFi3yVM5fTp08vvNEFVDe7PPIkdr6RBsOOVpI4ZvJLUMYNX\nkjrkdbyS1AM7XknqmB2vJHXIqQZJ6oFTDZLUMYNXkjpm8EpSh+bt0T/X9F2AJLVx8eLFVq83I8nj\nSZ5L8mySLya5qRlPkt9MstBsf9/Ydx5M8sfN68E2xzF4JQ3C5a53udeb9KtVdWtVvQf4PPCvmvF7\ngO3Nay/wcYAkPwY8BtwG7AQeS3LdcgcxeCXNvLah+2aDt6r+bOzjW4HLP7gb+GSNPA1cm+RG4G7g\naFW9WlV/ChwFdi13HOd4JQ3CFNfxbkpyYuzzgao60PbLSfYDHwa+B/xUM7wZGF/b8kwzttT4RAav\npEGYops9V1U7ltqY5BhwwyKb9lXV56pqH7AvyaPAw4ymEhZbYLwmjE9k8EqaeSt551pV3dFy1/8O\nfIFR8J4Bto5t2wK83Ix/4IrxLy33w87xShqELuZ4k2wf+3g/8M3m/WHgw83VDe8HvldVrwBPAXcl\nua45qXZXMzbRsh1vkq3AJxm15pcYzZf8+6n+NJL0JnW0VsMvJ/kJRln3XeDnm/EjwL3AAvAG8LMA\nVfVqkseB481+v1RVry53kDZTDReAf1ZVX0vyNuBkkqNV9fxUfxxJukpd3UBRVX9/ifECHlpi20Hg\n4DTHWTZ4m3b6leb9nyd5gdFZO4NXUmfm6c61qU6uJbkZeC/w1UW27WV0YbEkrbg1GbxJfhT4NPCL\nV1xkDEBzndyBZt/5+RuS1Ls1+Xj3JOsZhe6nquozq1uSJP2wNdXxJgnwCeCFqvq11S9Jkn7YPAVv\nm+t4bwc+BHywWbHn2ST3rnJdkvQDOlokpxNtrmr4CovfFidJnRlKqLbhLcOSZt6Qutk2DF5Jg+BT\nhiWpY3a8ktShlVydbBYYvJIGwY5Xkjpm8EpSx5xqkKQOeTmZJPXA4JWkjhm8ktQxg1eSOmbwSlKH\n1uRC6JLUNzteSeqYwStJHfI63pZGTwyafddc0+YhHLPjpptu6ruE1j72sY/1XcJUtmzZ0ncJrZ0/\nf77vEjpn8EpSx+YpeIfV7klasy5dutTqtRKS/PMklWRT8zlJfjPJQpLnkrxvbN8Hk/xx83qwze/b\n8UqaeV3O8SbZCtwJ/J+x4XuA7c3rNuDjwG1Jfgx4DNgBFHAyyeGq+tNJx7DjlTQIHT5l+NeBjzIK\n0st2A5+skaeBa5PcCNwNHK2qV5uwPQrsWu4AdrySBmGKaYRNSU6MfT5QVQfafDHJ/cCfVNXXr7hA\nYDNweuzzmWZsqfGJDF5JgzBFN3uuqnYstTHJMeCGRTbtA/4lcNdiX1uspAnjExm8kmbeSj5zraru\nWGw8yd8EtgGXu90twNeS7GTUyW4d230L8HIz/oErxr+0XA3O8UoahNWe462qP6qqH6+qm6vqZkah\n+r6q+r/AYeDDzdUN7we+V1WvAE8BdyW5Lsl1jLrlp5Y7lh2vpEHo+TreI8C9wALwBvCzTU2vJnkc\nON7s90tV9epyP2bwShqEroO36Xovvy/goSX2OwgcnOa3DV5JM8+1GiSpBwavJHXMx7tLUsfseCWp\nQ87xSlIPDF5J6tiaCt4kbwG+DGxs9v8fVfXYahcmSePW2sm188AHq+r1JOuBryT53WZpNEladWtu\njre5Y+P15uP65jU/fwOSBmGegrfVIjlJ1iV5FjjLaNHfr65uWZL0gzpcCH3VtQreqrpYVe9htOTZ\nziTvvnKfJHuTnLhiAWJJWhFrLngvq6rXGK01+UOPtqiqA1W1Y9ICxJJ0tdZU8Ca5Psm1zfsfAe4A\nvrnahUnSZZcXQu/qKcOrrc1VDTcC/y3JOkZB/WRVfX51y5KkHzSUbraNNlc1PAe8t4NaJGlJayp4\nJalvK/nMtVlg8EoaBDteSeqYwStJHXOqQZI6NKRrdNsweCUNgsErSR0zeCWpYwavJHWoqrh48WLf\nZayYqRbJkaS+dLFWQ5J/neRPkjzbvO4d2/ZokoUkLya5e2x8VzO2kOSRNsex45U08zq+quHXq+rf\njg8kuQXYA7wLuAk4luSdzeYngDuBM8DxJIer6vlJBzB4JQ1Cz9fx7gYOVdV54NtJFoCdzbaFqnoJ\nIMmhZt+JwetUg6RBmGI93k2XH8rQvPZOeaiHkzyX5GCS65qxzcDpsX3ONGNLjU9kxytpEKaYajg3\n6YEMSY4BNyyyaR/wceBxRs+VfBz4d8DPAVmsJBZvXpct1OCVNPNW8qqGqrqjzX5J/gtwee3xM8DW\nsc1bgJeb90uNL8mpBkmD0NFVDTeOffwZ4BvN+8PAniQbk2wDtgPPAMeB7Um2JdnA6ATc4eWOs2od\n7zXXDCPT169f33cJU3nHO97Rdwmt3XrrrX2XMJVXXnml7xJae+211/ouoVMdXtXwK0new2i64DvA\nP26OfyrJk4xOml0AHqqqiwBJHgaeAtYBB6vq1HIHcapB0iB0EbxV9aEJ2/YD+xcZPwIcmeY4Bq+k\nQXBZSEnqmGs1SFKH5m2tBoNX0iDY8UpSh3zKsCT1wI5XkjpmxytJHfLkmiT1wKkGSeqYwStJHfKq\nBknqgR2vJHXMjleSOuRUgyT1wOCVpI45xytJHZun4G39fJ4k65L8YZLPL7+3JK2cto92H0o4T9Px\nfgR4AXj7KtUiSUsaSqi20arjTbIF+Gngt1a3HEla3MWLF1u9hqBtx/sbwEeBty21Q5K9wN6VKEqS\nxg1pGqGNZTveJPcBZ6vq5KT9qupAVe2oqh0rVp0kNdbaHO/twP1J7gXeArw9yW9X1T9c3dIk6S8N\nJVTbWLbjrapHq2pLVd0M7AF+39CV1LW11vFKUq/mbSH01tfxAlTVl6rqvtUqRpKW0lXHm+SfJHkx\nyakkvzI2/miShWbb3WPju5qxhSSPtDmGHa+kQehiGiHJTwG7gVur6nySH2/Gb2E01fou4CbgWJJ3\nNl97ArgTOAMcT3K4qp6fdByDV9IgdDR/+wvAL1fV+eaYZ5vx3cChZvzbSRaAnc22hap6CSDJoWbf\nicE71VSDJPVhyluGNyU5Mfaa5v6CdwJ/K8lXk/yvJD/ZjG8GTo/td6YZW2p8IjteSYMwxbKQ5ybd\nT5DkGHDDIpv2McrE64D3Az8JPJnkrwNZZP9i8eZ12dbc4JU0CCu1Hm9V3bHUtiS/AHymRq3zM0ku\nAZsYdbJbx3bdArzcvF9qfElONUiaeR2uTvY/gQ8CNCfPNgDngMPAniQbk2wDtgPPAMeB7Um2JdnA\n6ATc4eUOYscraRA6Orl2EDiY5BvA94EHm+73VJInGZ00uwA8VFUXAZI8DDwFrAMOVtWp5Q5i8Eoa\nhC6Ct6q+Dyx6Z25V7Qf2LzJ+BDgyzXEMXkmD4DPXJKlDQ1qHoQ2DV9Ig2PFKUsfseCWpYwavJHWo\nqpxqkKSu2fFKUsfmqePNavxXJMn/A767wj+7idGte0MxpHqHVCsMq94h1QqrV+9fq6rrr/bLSX6P\nUW1tnKuqXVd7rC6sSvCuhiQnhvQE4yHVO6RaYVj1DqlWGF69Q+UiOZLUMYNXkjo2pOA90HcBUxpS\nvUOqFYZV75BqheHVO0iDmeOVpHkxpI5XkuaCwStJHRtE8CbZleTFJAtJHum7nkmSHExytlnBfqYl\n2ZrkD5K8kORUko/0XdNSkrwlyTNJvt7U+m/6rqmNJOuS/GGSz/ddyyRJvpPkj5I8m+RE3/XMu5mf\n402yDvgWcCejB84dBx6oqonPre9Lkr8NvA58sqre3Xc9kyS5Ebixqr6W5G3ASeDvzuLfbZIAb62q\n15OsB74CfKSqnu65tImS/FNgB/D2qrqv73qWkuQ7wI6qGtLNHoM1hI53J7BQVS81j+U4BOzuuaYl\nVdWXgVf7rqONqnqlqr7WvP9z4AVgc79VLa5GXm8+rm9eM901JNkC/DTwW33XotkyhODdDJwe+3yG\nGQ2HIUtyM/Be4Kv9VrK05n/bnwXOAkeramZrbfwG8FFgCIsMFPDFJCeT7O27mHk3hODNImMz3ekM\nTZIfBT4N/GJV/Vnf9Sylqi5W1XuALcDOJDM7lZPkPuBsVZ3su5aWbq+q9wH3AA81U2ZaJUMI3jPA\n1rHPW4CXe6pl7jTzpZ8GPlVVn+m7njaq6jXgS8AsL4RyO3B/M3d6CPhgkt/ut6SlVdXLzT/PAp9l\nNMWnVTKE4D0ObE+yLckGYA9wuOea5kJzwuoTwAtV9Wt91zNJkuuTXNu8/xHgDuCb/Va1tKp6tKq2\nVNXNjP6d/f2qWvSx4X1L8tbm5CpJ3grcBcz8VTlDNvPBW1UXgIeBpxid/Hmyqk71W9XSkvwO8L+B\nn0hyJsk/6rumCW4HPsSoG3u2ed3bd1FLuBH4gyTPMfqP8dGqmulLtAbkrwJfSfJ14BngC1X1ez3X\nNNdm/nIySZo3M9/xStK8MXglqWMGryR1zOCVpI4ZvJLUMYNXkjpm8EpSx/4/miFzZ+XDQ/AAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1f6d019f7b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "\n",
    "matQ =  np.reshape(vQ , (nstate, naction))\n",
    "V = np.zeros(nstate)\n",
    "for istate in range(nstate):\n",
    "    for iaction in range(naction):\n",
    "        V[istate] = sum(matQ[istate,:]*pi0[istate,:])\n",
    "\n",
    "plt.imshow(matQ, cmap=cm.Greys_r)\n",
    "plt.show()\n",
    "\n",
    "matV = np.reshape(V, (sizeY, sizeX))\n",
    "plt.imshow(matV, cmap=cm.Greys_r)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-642.10754835]\n",
      " [-642.10754835]\n",
      " [-641.36081911]\n",
      " [-638.85427759]\n",
      " [-638.85427759]\n",
      " [-642.10754835]\n",
      " [-638.71691393]\n",
      " [-631.73837049]\n",
      " [-631.73837049]\n",
      " [-638.85427759]\n",
      " [-631.73837049]\n",
      " [-620.62246339]\n",
      " [-620.62246339]\n",
      " [-631.73837049]\n",
      " [-620.62246339]\n",
      " [-605.5065563 ]\n",
      " [-605.5065563 ]\n",
      " [-620.62246339]\n",
      " [-598.58778337]\n",
      " [-593.30942213]\n",
      " [-593.30942213]\n",
      " [-605.5065563 ]\n",
      " [-577.11228796]\n",
      " [-593.30942213]\n",
      " [-642.10754835]\n",
      " [-641.36081911]\n",
      " [-639.25799506]\n",
      " [-638.71691393]\n",
      " [-638.85427759]\n",
      " [-641.85863861]\n",
      " [-638.41636903]\n",
      " [-631.73837049]\n",
      " [-631.73837049]\n",
      " [-638.85427759]\n",
      " [-631.73837049]\n",
      " [-620.62246339]\n",
      " [-620.62246339]\n",
      " [-631.73837049]\n",
      " [-620.62246339]\n",
      " [-605.5065563 ]\n",
      " [-605.5065563 ]\n",
      " [-620.62246339]\n",
      " [-576.31173637]\n",
      " [-587.9103774 ]\n",
      " [-593.30942213]\n",
      " [-598.58778337]\n",
      " [-535.43965837]\n",
      " [-577.11228796]\n",
      " [-641.36081911]\n",
      " [-639.25799506]\n",
      " [-634.59788683]\n",
      " [-637.81527923]\n",
      " [-638.80848971]\n",
      " [-640.90878751]\n",
      " [-635.80963174]\n",
      " [-631.73420796]\n",
      " [-631.73837049]\n",
      " [-638.80848971]\n",
      " [-631.73420796]\n",
      " [-620.62246339]\n",
      " [   0.        ]\n",
      " [   0.        ]\n",
      " [   0.        ]\n",
      " [   0.        ]\n",
      " [-603.20029865]\n",
      " [-414.08164226]\n",
      " [-537.13617248]\n",
      " [-568.62045615]\n",
      " [-577.11228796]\n",
      " [-531.75964239]\n",
      " [-493.44704478]\n",
      " [-535.43965837]\n",
      " [-639.25799506]\n",
      " [-634.59788683]\n",
      " [-629.63896338]\n",
      " [-630.89670206]\n",
      " [-638.46215692]\n",
      " [-638.405567  ]\n",
      " [-629.79734041]\n",
      " [-612.92174393]\n",
      " [-631.73837049]\n",
      " [-638.46215692]\n",
      " [-612.92174393]\n",
      " [-414.08164226]\n",
      " [-414.08164226]\n",
      " [-421.492247  ]\n",
      " [-207.54082113]\n",
      " [-401.69811322]\n",
      " [-578.61799402]\n",
      " [-327.94188977]\n",
      " [-478.35148595]\n",
      " [-535.33299704]\n",
      " [-535.43965837]\n",
      " [-481.06109169]\n",
      " [-459.84038427]\n",
      " [-493.44704478]\n",
      " [-634.59788683]\n",
      " [-629.63896338]\n",
      " [-629.63896338]\n",
      " [-620.68003993]\n",
      " [-635.80963174]\n",
      " [-634.49828176]\n",
      " [-624.08559397]\n",
      " [-584.32665224]\n",
      " [-631.73420796]\n",
      " [-635.80963174]\n",
      " [-584.32665224]\n",
      " [-327.94188977]\n",
      " [-207.54082113]\n",
      " [-402.67978296]\n",
      " [ -89.31244507]\n",
      " [-360.21629169]\n",
      " [-537.13617248]\n",
      " [-209.71351371]\n",
      " [-441.84284641]\n",
      " [-496.24236248]\n",
      " [-493.44704478]\n",
      " [-422.23372377]\n",
      " [-459.84038427]\n",
      " [-459.84038427]]\n"
     ]
    }
   ],
   "source": [
    "print(vQ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "matQ = np.reshape(vQ , (nstate, naction))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "pi1 = np.zeros((nstate, naction), dtype=np.float)\n",
    "\n",
    "matQ = np.reshape(vQ , (nstate, naction))\n",
    "\n",
    "\n",
    "for istate in range(nstate):\n",
    "    nQ = sum(matQ[istate,:]==max(matQ[istate,:]))\n",
    "    pi1[istate,matQ[istate,:]==max(matQ[istate,:])] = 1/nQ\n",
    "    \n",
    "matT = np.zeros((nstate*naction, nstate*naction), dtype=np.float)\n",
    "for istate in range(nstate):\n",
    "    for iaction in range(naction):\n",
    "        for jstate in range(nstate):\n",
    "            for jaction in range(naction):\n",
    "                matT[istate*naction +iaction, jstate*naction+jaction] = Tr[istate, iaction, jstate] * pi1[jstate, jaction]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "vER = np.ndarray.flatten(ER)\n",
    "vER = vER.reshape((nstate*naction,1))\n",
    "\n",
    "vQ = np.zeros((nstate*naction,1))\n",
    "for _ in range(1000):\n",
    "    vQ = vER + np.dot(matT, vQ)\n",
    "\n",
    "#print(vQ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEIAAAD8CAYAAADDlHLtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACZBJREFUeJztnX2IVWkdxz8/Z3xBR9N8mRVXUmME\ntyBTEWsTJ5ZEJFqFCvePXESypYQC/4kNagmC/aNa9w8prCSFau3NknCtZRMipM0Xxp31pVBxW19Q\nR8N1YWQZ59cf9zvDbbzn3HPPuffOneH3gcuce57zPOe5n3nuec6593d/x9ydACaMdgdahRAhQoQI\nESJEiBAhQoQIESJEiPYilc1sA/Ay0Ab81N1frLJ9Q05j29uTX8bAwECfu8+t2kbenZtZG7AH+Axw\nFThhZofd/VzeNvMyc+bMxLK+vr63s7RR5K2xGrjo7pfd/X3gFeDpAu2NKkVELADeKXt+VevGJEWO\nEVZh3SPHADPbAewosJ+mUETEVWBh2fPHgesjN3L3vcBeaNzBsh4UeWucALrMbLGZTQK2AIfr063m\nk3tEuPuAme0E/kxp+tzn7mdTd9benniE7+vrS6w3Z86c1L7MmzcvsSyt3f/rW6atEnD3I8CRIm20\nCnFmKUKECBEiRIgQIUKEKDR91ryz9vbUOT+JanUWLEi+xDl3LtvFcIwIESJEiBAhQoQIESJEU6fP\niRMnpk51SVSrk2dKHkmMCBEiRIgQIUKECBEiRNOnzzxT3d27d1PLly1blrdLw8SIECFChAgRIkSI\nECFCNP3D29mzZ9dcb+3atanlWb/oTaNoMNkV4D7wEBhw91WFezRK1GNEfNrdi/9LRpk4RoiiIhz4\ni5mdUqzUI5jZDjM7aWYn+/v7C+6ucRR9azzp7tfNbB7wmpldcPe/lW9QHkPV2dk5LmOocPfr+nsL\nOEQp9nJMkluEmU0zs+lDy8B64K16dazZWN5f+ZnZEkqjAEpvsV+6+/eq1Mm1s1mzZqWWt7W1JZb1\n9fWdyjKtF4mquwx8LG/9ViOmTxEiRIgQIUKECNH0y/A8sdhp0yPUJxY7RoQIESJEiBAhQoQIESJC\nkEWMCBEiRIgQIUKECBEiRIQgixgRIkSIECFChAgRIkSIECGqnkeY2T7gs8Atd/+o1n0QOAgsAq4A\nX3T3/1ZrK28s9uTJk1PL58+fX3ObI8kyIn4ObBix7pvA6+7eBbyu52OaqiIUATMyPP5pYL+W9wOb\n6tyvppP3GNHp7jcA9DdxvJfHUD148CDn7hpPww+W7r7X3Ve5+6opU6Y0ene5ySvippnNB9DfW/Xr\n0uiQV8Rh4FktPwv8sT7dGT2yTJ+/ArqBOWZ2FfgO8CLwazPbDvwH+EKmnaXEYl+4cCGx3vbt21Pb\nbUostrs/k1D0VOG9txBxZilChAgRIkSIECFyhyDn2llKCHJamPHcuenZnNN+KZw1BDlGhAgRIkSI\nECFChAgRomVCkO/du5dYr6OjI7XdCEGuIyFChAgRIkSIECFChAgxJmKxp0+fnloesdh1JESIECFC\nhAgRIkQM4e6pD2Afpa/93ypb9wJwDejRY2O1dtydlStXerMBTmbpW94YKoCX3H25HmP+zkx5Y6jG\nHUWOETvN7E0z22dm6UlgxgB5RfwI+DCwHLgB/CBpw/Jgstu3b+fcXePJJcLdb7r7Q3cfBH5CSv4p\nLwsmq/bV3WiSS8RQIJnYzBjOPzVE3hiqbjNbTilX3RXgK1l2dunSJTZv3lyxbM2aNYn1rl27ltpu\nPUZa3hiqnxXec4sRZ5YiRIgQIUKECBEiRIimfoo9depUVqxYUbEs7VvrdevWpbZ761bxHwfEiBAh\nQoQIESJEiBAhQjQ1Fru9vd1nzJhRsSztUrpaRpG0qbe3tzdisWshRIgQIUKECBEiRIimXn2aWWJq\n57QpsEgMVW9vb6a+xYgQIUKECBEiRIgQIUJk+RJ4IXAAeAwYBPa6+8t5clGlhSCnfQBb7Uve7u7u\nxLKjR4+m1h0iy4gYAHa5+zJgDfA1M3uCcZaLKksM1Q13P63l+8B5YAHjLBdVTccIM1sEfBx4gxpy\nUY0FMoswsw7gd8A33P3dGuoNx1A9fPgwTx+bQiYRZjaRkoRfuPvvtTpTLqryGKpqt5AZTaqKMDOj\nFCFz3t1/WFY0rnJRZbn6fBL4EtBrZj1a9zw5c1G1KlliqP4OWEJxTbmoZs+ezdatWyuW3b9/P7He\npk3pE9KxY8dq6UZF4sxShAgRIkSIECFChAjR1E+x+/v7OXPmTMWyLVu2JNar9mveahlHshAjQoQI\nESJEiBAhQoQI0dTps6Ojg7Vr11YsS/sUe+nSpant1uPXgzEiRIgQIUKECBEiRIgQTZ0+b968ye7d\nuyuW5bnZ4RCdnZ256w4RI0KECBEiRIgQIUKECBEiRJFgsheALwND18DPV8tHNTg4SH9/f8WytKwh\nXV1dqX1MutNTLWQ5oRoKJjttZtOBU2b2mspecvfvF+5FC5AlLOAGpRRLuPt9MxsKJhtXFAkmgwxJ\nucpjqAYHBwt1tpEUCSbLlJSrPIZqwoTWPTbnDiarJSnXWCB3MNl4S8pVJJjsmVqTcrW1tSX+LCkt\nhmrRokWp7a5fvz6xbM+ePdW6BRQLJhvzOSzLad2jV5MJESJEiBAhQoRo6qfYixcvZv/+/RXLjh8/\nnlhv27Ztqe0eOnSoUL8gRsQwIUKECBEiRIgQIUI0dfq8c+cOBw4cqFi2a9euxHo9PT2JZQDTpk0r\n1C+IETFMiBAhQoQIESJEiBAhQjQ9FjsptXPazZGr5dysx20lYkSIECFChAgRIkSIECGamg7azG4D\nb5etmgMkJ7LMTlo7H3L3qvNrU0U8snOzk1lyVjejnXhriBAhRlvE3lZpZ1SPEa3EaI+IlqHhIsxs\ng5n9y8wumtkjqR7NbLKZHVT5G4rlHLnNQjM7ZmbnzeysmX29wjbdZnbPzHr0+HZNHc1y4+C8D6AN\nuAQsASYBZ4AnRmzzVeDHWt4CHKzQznxghZanA/+u0E438Ke8fW30iFgNXHT3y+7+PvAKpTyY5ZTn\nxfwt8JRCGodJyalZNxotYgHwTtnzqzz6Aoa3cfcB4B6QGG5fIQy6nE+Y2Rkze9XMPlJLRxv9CVWl\nsMSR01SWbUobpufUPE3pdPo9M9sI/AFI/31DGY0eEVeBhWXPHweuJ21jZu3AB6hwe+6EnJrDuPu7\n7v6elo8AE81sTtaONlrECaDLzBab2SRKB8PDI7Ypz4v5eeCvPuLkJiWnZvk2jw0dW8xsNaXXdidz\nTxs5a+j1bKR0lL8EfEvrvgt8TstTgN8AF4F/AksqtPEpSm+XN4EePTYCzwHPaZudwFlKM9M/gE/W\n0s84sxRxZilChAgRIkSIECFChAgRIkSI/wGtk7+3q3xGcgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1f6d047b668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAD8CAYAAAAylrwMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEMVJREFUeJzt3X2MXFd9xvHnYf2GgCRFG780azWp\ngEpOiqBd3JeohQYHDLikUKgSCYrUqlYRVKEvoqRWQUggtaSioLZ/sCJRi0CkaUMKSoAQC1KUKm/r\nJEBsE5pSKK4pZoUiiCoSxX76x47pxuzs3tmZc+896+9HGmXv3JkzPzvW45/PnHuPkwgAMHlP67oA\nAFivCFgAKISABYBCCFgAKISABYBCCFgAKISABXDWsb3X9sO2H7H9jmKfwzpYAGcT21OSvibpcknH\nJN0n6aokRyb9WXSwAM42uyU9kuTrSZ6QdIOkK0p80IYSg9qupi3esKHIb0ExGzdu7LqExjZt2tR1\nCSOp6c9CTbVK0ne+852FJOev9f179+7NwsJCo9ceOnTosKQfLnlqLsnckuMLJH1ryfExSb+w1tpW\nUtf/pQKe/exnd13CSHbs2NF1CY3NzMx0XcJItm7d2nUJjU1PT3ddwkiuvfbab47z/oWFBc3Pzzd6\nre0fJpld6SXLPFekKWSKAMDZ5piknUuOZyQdL/FBBCyAKiRp9GjgPknPtX2R7U2SrpT0qRI1n/VT\nBADqMKkVT0metP1WSbdJmpJ0fZLDExn8DAQsgLNOkk9L+nTpzyFgAVShxjX7zMECQCF0sAB6b4Qv\nsHqFgAVQhRoDlikCACiEDhZAFehgAQA/QgcLoAo1drAELIAq1BiwTBEAQCEELAAUwhQBgN6r9UID\nOlgAKKRRwLa1AyMADDPB+8G2ZtWAHezA+HeSXiFpl6SrbO8qXRgALLUuA1Yt7sAIAOtJky+5Gu3A\naHu/pP0TqgsAnqJv3WkTTQK20Q6Mg21x56S6tu0GgFKaBGxrOzACwDA1drBN5mBb24ERAJbT9Auu\nvoXwqh1smzswAsB60uhKrrZ2YASAYfrWnTbBlVwAUAj3IgBQhRo7WAIWQBVqDFimCAD0XlurCGxf\na/urtr9s+2bb540zHgELoAotLdO6XdIlSZ4v6WuSrhlnMAIWAAaSfC7Jk4PDu7V4YdWaMQcLoAoj\ndKfTtueXHM8NLuUf1e9I+sc1vO9HCFgAVRghYBeSzA47afugpO3LnDqQ5JOD1xyQ9KSkj41a51IE\nLIAqTGoVQZI9K523/SZJ+yS9NGN+KAELoPfaus+A7b2S/lTSi5P877jjEbAAqtDSOti/lbRZ0u22\nJenuJL+/1sEIWAAYSPKcSY5HwAKoQo1XchGwAKpAwAJAIQQsABTQx90KmiBggYEjR450XUJjr3vd\n67ouoXUELAAUUmPAcrMXACiEDhZAFWrsYAlYAFUgYAGgAFYRAEBBBCwAFFJjwLKKAAAKoYMFUIUa\nO1gCFkAVCFgAKIBVBABQEAELAIUQsABQCAELAAUk0alTp7ouY2SsgwWAQuhgAVShximCVTtY29fb\nPmH7oTYKAoDlnF6qtdqjT5pMEfy9pL2F6wCAFa3LgE3yRUnfa6EWABiqzYC1/Se2Y3t6nHGYgwXQ\ne22uIrC9U9Llkv5r3LEmtorA9n7b87bnJzUmAJzWYgf715LeLmnswSbWwSaZkzQnSbb7NRECoHpt\nzK/afrWk/07yJdtjj8cUAYD1ZvqMf0nPDRpASZLtg5K2L/O+A5L+TNLLJlXIqgFr++OSXqLFoo9J\neleS6yZVAAA0MUIHu5BkdoVx9iz3vO2flXSRpNPd64yk+23vTvI/I5YrqUHAJrlqLQMDwKS0sQQr\nyVckbT19bPsbkmaTLKx1TKYIAFShxnsRELAAqtD2RQRJLhx3DAIWQBX6dpVWEwQsgN7r42WwTXC7\nQgAohA4WQBX4kgsACqlxioCABVAFAhYACqj1Sy4CFkAVCFgAKISABYBCCFgAKKDNHQ0miYAFUAU6\nWAAohIAFgEII2IGpqSmde+65JYZGRW655ZauSxjJO9/5zq5LaGzbtm1dl9CqJDp58mTXZYyMDhZA\nFehgAaAQAhYACiFgAaAA1sECQEF0sABQCB0sABTAFAEAFMQUAQAUQsACQCE1BizbdgPovdNbxjR5\njMv2H9h+2PZh2+8bZyw6WABVaONeBLZ/TdIVkp6f5HHbW8cZj4AFUIWWpgjeLOkvkjw++MwT4wzG\nFAGA3htximDa9vySx/4RPup5kn7F9j22/9X2i8apmw4WQBVGWAe7kGR22EnbByVtX+bUAS1m4k9I\n+kVJL5J0o+2fzhrbZwIWQBUmNUWQZM+wc7bfLOkTg0C91/YpSdOSvruWz2KKAEDvnb6Sq8ljTP8i\n6TJJsv08SZskLax1MDpYAFVo6Uuu6yVdb/shSU9IetNapwekBgFre6ekj2hxzuKUpLkkH1zrBwLA\nWrRxL4IkT0h6w6TGa9LBPinpj5Pcb/tZkg7Zvj3JkUkVAQArmdRFBG1bNWCTfFvStwc//8D2UUkX\nSCJgAbRmXQbsUrYvlPRCSfcsc26/pP2S9LSn8d0ZgMla1wFr+5mSbpL0tiTfP/N8kjlJc5K0YcOG\n+n4nAPTWut622/ZGLYbrx5J8omxJAPDj1mUHa9uSrpN0NMn7y5cEAD+uxoBtMll6qaQ3SrrM9oOD\nxysL1wUAT9HW7QonqckqgjsluYVaAGCovoVnE1zJBaD3+tidNkHAAqgCu8oCQCF0sABQwOm7adWG\ngAVQBTpYACiEgAWAQpgiAIACWKYFAAURsABQCAELAIUQsABQCAELAAWs6xtuA0DX6GABoBACFgAK\nYB3sEhs3btT27dtLDD1xmzdv7rqEkezbt6/rEhq76667ui5hJLt27eq6hMamp6e7LqF1NQYs+2sD\nqEIbW8bYfoHtuwdbY83b3j3OeEwRAKhCS/cieJ+kdyf5zGDvwfdJeslaByNgAfRei3OwkXTO4Odz\nJR0fZzACFkAVRgjYadvzS47nksw1fO/bJN1m+6+0OIX6yyOU+GMIWABVGGGKYCHJ7LCTtg9KWu5b\n+AOSXirpD5PcZPu3JF0nac+otZ5GwAKowqSmCJIMDUzbH5F09eDwnyR9eJzPYhUBgN47vSdXk8eY\njkt68eDnyyT9+ziD0cECqEJLX3L9nqQP2t4g6YeS9o8zGAELoAptBGySOyX9/KTGI2ABVKHGK7kI\nWAC9x70IAKAgAhYACmHbbgAohA4WAApgDhYACiJgAaCQdRmwtrdI+qKkzYPX/3OSd5UuDACWWq9f\ncj0u6bIkj9neKOlO259Jcnfh2gBA0jqeg83ir+qxweHGwaO+XymAqtUYsI3upmV7yvaDkk5Iuj3J\nPWXLAoCnamNPrklrFLBJTiZ5gaQZSbttX3Lma2zvH2wSNn/y5MlJ1wngLLduA/a0JI9KukPS3mXO\nzSWZTTI7NTU1ofIAYNG6DFjb59s+b/Dz07W4fcJXSxcGAKe1eMPtiWqyimCHpH+wPaXFQL4xyS1l\nywKAp+pbd9pEk1UEX5b0whZqAYCh1mXAAkDXTk8R1IaABVAFOlgAKISABYBCmCIAgAL6uMa1CQIW\nQBUIWAAopMaAHelSWQDoShuXytp+ve3Dtk/Znj3j3DW2H7H9sO2XNxmPDhZA7yVRSzeRekjSayV9\naOmTtndJulLSxZJ+UtJB289LsmJRBCyAKrSxiiDJUUmyfeapKyTdkORxSf9p+xFJuyXdtdJ4BCyA\n3hvxn//TtueXHM8lmRuzhAskLd3F5djguRURsACqMEIHu5BkdthJ2wclbV/m1IEknxz2tmWeWzXx\nCVgAVZjUKoIke9bwtmOSdi45npF0fLU3sYoAQBU6vuH2pyRdaXuz7YskPVfSvau9iQ4WQO+1tYrA\n9msk/Y2k8yXdavvBJC9Pctj2jZKOSHpS0ltWW0EgEbAAKtHSKoKbJd085Nx7Jb13lPGKBOymTZu0\nc+fO1V/YA1u3bu26hJHs2rWr6xIau/jii7suYSTbtm3ruoTGHn300a5LaBX3IgCAgghYACiE2xUC\nQCF0sABQQIv3IpgoAhZAFehgAaAAdpUFgILoYAGgEDpYACiAL7kAoCCmCACgEAIWAApgFQEAFEQH\nCwCF0MECQAFMEQBAQQQsABTCHCwAFFJjwDbeVdb2lO0HbN9SsiAAOFPTHWX7FsKjdLBXSzoq6ZxC\ntQDAUH0LzyYadbC2ZyS9StKHy5YDAMs7efJko0efNO1gPyDp7ZKeNewFtvdL2i9JW7ZsGb8yABjo\n4z//m1i1g7W9T9KJJIdWel2SuSSzSWY3bdo0sQIBQGo+D9snTTrYSyW92vYrJW2RdI7tjyZ5Q9nS\nAOD/9S08m1i1g01yTZKZJBdKulLS5wlXAG1ro4O1/Xrbh22fsj275PnLbR+y/ZXBfy9rMh7rYAH0\nXos33H5I0mslfeiM5xck/XqS47YvkXSbpAtWG2ykgE1yh6Q7RnkPAExCG1MESY5Kku0zn39gyeFh\nSVtsb07y+Erj0cECqMIIATtte37J8VySuQmW8puSHlgtXCUCFkAlRgjYhSSzw07aPihp+zKnDiT5\n5EoD275Y0l9KelmTQghYAL03ySVYSfas5X2DC65ulvTbSf6jyXsIWABV6PJ2hbbPk3SrpGuS/FvT\n9zW+2QsAdOnUqVONHuOw/RrbxyT9kqRbbd82OPVWSc+R9Oe2Hxw8tq42Hh0sgN5r6yqtJDdrcRrg\nzOffI+k9o45HwAKoQo1XchGwAKpAwAJAIezJBQAF9PFOWU0QsACqQAcLAIXQwQJAIQQsABSQhCkC\nACiFDhYACqmxg3WJvxVsf1fSNyc87LQW7ypei5rqralWqa56a6pVKlfvTyU5f61vtv1ZLdbWxEKS\nvWv9rEkqErAl2J5f6R6PfVNTvTXVKtVVb021SvXV23fcTQsACiFgAaCQmgJ2knvqtKGmemuqVaqr\n3ppqleqrt9eqmYMFgNrU1MECQFUIWAAopIqAtb3X9sO2H7H9jq7rWYnt622fsP1Q17WsxvZO21+w\nfdT2YdtXd13TMLa32L7X9pcGtb6765qasD1l+wHbt3Rdy0psf8P2VwZ7Tc13Xc960fs5WNtTkr4m\n6XJJxyTdJ+mqJEc6LWwI278q6TFJH0lySdf1rMT2Dkk7ktxv+1mSDkn6jT7+3tq2pGckecz2Rkl3\nSro6yd0dl7Yi238kaVbSOUn2dV3PMLa/IWk2SU0XRfReDR3sbkmPJPl6kick3SDpio5rGirJFyV9\nr+s6mkjy7ST3D37+gaSjki7otqrlZdFjg8ONg0evuwPbM5JeJenDXdeCbtQQsBdI+taS42PqaQjU\nzPaFkl4o6Z5uKxlu8M/tByWdkHR7kt7WOvABSW+XVMNF9JH0OduHbO/vupj1ooaA9TLP9bpzqY3t\nZ0q6SdLbkny/63qGSXIyyQskzUjabbu3UzC290k6keRQ17U0dGmSn5P0CklvGUx1YUw1BOwxSTuX\nHM9IOt5RLevOYD7zJkkfS/KJrutpIsmjku6Q1IsbegxxqaRXD+Y2b5B0me2PdlvScEmOD/57QtLN\nWpyaw5hqCNj7JD3X9kW2N0m6UtKnOq5pXRh8cXSdpKNJ3t91PSuxfb7t8wY/P13SHklf7baq4ZJc\nk2QmyYVa/DP7+SRv6LisZdl+xuBLTtl+hqSXSer9Kpga9D5gkzwp6a2SbtPilzA3JjncbVXD2f64\npLsk/YztY7Z/t+uaVnCppDdqsbt6cPB4ZddFDbFD0hdsf1mLf+nenqTXS58qsk3Snba/JOleSbcm\n+WzHNa0LvV+mBQC16n0HCwC1ImABoBACFgAKIWABoBACFgAKIWABoBACFgAK+T+C72niHMS+5wAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1f6d04a5b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "matQ =  np.reshape(vQ , (nstate, naction))\n",
    "V = np.zeros(nstate)\n",
    "for istate in range(nstate):\n",
    "    for iaction in range(naction):\n",
    "        V[istate] = sum(matQ[istate,:]*pi1[istate,:])\n",
    "\n",
    "plt.imshow(matQ, cmap=cm.Greys_r)\n",
    "plt.show()\n",
    "\n",
    "matV = np.reshape(V, (sizeY, sizeX))\n",
    "plt.imshow(matV, cmap=cm.Greys_r)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-13.9375     -13.9375     -13.9375     -12.9375    ]\n",
      " [-12.9375     -13.9375     -12.9375     -11.9375    ]\n",
      " [-11.9375     -12.9375     -11.9375     -10.9375    ]\n",
      " [-10.9375     -11.9375     -10.9375      -9.9375    ]\n",
      " [ -9.9375     -10.9375     -11.625       -8.9375    ]\n",
      " [ -8.9375      -9.9375      -7.9375      -8.9375    ]\n",
      " [-13.9375     -13.9375     -13.10416667 -12.9375    ]\n",
      " [-12.9375     -13.9375     -12.9375     -11.9375    ]\n",
      " [-11.9375     -12.9375     -11.9375     -10.9375    ]\n",
      " [-10.9375     -11.9375     -10.9375      -9.9375    ]\n",
      " [ -9.9375     -10.9375     -10.625       -8.60416667]\n",
      " [ -8.9375     -11.625       -6.9375      -7.9375    ]\n",
      " [-13.9375     -13.10416667 -12.10416667 -12.9375    ]\n",
      " [-12.9375     -13.65972222 -12.56944444 -11.9375    ]\n",
      " [-11.9375     -12.9375     -11.9375     -10.9375    ]\n",
      " [  0.           0.           0.           0.        ]\n",
      " [-10.5         -7.625       -9.          -7.9375    ]\n",
      " [ -7.9375      -8.625       -5.9375      -6.9375    ]\n",
      " [-13.10416667 -12.10416667 -11.10416667 -11.83333333]\n",
      " [-12.9375     -13.04861111 -11.625      -10.83333333]\n",
      " [-11.9375     -12.9375     -10.83333333  -7.625     ]\n",
      " [ -7.625       -8.29166667  -4.3125      -7.52083333]\n",
      " [-10.0625      -5.75        -6.4375      -6.9375    ]\n",
      " [ -6.9375      -6.75        -4.9375      -5.9375    ]\n",
      " [-12.10416667 -11.10416667 -11.10416667 -10.10416667]\n",
      " [-12.56944444 -12.10416667 -10.68055556  -9.10416667]\n",
      " [-11.9375     -12.56944444  -9.10416667  -5.75      ]\n",
      " [ -4.3125      -7.1875      -1.5         -6.45833333]\n",
      " [ -9.          -2.9375      -4.875       -5.9375    ]\n",
      " [ -5.9375      -3.9375      -4.9375      -4.9375    ]]\n"
     ]
    }
   ],
   "source": [
    "print(matQ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-13.9375    , -13.9375    , -13.9375    , -12.9375    ],\n",
       "       [-12.9375    , -13.9375    , -12.9375    , -11.9375    ],\n",
       "       [-11.9375    , -12.9375    , -11.9375    , -10.9375    ],\n",
       "       [-10.9375    , -11.9375    , -10.9375    ,  -9.9375    ],\n",
       "       [ -9.9375    , -10.9375    , -11.625     ,  -8.9375    ],\n",
       "       [ -8.9375    ,  -9.9375    ,  -7.9375    ,  -8.9375    ],\n",
       "       [-13.9375    , -13.9375    , -13.10416667, -12.9375    ],\n",
       "       [-12.9375    , -13.9375    , -12.9375    , -11.9375    ],\n",
       "       [-11.9375    , -12.9375    , -11.9375    , -10.9375    ],\n",
       "       [-10.9375    , -11.9375    , -10.9375    ,  -9.9375    ],\n",
       "       [ -9.9375    , -10.9375    , -10.625     ,  -8.60416667],\n",
       "       [ -8.9375    , -11.625     ,  -6.9375    ,  -7.9375    ],\n",
       "       [-13.9375    , -13.10416667, -12.10416667, -12.9375    ],\n",
       "       [-12.9375    , -13.65972222, -12.56944444, -11.9375    ],\n",
       "       [-11.9375    , -12.9375    , -11.9375    , -10.9375    ],\n",
       "       [  0.        ,   0.        ,   0.        ,   0.        ],\n",
       "       [-10.5       ,  -7.625     ,  -9.        ,  -7.9375    ],\n",
       "       [ -7.9375    ,  -8.625     ,  -5.9375    ,  -6.9375    ],\n",
       "       [-13.10416667, -12.10416667, -11.10416667, -11.83333333],\n",
       "       [-12.9375    , -13.04861111, -11.625     , -10.83333333],\n",
       "       [-11.9375    , -12.9375    , -10.83333333,  -7.625     ],\n",
       "       [ -7.625     ,  -8.29166667,  -4.3125    ,  -7.52083333],\n",
       "       [-10.0625    ,  -5.75      ,  -6.4375    ,  -6.9375    ],\n",
       "       [ -6.9375    ,  -6.75      ,  -4.9375    ,  -5.9375    ],\n",
       "       [-12.10416667, -11.10416667, -11.10416667, -10.10416667],\n",
       "       [-12.56944444, -12.10416667, -10.68055556,  -9.10416667],\n",
       "       [-11.9375    , -12.56944444,  -9.10416667,  -5.75      ],\n",
       "       [ -4.3125    ,  -7.1875    ,  -1.5       ,  -6.45833333],\n",
       "       [ -9.        ,  -2.9375    ,  -4.875     ,  -5.9375    ],\n",
       "       [ -5.9375    ,  -3.9375    ,  -4.9375    ,  -4.9375    ]])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.save('Qsol', matQ)\n",
    "np.load('Qsol.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    1.    0.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    1.    0.  ]\n",
      " [ 0.    0.    1.    0.  ]\n",
      " [ 0.    0.    1.    0.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.25  0.25  0.25  0.25]\n",
      " [ 0.    1.    0.    0.  ]\n",
      " [ 0.    0.    1.    0.  ]\n",
      " [ 0.    0.    1.    0.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    1.    0.  ]\n",
      " [ 0.    1.    0.    0.  ]\n",
      " [ 0.    0.    1.    0.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    0.    1.  ]\n",
      " [ 0.    0.    1.    0.  ]\n",
      " [ 0.    1.    0.    0.  ]\n",
      " [ 0.    1.    0.    0.  ]]\n"
     ]
    }
   ],
   "source": [
    "print(pi1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "( 1 1 ) right\n",
      "( 2 1 ) right\n",
      "( 3 1 ) right\n",
      "( 4 1 ) right\n",
      "( 5 1 ) right\n",
      "( 6 1 ) down\n",
      "( 1 2 ) right\n",
      "( 2 2 ) right\n",
      "( 3 2 ) right\n",
      "( 4 2 ) right\n",
      "( 5 2 ) down\n",
      "( 6 2 ) down\n",
      "( 1 3 ) down\n",
      "( 2 3 ) right\n",
      "( 3 3 ) right\n",
      "( 4 3 ) up\n",
      "( 4 3 ) left\n",
      "( 4 3 ) down\n",
      "( 4 3 ) right\n",
      "( 5 3 ) left\n",
      "( 6 3 ) down\n",
      "( 1 4 ) down\n",
      "( 2 4 ) right\n",
      "( 3 4 ) right\n",
      "( 4 4 ) down\n",
      "( 5 4 ) left\n",
      "( 6 4 ) down\n",
      "( 1 5 ) right\n",
      "( 2 5 ) right\n",
      "( 3 5 ) right\n",
      "( 4 5 ) down\n",
      "( 5 5 ) left\n",
      "( 6 5 ) left\n"
     ]
    }
   ],
   "source": [
    "#show me the policy\n",
    "action = ['up','left','down','right']\n",
    "for istate in range(nstate):    \n",
    "    for iaction in range(naction):\n",
    "        if (pi1[istate,iaction]>0):\n",
    "            x, y = stateNumToCoord(istate)\n",
    "            print(\"(\",x,y,\")\", action[iaction])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}