Cartpole_ActorCritic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 24)                120       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2)                 50        \n",
      "=================================================================\n",
      "Total params: 170\n",
      "Trainable params: 170\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_3 (Dense)              (None, 24)                120       \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 25        \n",
      "=================================================================\n",
      "Total params: 145\n",
      "Trainable params: 145\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "episode: 0   score: 13.0\n",
      "episode: 1   score: 16.0\n",
      "episode: 2   score: 8.0\n",
      "episode: 3   score: 23.0\n",
      "episode: 4   score: 32.0\n",
      "episode: 5   score: 8.0\n",
      "episode: 6   score: 10.0\n",
      "episode: 7   score: 16.0\n",
      "episode: 8   score: 12.0\n",
      "episode: 9   score: 17.0\n",
      "episode: 10   score: 26.0\n",
      "episode: 11   score: 12.0\n",
      "episode: 12   score: 19.0\n",
      "episode: 13   score: 11.0\n",
      "episode: 14   score: 13.0\n",
      "episode: 15   score: 25.0\n",
      "episode: 16   score: 14.0\n",
      "episode: 17   score: 14.0\n",
      "episode: 18   score: 12.0\n",
      "episode: 19   score: 21.0\n",
      "episode: 20   score: 34.0\n",
      "episode: 21   score: 11.0\n",
      "episode: 22   score: 11.0\n",
      "episode: 23   score: 9.0\n",
      "episode: 24   score: 14.0\n",
      "episode: 25   score: 17.0\n",
      "episode: 26   score: 11.0\n",
      "episode: 27   score: 8.0\n",
      "episode: 28   score: 22.0\n",
      "episode: 29   score: 34.0\n",
      "episode: 30   score: 15.0\n",
      "episode: 31   score: 9.0\n",
      "episode: 32   score: 12.0\n",
      "episode: 33   score: 9.0\n",
      "episode: 34   score: 18.0\n",
      "episode: 35   score: 8.0\n",
      "episode: 36   score: 10.0\n",
      "episode: 37   score: 15.0\n",
      "episode: 38   score: 41.0\n",
      "episode: 39   score: 31.0\n",
      "episode: 40   score: 12.0\n",
      "episode: 41   score: 46.0\n",
      "episode: 42   score: 25.0\n",
      "episode: 43   score: 14.0\n",
      "episode: 44   score: 14.0\n",
      "episode: 45   score: 23.0\n",
      "episode: 46   score: 22.0\n",
      "episode: 47   score: 17.0\n",
      "episode: 48   score: 21.0\n",
      "episode: 49   score: 18.0\n",
      "episode: 50   score: 20.0\n",
      "episode: 51   score: 15.0\n",
      "episode: 52   score: 22.0\n",
      "episode: 53   score: 18.0\n",
      "episode: 54   score: 18.0\n",
      "episode: 55   score: 34.0\n",
      "episode: 56   score: 47.0\n",
      "episode: 57   score: 18.0\n",
      "episode: 58   score: 16.0\n",
      "episode: 59   score: 14.0\n",
      "episode: 60   score: 36.0\n",
      "episode: 61   score: 18.0\n",
      "episode: 62   score: 22.0\n",
      "episode: 63   score: 45.0\n",
      "episode: 64   score: 17.0\n",
      "episode: 65   score: 17.0\n",
      "episode: 66   score: 14.0\n",
      "episode: 67   score: 25.0\n",
      "episode: 68   score: 25.0\n",
      "episode: 69   score: 21.0\n",
      "episode: 70   score: 20.0\n",
      "episode: 71   score: 23.0\n",
      "episode: 72   score: 11.0\n",
      "episode: 73   score: 21.0\n",
      "episode: 74   score: 10.0\n",
      "episode: 75   score: 26.0\n",
      "episode: 76   score: 53.0\n",
      "episode: 77   score: 14.0\n",
      "episode: 78   score: 51.0\n",
      "episode: 79   score: 22.0\n",
      "episode: 80   score: 14.0\n",
      "episode: 81   score: 24.0\n",
      "episode: 82   score: 20.0\n",
      "episode: 83   score: 57.0\n",
      "episode: 84   score: 106.0\n",
      "episode: 85   score: 44.0\n",
      "episode: 86   score: 32.0\n",
      "episode: 87   score: 33.0\n",
      "episode: 88   score: 35.0\n",
      "episode: 89   score: 17.0\n",
      "episode: 90   score: 19.0\n",
      "episode: 91   score: 47.0\n",
      "episode: 92   score: 33.0\n",
      "episode: 93   score: 25.0\n",
      "episode: 94   score: 45.0\n",
      "episode: 95   score: 61.0\n",
      "episode: 96   score: 34.0\n",
      "episode: 97   score: 58.0\n",
      "episode: 98   score: 99.0\n",
      "episode: 99   score: 43.0\n",
      "episode: 100   score: 28.0\n",
      "episode: 101   score: 27.0\n",
      "episode: 102   score: 42.0\n",
      "episode: 103   score: 42.0\n",
      "episode: 104   score: 37.0\n",
      "episode: 105   score: 26.0\n",
      "episode: 106   score: 35.0\n",
      "episode: 107   score: 95.0\n",
      "episode: 108   score: 36.0\n",
      "episode: 109   score: 18.0\n",
      "episode: 110   score: 18.0\n",
      "episode: 111   score: 37.0\n",
      "episode: 112   score: 58.0\n",
      "episode: 113   score: 75.0\n",
      "episode: 114   score: 43.0\n",
      "episode: 115   score: 40.0\n",
      "episode: 116   score: 48.0\n",
      "episode: 117   score: 53.0\n",
      "episode: 118   score: 32.0\n",
      "episode: 119   score: 48.0\n",
      "episode: 120   score: 37.0\n",
      "episode: 121   score: 76.0\n",
      "episode: 122   score: 152.0\n",
      "episode: 123   score: 42.0\n",
      "episode: 124   score: 36.0\n",
      "episode: 125   score: 32.0\n",
      "episode: 126   score: 40.0\n",
      "episode: 127   score: 45.0\n",
      "episode: 128   score: 47.0\n",
      "episode: 129   score: 98.0\n",
      "episode: 130   score: 44.0\n",
      "episode: 131   score: 77.0\n",
      "episode: 132   score: 58.0\n",
      "episode: 133   score: 81.0\n",
      "episode: 134   score: 46.0\n",
      "episode: 135   score: 65.0\n",
      "episode: 136   score: 64.0\n",
      "episode: 137   score: 50.0\n",
      "episode: 138   score: 52.0\n",
      "episode: 139   score: 43.0\n",
      "episode: 140   score: 56.0\n",
      "episode: 141   score: 135.0\n",
      "episode: 142   score: 115.0\n",
      "episode: 143   score: 72.0\n",
      "episode: 144   score: 43.0\n",
      "episode: 145   score: 44.0\n",
      "episode: 146   score: 46.0\n",
      "episode: 147   score: 40.0\n",
      "episode: 148   score: 29.0\n",
      "episode: 149   score: 43.0\n",
      "episode: 150   score: 66.0\n",
      "episode: 151   score: 62.0\n",
      "episode: 152   score: 48.0\n",
      "episode: 153   score: 71.0\n",
      "episode: 154   score: 56.0\n",
      "episode: 155   score: 93.0\n",
      "episode: 156   score: 70.0\n",
      "episode: 157   score: 94.0\n",
      "episode: 158   score: 49.0\n",
      "episode: 159   score: 59.0\n",
      "episode: 160   score: 52.0\n",
      "episode: 161   score: 81.0\n",
      "episode: 162   score: 67.0\n",
      "episode: 163   score: 112.0\n",
      "episode: 164   score: 52.0\n",
      "episode: 165   score: 92.0\n",
      "episode: 166   score: 37.0\n",
      "episode: 167   score: 96.0\n",
      "episode: 168   score: 49.0\n",
      "episode: 169   score: 59.0\n",
      "episode: 170   score: 111.0\n",
      "episode: 171   score: 127.0\n",
      "episode: 172   score: 122.0\n",
      "episode: 173   score: 65.0\n",
      "episode: 174   score: 133.0\n",
      "episode: 175   score: 88.0\n",
      "episode: 176   score: 73.0\n",
      "episode: 177   score: 197.0\n",
      "episode: 178   score: 155.0\n",
      "episode: 179   score: 95.0\n",
      "episode: 180   score: 106.0\n",
      "episode: 181   score: 112.0\n",
      "episode: 182   score: 73.0\n",
      "episode: 183   score: 98.0\n",
      "episode: 184   score: 167.0\n",
      "episode: 185   score: 86.0\n",
      "episode: 186   score: 127.0\n",
      "episode: 187   score: 195.0\n",
      "episode: 188   score: 98.0\n",
      "episode: 189   score: 77.0\n",
      "episode: 190   score: 111.0\n",
      "episode: 191   score: 138.0\n",
      "episode: 192   score: 67.0\n",
      "episode: 193   score: 173.0\n",
      "episode: 194   score: 288.0\n",
      "episode: 195   score: 220.0\n",
      "episode: 196   score: 162.0\n",
      "episode: 197   score: 254.0\n",
      "episode: 198   score: 245.0\n",
      "episode: 199   score: 272.0\n",
      "episode: 200   score: 207.0\n",
      "episode: 201   score: 123.0\n",
      "episode: 202   score: 278.0\n",
      "episode: 203   score: 133.0\n",
      "episode: 204   score: 108.0\n",
      "episode: 205   score: 46.0\n",
      "episode: 206   score: 120.0\n",
      "episode: 207   score: 129.0\n",
      "episode: 208   score: 134.0\n",
      "episode: 209   score: 167.0\n",
      "episode: 210   score: 149.0\n",
      "episode: 211   score: 112.0\n",
      "episode: 212   score: 160.0\n",
      "episode: 213   score: 146.0\n",
      "episode: 214   score: 130.0\n",
      "episode: 215   score: 146.0\n",
      "episode: 216   score: 180.0\n",
      "episode: 217   score: 172.0\n",
      "episode: 218   score: 113.0\n",
      "episode: 219   score: 179.0\n",
      "episode: 220   score: 424.0\n",
      "episode: 221   score: 328.0\n",
      "episode: 222   score: 395.0\n",
      "episode: 223   score: 315.0\n",
      "episode: 224   score: 247.0\n",
      "episode: 225   score: 168.0\n",
      "episode: 226   score: 200.0\n",
      "episode: 227   score: 250.0\n",
      "episode: 228   score: 167.0\n",
      "episode: 229   score: 330.0\n",
      "episode: 230   score: 192.0\n",
      "episode: 231   score: 149.0\n",
      "episode: 232   score: 36.0\n",
      "episode: 233   score: 39.0\n",
      "episode: 234   score: 28.0\n",
      "episode: 235   score: 40.0\n",
      "episode: 236   score: 33.0\n",
      "episode: 237   score: 44.0\n",
      "episode: 238   score: 100.0\n",
      "episode: 239   score: 121.0\n",
      "episode: 240   score: 164.0\n",
      "episode: 241   score: 246.0\n",
      "episode: 242   score: 208.0\n",
      "episode: 243   score: 191.0\n",
      "episode: 244   score: 146.0\n",
      "episode: 245   score: 111.0\n",
      "episode: 246   score: 190.0\n",
      "episode: 247   score: 93.0\n",
      "episode: 248   score: 244.0\n",
      "episode: 249   score: 219.0\n",
      "episode: 250   score: 323.0\n",
      "episode: 251   score: 155.0\n",
      "episode: 252   score: 88.0\n",
      "episode: 253   score: 298.0\n",
      "episode: 254   score: 222.0\n",
      "episode: 255   score: 155.0\n",
      "episode: 256   score: 500.0\n",
      "episode: 257   score: 354.0\n",
      "episode: 258   score: 390.0\n",
      "episode: 259   score: 161.0\n",
      "episode: 260   score: 439.0\n",
      "episode: 261   score: 226.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode: 262   score: 96.0\n",
      "episode: 263   score: 119.0\n",
      "episode: 264   score: 254.0\n",
      "episode: 265   score: 158.0\n",
      "episode: 266   score: 121.0\n",
      "episode: 267   score: 198.0\n",
      "episode: 268   score: 158.0\n",
      "episode: 269   score: 228.0\n",
      "episode: 270   score: 219.0\n",
      "episode: 271   score: 438.0\n",
      "episode: 272   score: 400.0\n",
      "episode: 273   score: 500.0\n",
      "episode: 274   score: 500.0\n",
      "episode: 275   score: 330.0\n",
      "episode: 276   score: 500.0\n",
      "episode: 277   score: 500.0\n",
      "episode: 278   score: 500.0\n",
      "episode: 279   score: 485.0\n",
      "episode: 280   score: 500.0\n",
      "episode: 281   score: 465.0\n",
      "episode: 282   score: 329.0\n",
      "episode: 283   score: 500.0\n",
      "episode: 284   score: 500.0\n",
      "episode: 285   score: 500.0\n",
      "episode: 286   score: 476.0\n",
      "episode: 287   score: 500.0\n",
      "episode: 288   score: 500.0\n",
      "episode: 289   score: 500.0\n",
      "episode: 290   score: 500.0\n",
      "episode: 291   score: 500.0\n",
      "episode: 292   score: 500.0\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[1;31mSystemExit\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\seul\\appdata\\local\\conda\\conda\\envs\\tf\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2918: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "# %load cartpole_a2c.py\n",
    "import sys\n",
    "import gym\n",
    "import pylab\n",
    "import numpy as np\n",
    "from keras.layers import Dense\n",
    "from keras.models import Sequential\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "EPISODES = 1000\n",
    "\n",
    "\n",
    "# A2C(Advantage Actor-Critic) agent for the Cartpole\n",
    "class A2CAgent:\n",
    "    def __init__(self, state_size, action_size):\n",
    "        # if you want to see Cartpole learning, then change to True\n",
    "        self.render = False\n",
    "        self.load_model = False\n",
    "        # get size of state and action\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.value_size = 1\n",
    "\n",
    "        # These are hyper parameters for the Policy Gradient\n",
    "        self.discount_factor = 0.99\n",
    "        self.actor_lr = 0.001\n",
    "        self.critic_lr = 0.005\n",
    "\n",
    "        # create model for policy network\n",
    "        self.actor = self.build_actor()\n",
    "        self.critic = self.build_critic()\n",
    "\n",
    "        if self.load_model:\n",
    "            self.actor.load_weights(\"./save_model/cartpole_actor.h5\")\n",
    "            self.critic.load_weights(\"./save_model/cartpole_critic.h5\")\n",
    "\n",
    "    # approximate policy and value using Neural Network\n",
    "    # actor: state is input and probability of each action is output of model\n",
    "    def build_actor(self):\n",
    "        actor = Sequential()\n",
    "        actor.add(Dense(24, input_dim=self.state_size, activation='relu',\n",
    "                        kernel_initializer='he_uniform'))\n",
    "        actor.add(Dense(self.action_size, activation='softmax',\n",
    "                        kernel_initializer='he_uniform'))\n",
    "        actor.summary()\n",
    "        # See note regarding crossentropy in cartpole_reinforce.py\n",
    "        actor.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=Adam(lr=self.actor_lr))\n",
    "        return actor\n",
    "\n",
    "    # critic: state is input and value of state is output of model\n",
    "    def build_critic(self):\n",
    "        critic = Sequential()\n",
    "        critic.add(Dense(24, input_dim=self.state_size, activation='relu',\n",
    "                         kernel_initializer='he_uniform'))\n",
    "        critic.add(Dense(self.value_size, activation='linear',\n",
    "                         kernel_initializer='he_uniform'))\n",
    "        critic.summary()\n",
    "        critic.compile(loss=\"mse\", optimizer=Adam(lr=self.critic_lr))\n",
    "        return critic\n",
    "\n",
    "    # using the output of policy network, pick action stochastically\n",
    "    def get_action(self, state):\n",
    "        policy = self.actor.predict(state, batch_size=1).flatten()\n",
    "        return np.random.choice(self.action_size, 1, p=policy)[0]\n",
    "\n",
    "    # update policy network every episode\n",
    "    def train_model(self, state, action, reward, next_state, done):\n",
    "        target = np.zeros((1, self.value_size))\n",
    "        advantages = np.zeros((1, self.action_size))\n",
    "\n",
    "        value = self.critic.predict(state)[0]\n",
    "        next_value = self.critic.predict(next_state)[0]\n",
    "\n",
    "        if done:\n",
    "            advantages[0][action] = reward - value\n",
    "            target[0][0] = reward\n",
    "        else:\n",
    "            advantages[0][action] = reward + self.discount_factor * (next_value) - value\n",
    "            target[0][0] = reward + self.discount_factor * next_value\n",
    "\n",
    "        self.actor.fit(state, advantages, epochs=1, verbose=0)\n",
    "        self.critic.fit(state, target, epochs=1, verbose=0)\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # In case of CartPole-v1, maximum length of episode is 500\n",
    "    env = gym.make('CartPole-v1')\n",
    "    # get size of state and action from environment\n",
    "    state_size = env.observation_space.shape[0]\n",
    "    action_size = env.action_space.n\n",
    "\n",
    "    # make A2C agent\n",
    "    agent = A2CAgent(state_size, action_size)\n",
    "\n",
    "    scores, episodes = [], []\n",
    "\n",
    "    for e in range(EPISODES):\n",
    "        done = False\n",
    "        score = 0\n",
    "        state = env.reset()\n",
    "        state = np.reshape(state, [1, state_size])\n",
    "\n",
    "        while not done:\n",
    "            if agent.render:\n",
    "                env.render()\n",
    "\n",
    "            action = agent.get_action(state)\n",
    "            next_state, reward, done, info = env.step(action)\n",
    "            next_state = np.reshape(next_state, [1, state_size])\n",
    "            # if an action make the episode end, then gives penalty of -100\n",
    "            reward = reward if not done or score == 499 else -100\n",
    "\n",
    "            agent.train_model(state, action, reward, next_state, done)\n",
    "\n",
    "            score += reward\n",
    "            state = next_state\n",
    "\n",
    "            if done:\n",
    "                # every episode, plot the play time\n",
    "                score = score if score == 500.0 else score + 100\n",
    "                scores.append(score)\n",
    "                episodes.append(e)\n",
    "                pylab.plot(episodes, scores, 'b')\n",
    "                pylab.savefig(\"./save_graph/cartpole_a2c.png\")\n",
    "                print(\"episode:\", e, \"  score:\", score)\n",
    "\n",
    "                # if the mean of scores of last 10 episode is bigger than 490\n",
    "                # stop training\n",
    "                if np.mean(scores[-min(10, len(scores)):]) > 490:\n",
    "                    agent.actor.save_weights(\"./save_model/cartpole_actor.h5\")\n",
    "                    agent.critic.save_weights(\"./save_model/cartpole_critic.h5\")\n",
    "                    sys.exit()\n",
    "\n",
    "        # save the model\n",
    "        if e % 50 == 0:\n",
    "            agent.actor.save_weights(\"./save_model/cartpole_actor.h5\")\n",
    "            agent.critic.save_weights(\"./save_model/cartpole_critic.h5\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_17 (Dense)             (None, 24)                120       \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 2)                 50        \n",
      "=================================================================\n",
      "Total params: 170\n",
      "Trainable params: 170\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_19 (Dense)             (None, 24)                120       \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 1)                 25        \n",
      "=================================================================\n",
      "Total params: 145\n",
      "Trainable params: 145\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "299.0\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v1')\n",
    "# get size of state and action from environment\n",
    "state_size = env.observation_space.shape[0]\n",
    "action_size = env.action_space.n\n",
    "\n",
    "# make A2C agent\n",
    "agent = A2CAgent(state_size, action_size)\n",
    "agent.actor.load_weights(\"./save_model/cartpole_actor.h5\")\n",
    "agent.critic.load_weights(\"./save_model/cartpole_critic.h5\")\n",
    "agent.render = True\n",
    "\n",
    "done = False\n",
    "score = 0\n",
    "state = env.reset()\n",
    "state = np.reshape(state, [1, state_size])\n",
    "\n",
    "while not done:\n",
    "    if agent.render:\n",
    "        env.render()\n",
    "\n",
    "    action = agent.get_action(state)\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "    next_state = np.reshape(next_state, [1, state_size])\n",
    "            # if an action make the episode end, then gives penalty of -100\n",
    "    reward = reward if not done or score == 499 else -100\n",
    "\n",
    "    agent.train_model(state, action, reward, next_state, done)\n",
    "\n",
    "    score += reward\n",
    "    state = next_state\n",
    "\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}