BCO_Pendulum.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementation for Inverted Pendulum:\n",
    "\n",
    "- Behavioral Cloning from Observation (BCO) <a href=\"https://arxiv.org/abs/1805.01954v2\">paper link</a>\n",
    "\n",
    "## Author: Montaser Mohammedalamen"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Installation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install gym\n",
    "! pip install box2d-py\n",
    "! pip install torchvision"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import numpy as np \n",
    "import gym\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.functional as F\n",
    "from matplotlib import pyplot as plt\n",
    "from IPython import display\n",
    "from matplotlib import style\n",
    "plt.style.use(\"ggplot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# init environment\n",
    "env_name = \"Pendulum-v0\"\n",
    "env = gym.make(env_name)\n",
    "action_space_size = env.action_space.shape[0]\n",
    "state_space_size  = env.observation_space.shape[0]\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"device\", device)\n",
    "n=2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Expert data (States only for BCO)\n",
    "expert_states  = torch.tensor(np.load(\"states_expert_Pendulum.npy\"), dtype=torch.float)\n",
    "print(\"expert_states\", expert_states.shape)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load new data (states and actions for BCO)\n",
    "states_new_agent  =  torch.tensor(np.load (\"states_Pendulum_exploration.npy\")[:1000], dtype= torch.float)\n",
    "actions_new_agent =  torch.tensor(np.load (\"actions_Pendulum_exploration.npy\")[:1000], dtype= torch.float)\n",
    "print(\"states_new_agent\",states_new_agent.shape)\n",
    "print(\"actions_new_agent\",actions_new_agent.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_input (states, actions=None,  n=2, compare=1):\n",
    "    '''\n",
    "    Data preperpation and filtering \n",
    "    Inputs:\n",
    "    states: expert states as tensor\n",
    "    actions: actions states as tensor\n",
    "    n: window size (how many states needed to predict the next action)\n",
    "    compare: for filtering data \n",
    "    return:\n",
    "    output_states: filtered states as tensor \n",
    "    output_actions: filtered actions as tensor if actions != None\n",
    "    '''\n",
    "    count=0\n",
    "    index= []\n",
    "\n",
    "    if type(actions) != torch.Tensor:\n",
    "        ep, t, state_size = states.shape\n",
    "    else:\n",
    "        ep, t, state_size = states.shape\n",
    "        _, _, action_size = actions.shape\n",
    "\n",
    "    \n",
    "    if type(actions) != torch.Tensor:\n",
    "        output_states = torch.zeros((ep*(t-n+1) , state_size*n), dtype = torch.float)\n",
    "    else:\n",
    "        output_states = torch.zeros((ep*(t-n+1) , state_size*n), dtype = torch.float)\n",
    "        output_actions = torch.zeros((ep*(t-n+1) , action_size), dtype = torch.float)\n",
    "        \n",
    "    \n",
    "    for i in range (ep):\n",
    "        for j in range (t-n+1):\n",
    "            if (states[i, j] == -compare*torch.ones(state_size)).all() or (states[i, j+1] == -compare*torch.ones(state_size)).all():\n",
    "                index.append([i,j])\n",
    "            else:\n",
    "                output_states[count] = states[i, j:j+n].view(-1)\n",
    "\n",
    "            if type(actions) != torch.Tensor:\n",
    "                count+=1\n",
    "                # do nothing\n",
    "            else:\n",
    "                output_actions[count] = actions[i,j]\n",
    "                count+=1\n",
    "   \n",
    "    if type(actions) != torch.Tensor:\n",
    "        output_states= output_states[:count]\n",
    "        return output_states\n",
    "    else:\n",
    "        output_states  = output_states[:count]\n",
    "        output_actions = output_actions[:count]\n",
    "        return output_states, output_actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# selecting number expert trajectories from expert data\n",
    "# number_expert_trajectories = 50\n",
    "# a= np.random.randint(expert_states.shape[0] - number_expert_trajectories)\n",
    "# print(a)\n",
    "# expert_state, expert_action = to_input (expert_states[a : a+number_expert_trajectories], expert_actions[a : a+number_expert_trajectories], n=n,  compare=5)\n",
    "# print(\"expert_state\", expert_state.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_transition (training_set, model, n=2,   batch_size = 256, n_epoch = 50):\n",
    "    '''\n",
    "    train transition model, given pair of states return action (s0,s1 ---> a0 if n=2)\n",
    "    Input:\n",
    "    training_set: \n",
    "    model: transition model want to train\n",
    "    n: window size (how many states needed to predict the next action)\n",
    "    batch_size: batch size\n",
    "    n_epoch: number of epoches\n",
    "    return:\n",
    "    model: trained transition model\n",
    "    '''\n",
    "    criterion = nn.L1Loss()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=0.001) \n",
    "    loss_list = []\n",
    "    for itr in range(n_epoch):\n",
    "        total_loss = 0\n",
    "        b=0\n",
    "        for batch in range (0,training_set.shape[0], batch_size):\n",
    "            data   = training_set  [batch : batch+batch_size , :n*state_space_size]\n",
    "            y      = training_set [batch : batch+batch_size, n*state_space_size:]\n",
    "            y_pred = model(data)\n",
    "            loss   = criterion(y_pred, y)\n",
    "            total_loss += loss.item() \n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            b += 1\n",
    "        print(\"[EPOCH]: %i, [LOSS]: %.6f\" % (itr+1, total_loss/b))\n",
    "        display.clear_output(wait=True)\n",
    "        loss_list.append(total_loss / training_set.shape[0])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_BC (training_set , policy,   batch_size = 256, n_epoch = 50):\n",
    "    '''\n",
    "    train Behavioral Cloning model, given pair of states return action (s0,s1 ---> a0 if n=2)\n",
    "    Input:\n",
    "    training_set: \n",
    "    policy: Behavioral Cloning model want to train\n",
    "    n: window size (how many states needed to predict the next action)\n",
    "    batch_size: batch size\n",
    "    n_epoch: number of epoches\n",
    "    return:\n",
    "    policy: trained Behavioral Cloning model\n",
    "    '''\n",
    "    \n",
    "    criterion = nn.L1Loss()\n",
    "    optimizer = torch.optim.Adam(policy.parameters(), lr=0.001) \n",
    "    loss_list = []\n",
    "    for itr in range(n_epoch):\n",
    "        total_loss = 0\n",
    "        b=0\n",
    "        for batch in range (0,training_set.shape[0], batch_size):\n",
    "            data   = training_set  [batch : batch+batch_size , :state_space_size]\n",
    "            y      = training_set [batch : batch+batch_size, state_space_size:]\n",
    "            y_pred = policy(data)\n",
    "            loss   = criterion(y_pred, y)\n",
    "            total_loss += loss.item() \n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            b += 1\n",
    "        print(\"[EPOCH]: %i, [LOSS]: %.6f\" % (itr+1, total_loss/b))\n",
    "        display.clear_output(wait=True)\n",
    "        loss_list.append(total_loss / training_set.shape[0])\n",
    "    return policy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1- Behavioral Cloning from Observation BCO and BCO($\\alpha$)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "states_new_agent, actions_new_agent = to_input(states_new_agent, actions_new_agent, n=n, compare=5 )\n",
    "new_agent_data = torch.cat((states_new_agent , actions_new_agent), 1)\n",
    "new_agent_data = new_agent_data.detach().numpy()\n",
    "np.random.shuffle(new_agent_data)\n",
    "new_agent_data = torch.tensor(new_agent_data[:])\n",
    "print(\"new_agent_data\", new_agent_data.shape)\n",
    "n_samples = int(new_agent_data.shape[0]*0.7)\n",
    "training_set = new_agent_data[:n_samples]\n",
    "testing_set = new_agent_data[n_samples:]\n",
    "print(\"training_set\", training_set.shape)\n",
    "print(\"testing_set\", testing_set.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_trainsition_model = nn.Sequential(\n",
    "    \n",
    "    nn.Linear(n*state_space_size, 20),\n",
    "    nn.ReLU(),\n",
    "    \n",
    "    nn.Linear(20, 40),\n",
    "    nn.ReLU(),\n",
    "    \n",
    "    nn.Linear(40, 10),\n",
    "    nn.ReLU(),\n",
    "    \n",
    "    nn.Linear(10, action_space_size)\n",
    ")\n",
    "\n",
    "criterion = nn.L1Loss() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epoch = 500\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "optimizer = torch.optim.Adam(state_trainsition_model.parameters(), lr = learning_rate) \n",
    "\n",
    "loss_list = []\n",
    "test_list = []\n",
    "\n",
    "for itr in range (n_epoch):\n",
    "    total_loss = 0\n",
    "    b = 0\n",
    "    for batch in range (0,training_set.shape[0], batch_size):\n",
    "        x      = training_set  [batch : batch+batch_size , :n*state_space_size]\n",
    "        y      = training_set  [batch : batch+batch_size , n*state_space_size:]\n",
    "        y_pred = state_trainsition_model(x)\n",
    "        loss   = criterion(y_pred, y)\n",
    "        total_loss += loss.item() \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        b += 1\n",
    "    print(\"[EPOCH]: %i, [LOSS]: %.6f\" % (itr+1, total_loss / b))\n",
    "    display.clear_output(wait=True)\n",
    "    loss_list.append(total_loss / b)\n",
    "    \n",
    "    test_list.append(criterion(state_trainsition_model(testing_set[:, :n*state_space_size]), testing_set[:, n*state_space_size: ]).item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(test_list, label=\"test loss\")\n",
    "plt.xlabel(\"Number of epochs\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 42511  # select any point to test the model\n",
    "print( state_trainsition_model(testing_set[p, :n*state_space_size]))\n",
    "print(testing_set[p, n*state_space_size:])\n",
    "criterion(state_trainsition_model(testing_set[p, :n*state_space_size]), testing_set[p, n*state_space_size:] ).item()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load pre-trained Transition Model and predict infered expert actions from expert states only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expert_states = to_input(expert_states,  actions=None, n=n, compare=5)\n",
    "expert_states = expert_states.detach().numpy()\n",
    "np.random.shuffle(expert_states)\n",
    "expert_states = torch.tensor(expert_states)\n",
    "\n",
    "# state_trainsition_model = torch.load(\"pendulum_transition_model_from_exploration_states_l1_n=2\")\n",
    "infered_expert_actions  = state_trainsition_model(expert_states).detach().numpy()\n",
    "infered_expert_actions = torch.tensor(infered_expert_actions, requires_grad=False)\n",
    "infered_expert_actions = torch.clamp(infered_expert_actions, -2, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Policy Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data = torch.cat((expert_states[:, :state_space_size], infered_expert_actions),1)\n",
    "\n",
    "n_samples = int(new_data.shape[0]*0.7)\n",
    "training_set = new_data[:n_samples]\n",
    "testing_set = new_data[n_samples:]\n",
    "print(\"training_set\", training_set.shape)\n",
    "print(\"testing_set\", testing_set.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Network arch, loss function and optimizer\n",
    "\n",
    "bco_pendulum = nn.Sequential(\n",
    "    nn.Linear(state_space_size,40),\n",
    "    nn.ReLU(),\n",
    "    \n",
    "    nn.Linear(40,60),\n",
    "    nn.ReLU(),\n",
    "    \n",
    "    nn.Linear(60,20),\n",
    "    nn.ReLU(),\n",
    "    \n",
    "    nn.Linear(20,action_space_size),\n",
    ")\n",
    "\n",
    "criterion = nn.L1Loss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train BCO model \n",
    "loss_list = []\n",
    "test_loss = []\n",
    "\n",
    "batch_size = 256\n",
    "n_epoch = 500\n",
    "\n",
    "learning_rate = 0.001\n",
    "optimizer = torch.optim.Adam(bco_pendulum.parameters(), lr = learning_rate) \n",
    "\n",
    "for itr in range(n_epoch):\n",
    "    total_loss = 0\n",
    "    b=0\n",
    "    for batch in range (0,training_set.shape[0], batch_size):\n",
    "        data   = training_set  [batch : batch+batch_size , :state_space_size]\n",
    "        y      = training_set  [batch : batch+batch_size , state_space_size:]\n",
    "        y_pred = bco_pendulum(data)\n",
    "        loss   = criterion(y_pred, y)\n",
    "        total_loss += loss.item() \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        b += 1\n",
    "    print(\"[EPOCH]: %i, [LOSS]: %.6f\" % (itr+1, total_loss / b))\n",
    "    display.clear_output(wait=True)\n",
    "    loss_list.append(total_loss / b)\n",
    "    x = testing_set[:, :state_space_size]\n",
    "    y = testing_set[:,state_space_size:]\n",
    "    y_pred = bco_pendulum(x)\n",
    "    test_loss.append(criterion(y_pred, y).item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot test loss for BCO\n",
    "# torch.save(bco_pendulum, \"bco_pendulum_n=2\") #uncomment to save model\n",
    "plt.plot(test_loss, label=\"Testing Loss BCO\")\n",
    "plt.xlabel(\"iterations\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 112 # select any point to test the model\n",
    "print( bco_pendulum(testing_set[p, :state_space_size]))\n",
    "print(testing_set[p, state_space_size:])\n",
    "criterion(bco_pendulum(testing_set[p, :state_space_size]), testing_set[p, state_space_size:] ).item()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train BCO($\\alpha$) in Pendulum with interacting with environmet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "################################## parameters ##################################\n",
    "n=2 # window size\n",
    "n_iterations = 100 # max number of interacting with environment\n",
    "n_ep = 1000 # number of epoches\n",
    "max_steps = 200 # max timesteps per epoch\n",
    "gamma = 1.0 # discount factor\n",
    "seeds = np.zeros(n_iterations) # random seeds\n",
    "target_reward = -300 # stop training when reward > targit_reward\n",
    "################################## parameters ##################################\n",
    "\n",
    "\n",
    "seed_reward_mean = []\n",
    "seed_reward  = []\n",
    "\n",
    "for itr in range (n_iterations):\n",
    "   ################################## interact with env ##################################\n",
    "    G= []\n",
    "    G_mean = []\n",
    "    env.seed(int(seeds[itr]))\n",
    "    torch.manual_seed(int(seeds[itr]))\n",
    "    torch.cuda.manual_seed_all(int(seeds[itr]))\n",
    "\n",
    "    states_from_env  = -5*np.ones((n_ep, max_steps, state_space_size)) # states in\n",
    "    actions_from_env = -5*np.ones((n_ep, max_steps, action_space_size))\n",
    "    \n",
    "    for ep in range (n_ep):\n",
    "        state = env.reset()\n",
    "        rewards = []\n",
    "        R=0\n",
    "        for t in range (max_steps):\n",
    "            action = bco_pendulum(torch.tensor(state, dtype=torch.float))\n",
    "            \n",
    "            action = np.clip(action.detach().numpy(), -2,2) # clip action to be between (-1, 1)\n",
    "            \n",
    "            states_from_env[ep,t]  = state\n",
    "            actions_from_env[ep,t] = action\n",
    "            next_state , r, done, _   = env.step(action)\n",
    "            rewards.append(r)\n",
    "            state = next_state\n",
    "            if done:\n",
    "                break\n",
    "        R = sum([rewards[i]*gamma**i for i in range (len(rewards))])\n",
    "        G.append(R)\n",
    "        G_mean.append(np.mean(G))\n",
    "        \n",
    "        \n",
    "        if ep % 1 ==0:\n",
    "            print(\"ep = {} , Mean Reward = {:.6f}\".format(ep, R))\n",
    "        display.clear_output(wait=True)\n",
    "    seed_reward.append(G)\n",
    "    seed_reward_mean.append(G_mean)\n",
    "    \n",
    "    print(\"Itr = {} overall reward  = {:.6f} \".format(itr, np.mean(seed_reward_mean[-1])))\n",
    "    print(\"Interacting with environment finished\")\n",
    "\n",
    "        \n",
    "    if np.mean(seed_reward_mean[-1]) > target_reward:\n",
    "        torch.save(bco_pendulum, \"bco_pendulum={}_BCO({})_best_{}_expert_states\".format(n,itr,expert_states.shape[0]))        \n",
    "        break\n",
    "    ################################ prepare collected states and actions ##################################\n",
    "    \n",
    "    states_from_env = torch.tensor(states_from_env, dtype=torch.float)\n",
    "    actions_from_env = torch.tensor(actions_from_env, dtype=torch.float)\n",
    "    states_from_env, actions_from_env = to_input(states_from_env, actions_from_env , n=n, compare=5)\n",
    "    data_env = torch.cat((states_from_env, actions_from_env), 1).detach().numpy()\n",
    "    np.random.shuffle(data_env)\n",
    "    data_env = torch.tensor(data_env)\n",
    "    print(\"data_env\", data_env.shape)\n",
    "    \n",
    "    #################################  Update Transition Model and return infeered expert actions ##################################\n",
    "    \n",
    "    state_trainsition_model = train_transition( data_env ,  state_trainsition_model,  n=n )\n",
    "    infered_expert_actions  = state_trainsition_model( torch.tensor(expert_states) )\n",
    "    infered_expert_actions = torch.tensor( infered_expert_actions, requires_grad=False )\n",
    "    print(\"Updated Transition Model and returned infeered expert actions\")\n",
    "    \n",
    "    ################################# Update BC model ##################################\n",
    "    \n",
    "    expert_data = torch.cat((expert_states, infered_expert_actions),1)\n",
    "    bco_pendulum = train_BC(expert_data, bco_pendulum, n_epoch =100)\n",
    "    print(\" Updated BC model itra= {}\".format(itr))\n",
    "    print(\"finished\")\n",
    "    \n",
    "print(\" Updated BC model itra= {}\".format(itr))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test BCO($\\alpha$) in Pendulum environment with 5 random seeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = itr\n",
    "n_iterations = 5\n",
    "n_ep = 1000\n",
    "max_steps = 200\n",
    "gamma = 1.0 # discount factor\n",
    "seeds = [684, 559, 629, 192, 835]\n",
    "\n",
    "\n",
    "seed_reward_mean = []\n",
    "seed_reward  = []\n",
    "for itr in range (n_iterations):\n",
    "   ################################## interact with env ##################################\n",
    "    G= []\n",
    "    G_mean = []\n",
    "    env.seed(int(seeds[itr]))\n",
    "    torch.manual_seed(int(seeds[itr]))\n",
    "    torch.cuda.manual_seed_all(int(seeds[itr]))\n",
    "\n",
    "    for ep in range (n_ep):\n",
    "        state = env.reset()\n",
    "        rewards = []\n",
    "        R=0\n",
    "        for t in range (max_steps):      \n",
    "            action = bco_pendulum(torch.tensor(state, dtype=torch.float))\n",
    "            action = np.clip(action.detach().numpy(), -2,2)\n",
    "            next_state , r, done, _   = env.step(action)\n",
    "            rewards.append(r)\n",
    "            state = next_state\n",
    "            if done:\n",
    "                break\n",
    "        R = sum([rewards[i]*gamma**i for i in range (len(rewards))])\n",
    "        G.append(R)\n",
    "        G_mean.append(np.mean(G))\n",
    "        if ep % 1 ==0:\n",
    "            print(\"ep = {} , Mean Reward = {:.6f}\".format(ep, R))\n",
    "        display.clear_output(wait=True)\n",
    "    seed_reward.append(G)\n",
    "    seed_reward_mean.append(G_mean)\n",
    "    print(\"Itr = {} overall reward  = {:.6f} \".format(itr, np.mean(seed_reward_mean[-1])))\n",
    "    print(\"Interacting with environment finished\")\n",
    "# np.save(\"reward_mean_pendulum_n={}_bco({})_expert_states={}\".format(n, x , expert_states.shape[0]), seed_reward_mean) #uncomment to save the reward over 5 random seeds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BCO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_reward_mean_bco = np.array(seed_reward_mean)\n",
    "mean_bco  = np.mean(seed_reward_mean_bco,axis=0)\n",
    "std_bco  = np.std(seed_reward_mean_bco,axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Expert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expert  = np.load(\"reward_mean_pendulum_expert.npy\")\n",
    "mean_expert= np.mean(expert,axis=0)\n",
    "std_expert = np.std(expert,axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_mean  = np.load(\"reward_mean_pendulum_random.npy\")\n",
    "mean_random= np.mean(random_mean,axis=0)\n",
    "std_random  = np.std(random_mean,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scaled performance\n",
    "def scaled (x, min_value, max_value):\n",
    "    return (x - min_value) / (max_value - min_value)\n",
    "\n",
    "bco_score = scaled( mean_bco[-1]  , mean_random[-1] , mean_expert[-1] )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare BCO VS Expert VS Random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.arange(1000)\n",
    "\n",
    "plt.plot(x, mean_expert, \"-\", label=\"Expert\")\n",
    "plt.fill_between(x, mean_expert+std_expert, mean_expert-std_expert, alpha=0.2)\n",
    "\n",
    "plt.plot(x, mean_bco, \"-\", label=\"BCO\")\n",
    "plt.fill_between(x, mean_bco + std_bco, mean_bco - std_bco, alpha=0.2)\n",
    "\n",
    "plt.plot(x, mean_random, \"-\", label=\"Random\")\n",
    "plt.fill_between(x, mean_random+std_random, mean_random-std_random, alpha=0.2)\n",
    "\n",
    "plt.xlabel(\"Episodes\")\n",
    "plt.ylabel(\"Mean Reward\")\n",
    "plt.title(\"Random VS Expert VS BCO in Pendulum\")\n",
    "plt.legend()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}