Merge pull request #7 from wisnunugroho21/version_2

fix monte carlo method

PPO/pytorch/ppo_pong_pytorch.py

@@ -110,12 +110,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return torch.stack(returns)

PPO/pytorch/ppo_pytorch.py

@@ -111,12 +111,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return torch.stack(returns)

PPO/tensorflow 2/ppo_pong_tensorflow.py

@@ -104,12 +104,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return tf.stack(returns)

PPO/tensorflow 2/ppo_tensorflow.py

@@ -104,12 +104,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return tf.stack(returns)

PPO_RND/pytorch/ppo_rnd_frozen_notslippery_pytorch.py

@@ -174,12 +174,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return torch.stack(returns)

PPO_RND/pytorch/ppo_rnd_pytorch.py

@@ -174,12 +174,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return torch.stack(returns)

PPO_RND/tensorflow 2/ppo_frozenlake_notslippery_tensorflow.py

@@ -165,12 +165,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return tf.stack(returns)

PPO_RND/tensorflow 2/ppo_rnd_tensorflow.py

@@ -165,12 +165,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return tf.stack(returns)

PPO_continous/pytorch/ppo_continous_bipedal_pytorch.py

@@ -100,12 +100,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return torch.stack(returns)

PPO_continous/pytorch/ppo_continous_pytorch.py

@@ -100,12 +100,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return torch.stack(returns)

PPO_continous/tensorflow/ppo_continous_bipedal_tensorflow.py

@@ -104,12 +104,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return tf.stack(returns)

PPO_continous/tensorflow/ppo_continous_tensorflow.py

@@ -104,12 +104,12 @@ def __init__(self, gamma = 0.99, lam = 0.95):
  self.gamma = gamma
  self.lam = lam
 
- def monte_carlo_discounted(self, datas):
- returns = [] 
+ def monte_carlo_discounted(self, rewards, dones):
  running_add = 0
+ returns = [] 
 
- for i in reversed(range(len(datas))):
- running_add = running_add * self.gamma + datas[i]
+ for step in reversed(range(len(rewards))):
+ running_add = rewards[step] + (1.0 - dones[step]) * self.gamma * running_add
  returns.insert(0, running_add)
 
  return tf.stack(returns)