fix algorithm

wisnunugroho21 · Oct 7, 2020 · 3f51f38 · 3f51f38
1 parent e5b9de7
commit 3f51f38
Show file tree

Hide file tree

Showing 12 changed files with 12 additions and 27 deletions.
diff --git a/PPO/pytorch/ppo_pong_pytorch.py b/PPO/pytorch/ppo_pong_pytorch.py
@@ -7,7 +7,6 @@
 from torch.distributions.kl import kl_divergence
 from torch.utils.data import Dataset, DataLoader
 from torch.optim import Adam
-import torchvision
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -190,7 +189,7 @@ def get_loss(self, action_probs, values, old_action_probs, old_values, next_valu
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = torch.where(
- (Kl >= self.policy_kl_range) & (ratios >= 1),
+ (Kl >= self.policy_kl_range) & (ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  ) 

diff --git a/PPO/pytorch/ppo_pytorch.py b/PPO/pytorch/ppo_pytorch.py
@@ -7,9 +7,7 @@
 from torch.distributions.kl import kl_divergence
 from torch.utils.data import Dataset, DataLoader
 from torch.optim import Adam
-import torchvision
 
-from tensorflow.keras.utils import to_categorical
 import matplotlib.pyplot as plt
 import numpy as np
 import sys
@@ -191,7 +189,7 @@ def get_loss(self, action_probs, values, old_action_probs, old_values, next_valu
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = torch.where(
- (Kl >= self.policy_kl_range) & (ratios >= 1),
+ (Kl >= self.policy_kl_range) & (ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  ) 

diff --git a/PPO/tensorflow 2/ppo_pong_tensorflow.py b/PPO/tensorflow 2/ppo_pong_tensorflow.py
@@ -5,7 +5,6 @@
 import tensorflow_probability as tfp
 from tensorflow.keras.layers import Dense
 from tensorflow.keras import Model
-from tensorflow.keras.utils import to_categorical
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -176,7 +175,7 @@ def get_loss(self, action_probs, values, old_action_probs, old_values, next_valu
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = tf.where(
- tf.logical_and(Kl >= self.policy_kl_range, ratios >= 1),
+ tf.logical_and(Kl >= self.policy_kl_range, ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  )

diff --git a/PPO/tensorflow 2/ppo_tensorflow.py b/PPO/tensorflow 2/ppo_tensorflow.py
@@ -5,7 +5,6 @@
 import tensorflow_probability as tfp
 from tensorflow.keras.layers import Dense
 from tensorflow.keras import Model
-from tensorflow.keras.utils import to_categorical
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -176,7 +175,7 @@ def get_loss(self, action_probs, values, old_action_probs, old_values, next_valu
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = tf.where(
- tf.logical_and(Kl >= self.policy_kl_range, ratios >= 1),
+ tf.logical_and(Kl >= self.policy_kl_range, ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  )

diff --git a/PPO_RND/pytorch/ppo_rnd_frozen_notslippery_pytorch.py b/PPO_RND/pytorch/ppo_rnd_frozen_notslippery_pytorch.py
@@ -7,9 +7,7 @@
 from torch.distributions.kl import kl_divergence
 from torch.utils.data import Dataset, DataLoader
 from torch.optim import Adam
-import torchvision
 
-from tensorflow.keras.utils import to_categorical
 import matplotlib.pyplot as plt
 import numpy as np
 import sys
@@ -310,7 +308,7 @@ def get_PPO_loss(self, action_probs, ex_values, old_action_probs, old_ex_values,
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = torch.where(
- (Kl >= self.policy_kl_range) & (ratios >= 1),
+ (Kl >= self.policy_kl_range) & (ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  ) 

diff --git a/PPO_RND/pytorch/ppo_rnd_pytorch.py b/PPO_RND/pytorch/ppo_rnd_pytorch.py
@@ -7,9 +7,7 @@
 from torch.distributions.kl import kl_divergence
 from torch.utils.data import Dataset, DataLoader
 from torch.optim import Adam
-import torchvision
 
-from tensorflow.keras.utils import to_categorical
 import matplotlib.pyplot as plt
 import numpy as np
 import sys
@@ -310,7 +308,7 @@ def get_PPO_loss(self, action_probs, ex_values, old_action_probs, old_ex_values,
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = torch.where(
- (Kl >= self.policy_kl_range) & (ratios >= 1),
+ (Kl >= self.policy_kl_range) & (ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  ) 

diff --git a/PPO_RND/tensorflow 2/ppo_frozenlake_notslippery_tensorflow.py b/PPO_RND/tensorflow 2/ppo_frozenlake_notslippery_tensorflow.py
@@ -5,7 +5,6 @@
 import tensorflow_probability as tfp
 from tensorflow.keras.layers import Dense
 from tensorflow.keras import Model
-from tensorflow.keras.utils import to_categorical
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -291,7 +290,7 @@ def get_PPO_loss(self, action_probs, ex_values, old_action_probs, old_ex_values,
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = tf.where(
- tf.logical_and(Kl >= self.policy_kl_range, ratios >= 1),
+ tf.logical_and(Kl >= self.policy_kl_range, ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  )

diff --git a/PPO_RND/tensorflow 2/ppo_rnd_tensorflow.py b/PPO_RND/tensorflow 2/ppo_rnd_tensorflow.py
@@ -5,7 +5,6 @@
 import tensorflow_probability as tfp
 from tensorflow.keras.layers import Dense
 from tensorflow.keras import Model
-from tensorflow.keras.utils import to_categorical
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -291,7 +290,7 @@ def get_PPO_loss(self, action_probs, ex_values, old_action_probs, old_ex_values,
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = tf.where(
- tf.logical_and(Kl >= self.policy_kl_range, ratios >= 1),
+ tf.logical_and(Kl >= self.policy_kl_range, ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  )

diff --git a/PPO_continous/pytorch/ppo_continous_bipedal_pytorch.py b/PPO_continous/pytorch/ppo_continous_bipedal_pytorch.py
@@ -7,7 +7,6 @@
 from torch.distributions.kl import kl_divergence
 from torch.utils.data import Dataset, DataLoader
 from torch.optim import Adam
-import torchvision
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -181,7 +180,7 @@ def get_loss(self, action_mean, values, old_action_mean, old_values, next_values
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = torch.where(
- (Kl >= self.policy_kl_range) & (ratios * Advantages >= 1 * Advantages),
+ (Kl >= self.policy_kl_range) & (ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  ) 

diff --git a/PPO_continous/pytorch/ppo_continous_pytorch.py b/PPO_continous/pytorch/ppo_continous_pytorch.py
@@ -7,7 +7,6 @@
 from torch.distributions.kl import kl_divergence
 from torch.utils.data import Dataset, DataLoader
 from torch.optim import Adam
-import torchvision
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -181,7 +180,7 @@ def get_loss(self, action_mean, values, old_action_mean, old_values, next_values
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = torch.where(
- (Kl >= self.policy_kl_range) & (ratios * Advantages >= 1 * Advantages),
+ (Kl >= self.policy_kl_range) & (ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  ) 

diff --git a/PPO_continous/tensorflow/ppo_continous_bipedal_tensorflow.py b/PPO_continous/tensorflow/ppo_continous_bipedal_tensorflow.py
@@ -5,7 +5,6 @@
 import tensorflow_probability as tfp
 from tensorflow.keras.layers import Dense
 from tensorflow.keras import Model
-from tensorflow.keras.utils import to_categorical
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -177,7 +176,7 @@ def get_loss(self, action_mean, values, old_action_mean, old_values, next_values
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = tf.where(
- tf.logical_and(Kl >= self.policy_kl_range, ratios >= 1),
+ tf.logical_and(Kl >= self.policy_kl_range, ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  )

diff --git a/PPO_continous/tensorflow/ppo_continous_tensorflow.py b/PPO_continous/tensorflow/ppo_continous_tensorflow.py
@@ -5,7 +5,6 @@
 import tensorflow_probability as tfp
 from tensorflow.keras.layers import Dense
 from tensorflow.keras import Model
-from tensorflow.keras.utils import to_categorical
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -177,7 +176,7 @@ def get_loss(self, action_mean, values, old_action_mean, old_values, next_values
 
  # Combining TR-PPO with Rollback (Truly PPO)
  pg_loss = tf.where(
- tf.logical_and(Kl >= self.policy_kl_range, ratios >= 1),
+ tf.logical_and(Kl >= self.policy_kl_range, ratios > 1),
  ratios * Advantages - self.policy_params * Kl,
  ratios * Advantages
  )