Add option to use vecenv in BasePlayer. Notebook checkpoint minor fix. (

Denys88#220)

* mujoco_envpool

* add use of vecenv as an option. disable it by default.

* add debug comments

* small codestyle fix

* update readme

* rename 'determenistic' to 'deterministic' in player.py and configs

---------

Co-authored-by: Denys Makoviichuk <[email protected]>
Co-authored-by: Denys Makoviichuk <[email protected]>

notebooks/mujoco_envpool_training.ipynb

@@ -518,7 +518,7 @@
         "\n",
         "runner.load(config)\n",
         "agent = runner.create_player()\n",
-        "agent.restore('runs/mujoco/nn/Walker2d-v4.pth')"
+        "agent.restore('runs/Walker2d_mujoco/nn/Walker2d-v4.pth')"
       ]
     },
     {

rl_games/common/player.py

@@ -20,11 +20,19 @@ def __init__(self, params):
         self.clip_actions = config.get('clip_actions', True)
         self.seed = self.env_config.pop('seed', None)
         if self.env_info is None:
-            self.env = vecenv.create_vec_env(self.env_name, self.config['num_actors'], **self.env_config)
-            self.env_info = self.env.get_env_info()
+            use_vecenv = self.player_config.get('use_vecenv', False)
+            if use_vecenv:
+                print('[BasePlayer] Creating vecenv: ', self.env_name)
+                self.env = vecenv.create_vec_env(
+                    self.env_name, self.config['num_actors'], **self.env_config)
+                self.env_info = self.env.get_env_info()
+            else:
+                print('[BasePlayer] Creating regular env: ', self.env_name)
+                self.env = self.create_env()
+                self.env_info = env_configurations.get_env_info(self.env)
         else:
             self.env = config.get('vec_env')
-        
+
         self.num_agents = self.env_info.get('agents', 1)
         self.value_size = self.env_info.get('value_size', 1)
         self.action_space = self.env_info['action_space']
@@ -43,14 +51,16 @@ def __init__(self, params):
         self.use_cuda = True
         self.batch_size = 1
         self.has_batch_dimension = False
-        self.has_central_value = self.config.get('central_value_config') is not None
+        self.has_central_value = self.config.get(
+            'central_value_config') is not None
         self.device_name = self.config.get('device_name', 'cuda')
         self.render_env = self.player_config.get('render', False)
         self.games_num = self.player_config.get('games_num', 2000)
         if 'deterministic' in self.player_config:
             self.is_deterministic = self.player_config['deterministic']
         else:
-            self.is_deterministic = self.player_config.get('determenistic', True)
+            self.is_deterministic = self.player_config.get(
+                'deterministic', True)
         self.n_game_life = self.player_config.get('n_game_life', 1)
         self.print_stats = self.player_config.get('print_stats', True)
         self.render_sleep = self.player_config.get('render_sleep', 0.002)
@@ -64,7 +74,7 @@ def load_networks(self, params):
     def _preproc_obs(self, obs_batch):
         if type(obs_batch) is dict:
             obs_batch = copy.copy(obs_batch)
-            for k,v in obs_batch.items():
+            for k, v in obs_batch.items():
                 if v.dtype == torch.uint8:
                     obs_batch[k] = v.float() / 255.0
                 else:
@@ -117,7 +127,7 @@ def cast_obs(self, obs):
         if isinstance(obs, torch.Tensor):
             self.is_tensor_obses = True
         elif isinstance(obs, np.ndarray):
-            assert(obs.dtype != np.int8)
+            assert (obs.dtype != np.int8)
             if obs.dtype == np.uint8:
                 obs = torch.ByteTensor(obs).to(self.device)
             else:
@@ -146,7 +156,8 @@ def get_weights(self):
     def set_weights(self, weights):
         self.model.load_state_dict(weights['model'])
         if self.normalize_input and 'running_mean_std' in weights:
-            self.model.running_mean_std.load_state_dict(weights['running_mean_std'])
+            self.model.running_mean_std.load_state_dict(
+                weights['running_mean_std'])
 
     def create_env(self):
         return env_configurations.configurations[self.env_name]['env_creator'](**self.env_config)
@@ -182,7 +193,7 @@ def run(self):
         op_agent = getattr(self.env, "create_agent", None)
         if op_agent:
             agent_inited = True
-            #print('setting agent weights for selfplay')
+            # print('setting agent weights for selfplay')
             # self.env.create_agent(self.env.config)
             # self.env.set_weights(range(8),self.get_weights())
 
@@ -231,7 +242,8 @@ def run(self):
                 if done_count > 0:
                     if self.is_rnn:
                         for s in self.states:
-                            s[:, all_done_indices, :] = s[:,all_done_indices, :] * 0.0
+                            s[:, all_done_indices, :] = s[:,
+                                                          all_done_indices, :] * 0.0
 
                     cur_rewards = cr[done_indices].sum().item()
                     cur_steps = steps[done_indices].sum().item()

rl_games/configs/atari/ppo_breakout.yaml

@@ -80,4 +80,4 @@ params:
       render: False
       games_num: 200
       n_game_life: 5
-      determenistic: False
+      deterministic: False

rl_games/configs/atari/ppo_breakout_cule.yaml

@@ -78,4 +78,4 @@ params:
       render: False
       games_num: 200
       n_game_life: 5
-      determenistic: False
+      deterministic: False

rl_games/configs/atari/ppo_breakout_envpool.yaml

@@ -78,4 +78,4 @@ params:
       render: False
       games_num: 200
       n_game_life: 5
-      determenistic: False
+      deterministic: False

rl_games/configs/atari/ppo_breakout_envpool_resnet.yaml

@@ -73,5 +73,5 @@ params:
         render: False
         games_num: 20
         n_game_life: 5
-        determenistic: True
+        deterministic: True
 

rl_games/configs/atari/ppo_breakout_torch_impala.yaml

@@ -73,4 +73,4 @@ params:
       render: False
       games_num: 100
       n_game_life: 5
-      determenistic: False
+      deterministic: False

rl_games/configs/atari/ppo_gopher.yaml

@@ -82,5 +82,5 @@ params:
         render: True
         games_num: 10
         n_game_life: 1
-        determenistic: True
+        deterministic: True
         render_sleep: 0.001

rl_games/configs/atari/ppo_invaders_envpool.yaml

@@ -82,5 +82,5 @@ params:
         render: True
         games_num: 10
         n_game_life: 3
-        determenistic: True
+        deterministic: True
         render_sleep: 0.05

rl_games/configs/atari/ppo_invaders_envpool_rnn.yaml

@@ -91,5 +91,5 @@ params:
         render: True
         games_num: 10
         n_game_life: 3
-        determenistic: True
+        deterministic: True
         render_sleep: 0.05

rl_games/configs/atari/ppo_pacman_envpool.yaml

@@ -85,5 +85,5 @@ params:
         render: True
         games_num: 10
         n_game_life: 3
-        determenistic: True
+        deterministic: True
         render_sleep: 0.05

rl_games/configs/atari/ppo_pacman_envpool_resnet.yaml

@@ -75,5 +75,5 @@ params:
         render: False
         games_num: 20
         n_game_life: 3
-        determenistic: True
+        deterministic: True
 

rl_games/configs/atari/ppo_pacman_envpool_rnn.yaml

@@ -91,5 +91,5 @@ params:
         render: True
         games_num: 10
         n_game_life: 3
-        determenistic: True
+        deterministic: True
         render_sleep: 0.05

rl_games/configs/atari/ppo_pacman_torch.yaml

@@ -80,5 +80,5 @@ params:
         render: True
         games_num: 10
         n_game_life: 3
-        determenistic: True
+        deterministic: True
         render_sleep: 0.05

rl_games/configs/atari/ppo_pacman_torch_rnn.yaml

@@ -86,5 +86,5 @@ params:
         render: True
         games_num: 10
         n_game_life: 3
-        determenistic: False
+        deterministic: False
         render_sleep: 0.05

rl_games/configs/atari/ppo_pong.yaml

@@ -82,4 +82,4 @@ params:
       render: True
       games_num: 100
       n_game_life: 1
-      determenistic: True
+      deterministic: True

rl_games/configs/atari/ppo_pong_cule.yaml

@@ -81,4 +81,4 @@ params:
       render: False
       games_num: 100
       n_game_life: 1
-      determenistic: True
+      deterministic: True

rl_games/configs/atari/ppo_pong_envpool.yaml

@@ -81,4 +81,4 @@ params:
       render: False
       games_num: 100
       n_game_life: 1
-      determenistic: True
+      deterministic: True

rl_games/configs/atari/ppo_pong_envpool_resnet.yaml

@@ -72,5 +72,5 @@ params:
         render: True
         games_num: 10
         n_game_life: 1
-        determenistic: True
+        deterministic: True
 

rl_games/configs/atari/ppo_space_invaders_resnet.yaml

@@ -68,5 +68,5 @@ params:
         render: True
         games_num: 10
         n_game_life: 1
-        determenistic: True
+        deterministic: True
 

rl_games/configs/atari/ppo_space_invaders_torch.yaml

@@ -81,4 +81,4 @@ params:
         render: True
         games_num: 10
         n_game_life: 1
-        determenistic: True
+        deterministic: True

rl_games/configs/ma/ppo_slime_self_play.yaml

@@ -59,5 +59,5 @@ params:
       render: True
       games_num: 200
       n_game_life: 1
-      determenistic: True
+      deterministic: True
       device_name: 'cpu'

rl_games/configs/ma/ppo_slime_v0.yaml

@@ -54,4 +54,4 @@ params:
       render: True
       games_num: 200
       n_game_life: 1
-      determenistic: True
+      deterministic: True

rl_games/configs/minigrid/lava_rnn_img.yaml

@@ -81,6 +81,6 @@ params:
     player:
       games_num: 100
       render: True
-      determenistic: False
+      deterministic: False
       render_sleep: 0.0
 

rl_games/configs/minigrid/minigrid_rnn_img.yaml

@@ -81,4 +81,4 @@ params:
     player:
       games_num: 100
       render: True
-      determenistic: False
+      deterministic: False

rl_games/configs/mujoco/hopper.yaml

@@ -62,4 +62,4 @@ params:
 
     player:
       render: True
-      determenistic: True
+      deterministic: True

rl_games/configs/ppo_multiwalker.yaml

@@ -60,7 +60,7 @@ params:
     player:
       render: True
       games_num: 200
-      determenistic: False
+      deterministic: False
 
     env_config:
       central_value: True

rl_games/configs/ppo_walker.yaml

@@ -62,5 +62,5 @@ params:
 
     player:
       render: True
-      determenistic: True
+      deterministic: True
       games_num: 200

rl_games/configs/ppo_walker_hardcore.yaml

@@ -61,5 +61,5 @@ params:
     player:
       render: False
       games_num: 200
-      determenistic: True 
+      deterministic: True 
 

rl_games/configs/ppo_walker_rnn.yaml

@@ -71,5 +71,5 @@ params:
 
     player:
       render: True
-      determenistic: True
+      deterministic: True
       games_num: 200

rl_games/configs/ppo_walker_tcnn.yaml

@@ -54,5 +54,5 @@ params:
 
     player:
       render: True
-      determenistic: True
+      deterministic: True
       games_num: 200

rl_games/configs/smac/5m_vs_6m_rnn.yaml

@@ -62,6 +62,6 @@ params:
       render: False
       games_num: 200
       n_game_life: 1
-      determenistic: True
+      deterministic: True
 
       #reward_negative_scale: 0.1

rl_games/configs/smac/5m_vs_6m_rnn_cv.yaml

@@ -63,7 +63,7 @@ params:
       render: False
       games_num: 200
       n_game_life: 1
-      determenistic: True
+      deterministic: True
 
     central_value_config:
       minibatch_size: 512

rl_games/configs/smac/runs/MMM2_rnn.yaml

@@ -59,7 +59,7 @@ params:
       render: False
       games_num: 200
       n_game_life: 1
-      determenistic: True
+      deterministic: True
 
     central_value_config:
       minibatch_size: 512

rl_games/configs/test/test_asymmetric_discrete_mhv.yaml

@@ -66,7 +66,7 @@ params:
       multi_head_value: False
     player:
         games_num: 100
-        determenistic: True
+        deterministic: True
 
     central_value_config:
       minibatch_size: 512

rl_games/configs/test/test_asymmetric_discrete_mhv_mops.yaml

@@ -48,7 +48,7 @@ params:
       multi_head_value: False
     player:
         games_num: 100
-        determenistic: True
+        deterministic: True
 
     central_value_config:
       minibatch_size: 512

rl_games/configs/test/test_discrete.yaml

@@ -57,5 +57,5 @@ params:
       multi_head_value: False
     player:
         games_num: 100
-        determenistic: True
+        deterministic: True
 

rl_games/configs/test/test_discrete_multidiscrete_mhv.yaml

@@ -57,5 +57,5 @@ params:
       multi_head_value: False
     player:
         games_num: 100
-        determenistic: True
+        deterministic: True
 

rl_games/configs/test/test_ppo_walker_truncated_time.yaml

@@ -66,5 +66,5 @@ params:
 
     player:
       render: True
-      determenistic: True
+      deterministic: True
       games_num: 200

rl_games/configs/test/test_rnn.yaml

@@ -62,5 +62,5 @@ params:
 
     player:
         games_num: 100
-        determenistic: True
+        deterministic: True
 

rl_games/configs/test/test_rnn_multidiscrete.yaml

@@ -64,7 +64,7 @@ params:
       multi_discrete_space: True
     player:
         games_num: 100
-        determenistic: True
+        deterministic: True
 
     central_value_config:
       minibatch_size: 512