polish config

Author: rongkunxue

ding/policy/qtransformer.py

@@ -216,21 +216,6 @@ def _init_learn(self) -> None:
             )
 
         self._with_q_entropy = self._cfg.learn.with_q_entropy
-
-        # # Weight Init
-        # init_w = self._cfg.learn.init_w
-        # self._model.actor_head[-1].mu.weight.data.uniform_(-init_w, init_w)
-        # self._model.actor_head[-1].mu.bias.data.uniform_(-init_w, init_w)
-        # self._model.actor_head[-1].log_sigma_layer.weight.data.uniform_(-init_w, init_w)
-        # self._model.actor_head[-1].log_sigma_layer.bias.data.uniform_(-init_w, init_w)
-        # if self._twin_critic:
-        #     self._model.critic_head[0][-1].last.weight.data.uniform_(-init_w, init_w)
-        #     self._model.critic_head[0][-1].last.bias.data.uniform_(-init_w, init_w)
-        #     self._model.critic_head[1][-1].last.weight.data.uniform_(-init_w, init_w)
-        #     self._model.critic_head[1][-1].last.bias.data.uniform_(-init_w, init_w)
-        # else:
-        #     self._model.critic_head[2].last.weight.data.uniform_(-init_w, init_w)
-        #     self._model.critic_head[-1].last.bias.data.uniform_(-init_w, init_w)
         # Optimizers
         self._optimizer_q = Adam(
             self._model.parameters(),
@@ -288,8 +273,8 @@ def _init_learn(self) -> None:
             update_type="momentum",
             update_kwargs={"theta": self._cfg.learn.target_theta},
         )
-        self._low = np.array(self._cfg.other["low"])
-        self._high = np.array(self._cfg.other["high"])
+        self._low = np.array([-1, -1, -1])
+        self._high = np.array([1, 1, 1])
         self._action_bin = self._cfg.model.action_bins
         self._action_values = np.array(
             [

dizoo/d4rl/config/hopper_medium_expert_qtransformer_config.py

@@ -5,43 +5,47 @@
 main_config = dict(
     exp_name="hopper_medium_expert_qtransformer_seed0",
     env=dict(
-        env_id='hopper-medium-expert-v0',
+        env_id="hopper-medium-expert-v0",
         collector_env_num=5,
         evaluator_env_num=8,
         use_act_scale=True,
         n_evaluator_episode=8,
         stop_value=6000,
     ),
-
     policy=dict(
         cuda=True,
-        
         model=dict(
-            num_actions = 3,
-            action_bins = 16,
-            obs_dim = 11,
-            dueling = False,
-            attend_dim = 512,
+            num_actions=3,
+            action_bins=16,
+            obs_dim=11,
+            dueling=False,
+            attend_dim=512,
         ),
-        
         learn=dict(
             data_path=None,
             train_epoch=3000,
             batch_size=2048,
             learning_rate_q=3e-4,
             alpha=0.2,
             discount_factor_gamma=0.99,
-            min_reward = 0.0,
+            min_reward=0.0,
             auto_alpha=False,
             lagrange_thresh=-1.0,
             min_q_weight=5.0,
         ),
-        collect=dict(data_type='d4rl', ),
-        eval=dict(evaluator=dict(eval_freq=5, )),
-        other=dict(replay_buffer=dict(replay_buffer_size=2000000, ),
-                    low = [-1, -1, -1],
-                    high = [1, 1, 1],
-         ),
+        collect=dict(
+            data_type="d4rl",
+        ),
+        eval=dict(
+            evaluator=dict(
+                eval_freq=5,
+            )
+        ),
+        other=dict(
+            replay_buffer=dict(
+                replay_buffer_size=2000000,
+            ),
+        ),
     ),
 )
 
@@ -50,15 +54,17 @@
 
 create_config = dict(
     env=dict(
-        type='d4rl',
-        import_names=['dizoo.d4rl.envs.d4rl_env'],
+        type="d4rl",
+        import_names=["dizoo.d4rl.envs.d4rl_env"],
     ),
-    env_manager=dict(type='base'),
+    env_manager=dict(type="base"),
     policy=dict(
-        type='qtransformer',
-        import_names=['ding.policy.qtransformer'],
+        type="qtransformer",
+        import_names=["ding.policy.qtransformer"],
+    ),
+    replay_buffer=dict(
+        type="naive",
     ),
-    replay_buffer=dict(type='naive', ),
 )
 create_config = EasyDict(create_config)
 create_config = create_config