fix(nyz): fix evaluator return episode_info compatibility bug

Author: PaParaZz1

ding/entry/application_entry.py

@@ -72,8 +72,7 @@ def eval(
 
     # Evaluate
     _, episode_info = evaluator.eval()
-    reward = [e['eval_episode_return'] for e in episode_info]
-    episode_return = np.mean(to_ndarray(reward))
+    episode_return = np.mean(episode_info['eval_episode_return'])
     print('Eval is over! The performance of your RL policy is {}'.format(episode_return))
     return episode_return
 

ding/entry/serial_entry_reward_model_offpolicy.py

@@ -89,16 +89,16 @@ def serial_pipeline_reward_model_offpolicy(
     if cfg.policy.get('random_collect_size', 0) > 0:
         random_collect(cfg.policy, policy, collector, collector_env, commander, replay_buffer)
     count = 0
-    best_reward = -np.inf
+    best_return = -np.inf
     while True:
         collect_kwargs = commander.step()
         # Evaluate policy performance
         if evaluator.should_eval(learner.train_iter):
-            stop, reward = evaluator.eval(learner.save_checkpoint, learner.train_iter, collector.envstep)
-            reward_mean = np.array([r['eval_episode_return'] for r in reward]).mean()
-            if reward_mean >= best_reward:
+            stop, eval_info = evaluator.eval(learner.save_checkpoint, learner.train_iter, collector.envstep)
+            eval_return_mean = np.mean(eval_info['eval_episode_return'])
+            if eval_return_mean >= best_return:
                 reward_model.save(path=cfg.exp_name, name='best')
-                best_reward = reward_mean
+                best_return = eval_return_mean
             if stop:
                 break
         new_data_count, target_new_data_count = 0, cfg.reward_model.get('target_new_data_count', 1)

ding/entry/serial_entry_reward_model_onpolicy.py

@@ -89,16 +89,16 @@ def serial_pipeline_reward_model_onpolicy(
     if cfg.policy.get('random_collect_size', 0) > 0:
         random_collect(cfg.policy, policy, collector, collector_env, commander, replay_buffer)
     count = 0
-    best_reward = -np.inf
+    best_return = -np.inf
     while True:
         collect_kwargs = commander.step()
         # Evaluate policy performance
         if evaluator.should_eval(learner.train_iter):
-            stop, reward = evaluator.eval(learner.save_checkpoint, learner.train_iter, collector.envstep)
-            reward_mean = np.array([r['eval_episode_return'] for r in reward]).mean()
-            if reward_mean >= best_reward:
+            stop, eval_info = evaluator.eval(learner.save_checkpoint, learner.train_iter, collector.envstep)
+            eval_return_mean = np.mean(eval_info['eval_episode_return'])
+            if eval_return_mean >= best_return:
                 reward_model.save(path=cfg.exp_name, name='best')
-                best_reward = reward_mean
+                best_return = eval_return_mean
             if stop:
                 break
         new_data_count, target_new_data_count = 0, cfg.reward_model.get('target_new_data_count', 1)

ding/worker/collector/interaction_serial_evaluator.py

@@ -31,8 +31,6 @@ class InteractionSerialEvaluator(ISerialEvaluator):
         ),
         # (str) File path for visualize environment information.
         figure_path=None,
-        # (bool) Whether to return env info in termination step.
-        return_env_info=True,
     )
 
     def __init__(
@@ -253,10 +251,10 @@ def eval(
                                 self._env.enable_save_figure(env_id, self._cfg.figure_path)
                             self._policy.reset([env_id])
                             reward = t.info['eval_episode_return']
+                            saved_info = {'eval_episode_return': t.info['eval_episode_return']}
                             if 'episode_info' in t.info:
-                                eval_monitor.update_info(env_id, t.info['episode_info'])
-                            elif self._cfg.return_env_info:
-                                eval_monitor.update_info(env_id, t.info)
+                                saved_info.update(t.info['episode_info'])
+                            eval_monitor.update_info(env_id, saved_info)
                             eval_monitor.update_reward(env_id, reward)
                             self._logger.info(
                                 "[EVALUATOR]env {} finish episode, final reward: {:.4f}, current episode: {}".format(

dizoo/gym_anytrading/worker/trading_serial_evaluator.py

@@ -6,7 +6,7 @@
 
 from ding.envs import BaseEnvManager
 from ding.worker import VectorEvalMonitor, InteractionSerialEvaluator
-from ding.torch_utils import to_tensor, to_ndarray
+from ding.torch_utils import to_tensor, to_ndarray, to_item
 from ding.utils import SERIAL_EVALUATOR_REGISTRY, import_module
 
 
@@ -66,15 +66,14 @@ def eval(
             - n_episode (:obj:`int`): Number of evaluation episodes.
         Returns:
             - stop_flag (:obj:`bool`): Whether this training program can be ended.
-            - return_info (:obj:`dict`): Current evaluation return information.
+            - episode_info (:obj:`dict`): Current evaluation return information.
         '''
 
         if n_episode is None:
             n_episode = self._default_n_episode
         assert n_episode is not None, "please indicate eval n_episode"
         envstep_count = 0
         info = {}
-        return_info = []
         eval_monitor = TradingEvalMonitor(self._env.env_num, n_episode)
         self._env.reset()
         self._policy.reset()
@@ -105,10 +104,8 @@ def eval(
                         # Env reset is done by env_manager automatically.
                         self._policy.reset([env_id])
                         reward = t.info['eval_episode_return']
-                        if 'episode_info' in t.info:
-                            eval_monitor.update_info(env_id, t.info['episode_info'])
+                        eval_monitor.update_info(env_id, t.info)
                         eval_monitor.update_reward(env_id, reward)
-                        return_info.append(t.info)
 
                         #========== only used by anytrading =======
                         if 'max_possible_profit' in t.info:
@@ -185,7 +182,8 @@ def eval(
                 "Current episode_return: {} is greater than stop_value: {}".format(episode_return, self._stop_value) +
                 ", so your RL agent is converged, you can refer to 'log/evaluator/evaluator_logger.txt' for details."
             )
-        return stop_flag, return_info
+        episode_info = to_item(episode_info)
+        return stop_flag, episode_info
 
 
 class TradingEvalMonitor(VectorEvalMonitor):