Remove any hacks and the anticipator

src/emma_experience_hub/api/clients/simbot/__init__.py

@@ -6,7 +6,6 @@
     SimBotPydanticCacheClient,
 )
 from emma_experience_hub.api.clients.simbot.features import SimBotFeaturesClient
-from emma_experience_hub.api.clients.simbot.hacks import SimBotHacksClient
 from emma_experience_hub.api.clients.simbot.nlu_intent import SimBotNLUIntentClient
 from emma_experience_hub.api.clients.simbot.placeholder_vision import SimBotPlaceholderVisionClient
 from emma_experience_hub.api.clients.simbot.qa_intent import SimBotQAIntentClient

src/emma_experience_hub/api/controllers/simbot/clients.py

@@ -18,7 +18,6 @@
     SimBotAuxiliaryMetadataClient,
     SimBotExtractedFeaturesClient,
     SimBotFeaturesClient,
-    SimBotHacksClient,
     SimBotNLUIntentClient,
     SimBotPlaceholderVisionClient,
     SimBotQAIntentClient,
@@ -38,7 +37,6 @@ class SimBotControllerClients(BaseModel, arbitrary_types_allowed=True):
     session_db: SimBotSessionDbClient
     confirmation_response_classifier: ConfirmationResponseClassifierClient
     compound_splitter: CompoundSplitterClient
-    simbot_hacks: SimBotHacksClient
     simbot_qa: SimBotQAIntentClient
 
     @classmethod
@@ -81,11 +79,6 @@ def from_simbot_settings(cls, simbot_settings: SimBotSettings) -> "SimBotControl
                 endpoint=simbot_settings.compound_splitter_url,
                 timeout=simbot_settings.client_timeout,
             ),
-            simbot_hacks=SimBotHacksClient(
-                endpoint=simbot_settings.simbot_hacks_url,
-                timeout=simbot_settings.client_timeout,
-                disable=not simbot_settings.feature_flags.enable_simbot_raw_text_match,
-            ),
             simbot_qa=SimBotQAIntentClient(
                 endpoint=simbot_settings.simbot_qa_url,
                 timeout=simbot_settings.client_timeout,

src/emma_experience_hub/api/controllers/simbot/controller.py

@@ -185,8 +185,6 @@ def generate_interaction_action_if_needed(self, session: SimBotSession) -> SimBo
                 session
             )
 
-            self.pipelines.anticipator.run(session)
-
         logger.info(f"[ACTION] Interaction: `{session.current_turn.actions.interaction}`")
         return session
 

src/emma_experience_hub/api/controllers/simbot/pipelines.py

@@ -14,7 +14,6 @@
     SimBotAgentActionGenerationPipeline,
     SimBotAgentIntentSelectionPipeline,
     SimBotAgentLanguageGenerationPipeline,
-    SimbotAnticipatorPipeline,
     SimBotCompoundSplitterPipeline,
     SimBotEnvironmentErrorCatchingPipeline,
     SimBotEnvironmentIntentExtractionPipeline,
@@ -35,7 +34,6 @@ class SimBotControllerPipelines(BaseModel, arbitrary_types_allowed=True):
     agent_language_generator: SimBotAgentLanguageGenerationPipeline
     find_object: SimBotFindObjectPipeline
     compound_splitter: SimBotCompoundSplitterPipeline
-    anticipator: SimbotAnticipatorPipeline
 
     @classmethod
     def from_clients(
@@ -87,30 +85,21 @@ def from_clients(
                 environment_error_pipeline=SimBotEnvironmentErrorCatchingPipeline(),
                 action_predictor_client=clients.action_predictor,
                 compound_splitter_pipeline=compound_splitter,
-                simbot_hacks_client=clients.simbot_hacks,
                 _enable_clarification_questions=simbot_settings.feature_flags.enable_clarification_questions,
                 _enable_confirmation_questions=simbot_settings.feature_flags.enable_confirmation_questions,
                 _enable_search_actions=simbot_settings.feature_flags.enable_search_actions,
                 _enable_search_after_no_match=simbot_settings.feature_flags.enable_search_after_no_match,
                 _enable_search_after_missing_inventory=simbot_settings.feature_flags.enable_search_after_missing_inventory,
                 _enable_high_level_planner=simbot_settings.feature_flags.enable_rasa_high_level_planner,
-                _enable_simbot_raw_text_match=simbot_settings.feature_flags.enable_simbot_raw_text_match,
             ),
             agent_action_generator=SimBotAgentActionGenerationPipeline(
                 features_client=clients.features,
                 action_predictor_client=clients.action_predictor,
                 action_predictor_response_parser=action_predictor_response_parser,
                 previous_action_parser=SimBotPreviousActionParser(),
                 find_object_pipeline=find_object,
-                simbot_hacks_client=clients.simbot_hacks,
-                _enable_simbot_raw_text_match=simbot_settings.feature_flags.enable_simbot_raw_text_match,
             ),
             agent_language_generator=SimBotAgentLanguageGenerationPipeline(
                 prevent_default_response_as_lightweight=simbot_settings.feature_flags.prevent_default_response_as_lightweight
             ),
-            anticipator=SimbotAnticipatorPipeline(
-                simbot_hacks_client=clients.simbot_hacks,
-                features_client=clients.features,
-                _is_offline_evaluation=simbot_settings.feature_flags.enable_offline_evaluation,
-            ),
         )

src/emma_experience_hub/common/settings/simbot.py

@@ -31,7 +31,6 @@ class SimBotFeatureFlags(BaseModel):
     enable_search_after_missing_inventory: bool = True
     enable_search_after_no_match: bool = True
     enable_simbot_qa: bool = True
-    enable_simbot_raw_text_match: bool = True
 
     prevent_default_response_as_lightweight: bool = True
 
@@ -56,7 +55,6 @@ def set_offline_evaluation_flags(
             values["enable_search_after_missing_inventory"] = False
             values["enable_simbot_qa"] = False
             values["prevent_default_response_as_lightweight"] = False
-            values["enable_simbot_raw_text_match"] = False
 
             values["gfh_location_type"] = GFHLocationType.viewpoint
         return values
@@ -98,8 +96,6 @@ class SimBotSettings(BaseSettings):
 
     compound_splitter_url: AnyHttpUrl = AnyHttpUrl(url=f"{scheme}://0.0.0.0:5508", scheme=scheme)
 
-    simbot_hacks_url: AnyHttpUrl = AnyHttpUrl(url=f"{scheme}://0.0.0.0:5509", scheme=scheme)
-
     simbot_qa_url: AnyHttpUrl = AnyHttpUrl(url=f"{scheme}://0.0.0.0:5510", scheme=scheme)
 
     placeholder_vision_url: AnyHttpUrl = AnyHttpUrl(url=f"{scheme}://0.0.0.0:5506", scheme=scheme)

src/emma_experience_hub/functions/simbot/agent_intent_selection/instruction_handler.py

@@ -6,7 +6,6 @@
 from emma_experience_hub.api.clients.simbot import (
     SimbotActionPredictionClient,
     SimBotFeaturesClient,
-    SimBotHacksClient,
     SimBotNLUIntentClient,
 )
 from emma_experience_hub.datamodels.simbot import (
@@ -33,20 +32,17 @@ def __init__(
         nlu_intent_parser: NeuralParser[SimBotIntent[SimBotNLUIntentType]],
         action_predictor_client: SimbotActionPredictionClient,
         compound_splitter_pipeline: SimBotCompoundSplitterPipeline,
-        simbot_hacks_client: SimBotHacksClient,
         _enable_clarification_questions: bool = True,
         _enable_confirmation_questions: bool = True,
         _enable_search_actions: bool = True,
         _enable_search_after_no_match: bool = True,
         _enable_search_after_missing_inventory: bool = True,
         _enable_high_level_planner: bool = True,
-        _enable_simbot_raw_text_match: bool = True,
     ) -> None:
         self._features_client = features_client
 
         self._nlu_intent_client = nlu_intent_client
         self._nlu_intent_parser = nlu_intent_parser
-        self._simbot_hacks_client = simbot_hacks_client
         self._action_predictor_client = action_predictor_client
         self._compound_splitter_pipeline = compound_splitter_pipeline
 
@@ -56,7 +52,6 @@ def __init__(
         self._enable_search_after_no_match = _enable_search_after_no_match
         self._enable_missing_inventory = _enable_search_after_missing_inventory
         self._enable_high_level_planner = _enable_high_level_planner
-        self._enable_simbot_raw_text_match = _enable_simbot_raw_text_match
 
     def run(self, session: SimBotSession) -> Optional[SimBotAgentIntents]:
         """Get the agent intent."""
@@ -70,11 +65,6 @@ def run(self, session: SimBotSession) -> Optional[SimBotAgentIntents]:
         # Call the high-level planner
         if self._enable_high_level_planner:
             session = self._compound_splitter_pipeline.run_high_level_planner(session)
-        # Check if the utterance matches one of the known templates
-        if self._enable_simbot_raw_text_match:
-            intents = self._process_utterance_with_raw_text_matcher(session)
-            if intents is not None:
-                return intents
 
         # Otherwise, use the NLU to detect it
         intents = self._process_utterance_with_nlu(session)
@@ -97,61 +87,6 @@ def _utterance_has_been_processed_by_nlu(self, session: SimBotSession) -> bool:
             and session.previous_turn.actions.is_successful
         )
 
-    def _process_utterance_with_raw_text_matcher(  # noqa: WPS212
-        self, session: SimBotSession
-    ) -> Optional[SimBotAgentIntents]:
-        """Use raw text matching to determine the agent's intent."""
-        if not session.current_turn.speech:
-            return None
-        current_utterance = session.current_turn.speech.utterance
-
-        # Check if the instruction matches an instruction template
-        raw_text_match_prediction = (
-            self._simbot_hacks_client.get_low_level_prediction_from_raw_text(
-                utterance=current_utterance,
-            )
-        )
-        if raw_text_match_prediction is not None:
-            return SimBotAgentIntents(
-                physical_interaction=SimBotIntent(type=SimBotIntentType.act_one_match)
-            )
-
-        # Check if the instruction requires us to change rooms
-        room_text_match = self._simbot_hacks_client.get_room_prediction_from_raw_text(
-            utterance=current_utterance,
-        )
-        # No room in the instruction
-        if room_text_match is None:
-            return None
-
-        # Current room in the instruction
-        if room_text_match.arena_room == session.current_turn.environment.current_room:
-            return None
-
-        # Check if we need to search for the inventory before going to the other room
-        session.current_turn.speech = SimBotUserSpeech.update_user_utterance(
-            utterance=room_text_match.modified_utterance,
-            original_utterance=session.current_turn.speech.original_utterance,
-        )
-        intents = self._process_utterance_with_nlu(session)
-        if self._should_search_missing_inventory(session, intents):
-            return self._handle_act_missing_inventory_intent(
-                session=session, intents=intents, target_room=room_text_match.arena_room
-            )
-
-        # Other room in the instruction
-        queue_elem = SimBotQueueUtterance(
-            utterance=room_text_match.modified_utterance, role=SpeakerRole.agent
-        )
-        session.current_state.utterance_queue.append_to_head(queue_elem)
-        session.current_turn.speech = SimBotUserSpeech.update_user_utterance(
-            utterance=f"go to the {room_text_match.room_name}",
-            original_utterance=session.current_turn.speech.original_utterance,
-        )
-        return SimBotAgentIntents(
-            physical_interaction=SimBotIntent(type=SimBotIntentType.act_one_match)
-        )
-
     def _process_utterance_with_nlu(self, session: SimBotSession) -> SimBotAgentIntents:
         """Perform NLU on the utterance to determine what the agent should do next.
 

src/emma_experience_hub/pipelines/simbot/__init__.py

@@ -7,7 +7,6 @@
 from emma_experience_hub.pipelines.simbot.agent_language_generation import (
     SimBotAgentLanguageGenerationPipeline,
 )
-from emma_experience_hub.pipelines.simbot.anticipator import SimbotAnticipatorPipeline
 from emma_experience_hub.pipelines.simbot.compound_splitter import SimBotCompoundSplitterPipeline
 from emma_experience_hub.pipelines.simbot.environment_error_catching import (
     SimBotEnvironmentErrorCatchingPipeline,