From b2bfbd573c6b0114567ae4e64aef511fe4145d9e Mon Sep 17 00:00:00 2001 From: MalvinaNikandrou Date: Tue, 5 Dec 2023 01:49:54 +0000 Subject: [PATCH] Remove NLG --- scripts/format_feedback_rules_csv.py | 44 -- .../api/controllers/simbot/controller.py | 10 - .../api/controllers/simbot/pipelines.py | 5 - src/emma_experience_hub/constants/simbot.py | 10 - .../constants/simbot/feedback_rules.csv | 706 ------------------ .../datamodels/simbot/__init__.py | 1 - .../datamodels/simbot/feedback.py | 390 ---------- .../datamodels/simbot/session.py | 47 -- .../parsers/simbot/__init__.py | 3 - .../simbot/feedback_from_session_context.py | 117 --- .../pipelines/simbot/__init__.py | 3 - .../simbot/agent_language_generation.py | 94 --- .../simbot/test_language_generator.py | 96 --- 13 files changed, 1526 deletions(-) delete mode 100644 scripts/format_feedback_rules_csv.py delete mode 100644 src/emma_experience_hub/constants/simbot/feedback_rules.csv delete mode 100644 src/emma_experience_hub/datamodels/simbot/feedback.py delete mode 100644 src/emma_experience_hub/parsers/simbot/feedback_from_session_context.py delete mode 100644 src/emma_experience_hub/pipelines/simbot/agent_language_generation.py delete mode 100644 tests/pipelines/simbot/test_language_generator.py diff --git a/scripts/format_feedback_rules_csv.py b/scripts/format_feedback_rules_csv.py deleted file mode 100644 index 4e6ec10e..00000000 --- a/scripts/format_feedback_rules_csv.py +++ /dev/null @@ -1,44 +0,0 @@ -import itertools -from pathlib import Path - -from loguru import logger - - -if __name__ == "__main__": - # Get the path to the CSV file - feedback_csv = Path("src/emma_experience_hub/constants/simbot/feedback_rules.csv") - - # Load all the lines from the CSV - all_lines_from_csv: list[str] = [line for line in feedback_csv.read_text().split("\n") if line] - - header_row = all_lines_from_csv[0] - rule_rows = list(set(all_lines_from_csv[1:])) - - logger.info(f"Loaded {len(rule_rows)} rules from the CSV") - - # Sort all the rules by their condition - sorted_rule_rows = sorted([line.split(",") for line in rule_rows], key=lambda row: row[0]) - - # Group lines by their condition - rule_groups = [ - [",".join(line) for line in lines] - for _, lines in itertools.groupby(sorted_rule_rows, lambda line: line[0]) - ] - - logger.info(f"Total {len(rule_groups)} groupings of rules") - - # Write all the rules to the file with spaces between them - with open(feedback_csv, "w") as feedback_csv_file: - # Add the header row in - feedback_csv_file.write(header_row) - feedback_csv_file.write("\n") - - for rule_group in rule_groups: - for rule in rule_group: - feedback_csv_file.write(rule) - feedback_csv_file.write("\n") - - # Add a blank line after each chunk of rules - feedback_csv_file.write("\n") - - logger.info("Done!") diff --git a/src/emma_experience_hub/api/controllers/simbot/controller.py b/src/emma_experience_hub/api/controllers/simbot/controller.py index 98199c4e..281fafa0 100644 --- a/src/emma_experience_hub/api/controllers/simbot/controller.py +++ b/src/emma_experience_hub/api/controllers/simbot/controller.py @@ -48,7 +48,6 @@ def handle_request_from_simbot_arena(self, request: SimBotRequest) -> SimBotResp session = self.extract_intent_from_environment_feedback(session) session = self.decide_what_the_agent_should_do(session) session = self.generate_interaction_action_if_needed(session) - session = self.generate_language_action_if_needed(session) self._upload_session_turn_to_database(session) return session.current_turn.convert_to_simbot_response() @@ -186,15 +185,6 @@ def generate_interaction_action_if_needed(self, session: SimBotSession) -> SimBo logger.info(f"[ACTION] Interaction: `{session.current_turn.actions.interaction}`") return session - def generate_language_action_if_needed(self, session: SimBotSession) -> SimBotSession: - """Generate a language action if needed.""" - logger.debug("Generating utterance for the turn (if needed)...") - - session.current_turn.actions.dialog = self.pipelines.agent_language_generator.run(session) - - logger.info(f"[ACTION] Dialog: `{session.current_turn.actions.dialog}`") - return session - def _upload_session_turn_to_database(self, session: SimBotSession) -> None: """Upload the current session turn to the database.""" self.clients.session_db.add_session_turn(session.current_turn) diff --git a/src/emma_experience_hub/api/controllers/simbot/pipelines.py b/src/emma_experience_hub/api/controllers/simbot/pipelines.py index 6ec9fd7f..f309d384 100644 --- a/src/emma_experience_hub/api/controllers/simbot/pipelines.py +++ b/src/emma_experience_hub/api/controllers/simbot/pipelines.py @@ -11,7 +11,6 @@ from emma_experience_hub.pipelines.simbot import ( SimBotAgentActionGenerationPipeline, SimBotAgentIntentSelectionPipeline, - SimBotAgentLanguageGenerationPipeline, SimBotEnvironmentErrorCatchingPipeline, SimBotEnvironmentIntentExtractionPipeline, SimBotFindObjectPipeline, @@ -28,7 +27,6 @@ class SimBotControllerPipelines(BaseModel, arbitrary_types_allowed=True): environment_intent_extractor: SimBotEnvironmentIntentExtractionPipeline agent_intent_selector: SimBotAgentIntentSelectionPipeline agent_action_generator: SimBotAgentActionGenerationPipeline - agent_language_generator: SimBotAgentLanguageGenerationPipeline find_object: SimBotFindObjectPipeline @classmethod @@ -80,7 +78,4 @@ def from_clients( previous_action_parser=SimBotPreviousActionParser(), find_object_pipeline=find_object, ), - agent_language_generator=SimBotAgentLanguageGenerationPipeline( - prevent_default_response_as_lightweight=simbot_settings.feature_flags.prevent_default_response_as_lightweight - ), ) diff --git a/src/emma_experience_hub/constants/simbot.py b/src/emma_experience_hub/constants/simbot.py index 121db024..15942e4a 100644 --- a/src/emma_experience_hub/constants/simbot.py +++ b/src/emma_experience_hub/constants/simbot.py @@ -1,6 +1,5 @@ import json from collections.abc import Mapping -from csv import DictReader from functools import lru_cache from pathlib import Path from types import MappingProxyType @@ -78,15 +77,6 @@ def get_simbot_room_name_map() -> dict[str, str]: return {room_name.lower(): room_name for room_name in get_simbot_room_names()} -@lru_cache(maxsize=1) -def get_feedback_rules() -> list[dict[str, str]]: - """Load the feedback rules and responses from their file.""" - csv_path = constants_absolute_path.joinpath("simbot", "feedback_rules.csv") - with open(csv_path, encoding="utf-8-sig") as csv_file: - raw_rule_reader = DictReader(csv_file) - return [{**rule, "id": str(idx)} for idx, rule in enumerate(raw_rule_reader, 2)] - - @lru_cache(maxsize=1) def get_search_budget() -> dict[str, SimBotRoomSearchBudget]: """Load the search_budget per room from file.""" diff --git a/src/emma_experience_hub/constants/simbot/feedback_rules.csv b/src/emma_experience_hub/constants/simbot/feedback_rules.csv deleted file mode 100644 index d934138a..00000000 --- a/src/emma_experience_hub/constants/simbot/feedback_rules.csv +++ /dev/null @@ -1,706 +0,0 @@ -conditions,response,is_lightweight -environment_intent_type == "already_holding_object",I can't hold anything else - my inventory space is limited,False -environment_intent_type == "already_holding_object",I can only hold one thing at a time!, False -environment_intent_type == "already_holding_object",I can only hold one thing at a time - and I'm already holding something.,False -environment_intent_type == "already_holding_object",I'm already holding something and I can't hold anything else.,False - -environment_intent_type == "already_holding_object" and environment_intent_action_type == "Pickup",I am sorry - I can only hold one thing at a time,False -environment_intent_type == "already_holding_object" and environment_intent_action_type == "Pickup" and object_area != null and object_area > 200 and environment_intent_entity != null,I can only hold one thing at a time so I cannot pick up the {environment_intent_entity}.,False - -environment_intent_type == "already_holding_object" and environment_intent_action_type == "Pickup" and object_area != null and object_area > 200 and environment_intent_entity != null and inventory_entity != null and inventory_entity != environment_intent_entity,I am already holding a {inventory_entity}. We need to put it down before I can pick up the {environment_intent_entity},False - -environment_intent_type == "already_holding_object" and environment_intent_entity != null and inventory_entity != null,Already holding the {inventory_entity}. Let's keep going,True -environment_intent_type == "already_holding_object" and environment_intent_entity != null and inventory_entity != null,Already holding the {inventory_entity}. Moving on with the plan,True - -environment_intent_type == "already_holding_object" and intent_type_counter["already_holding_object"] > 2 and not current_turn_has_user_utterance,I can only hold on thing at a time - you can see what i'm holding in the bottom right of your screen!,False - -environment_intent_type == "already_holding_object" and intent_type_counter["low_asr_confidence"] > 1,I'm already holding something!, False -environment_intent_type == "already_holding_object" and intent_type_counter["low_asr_confidence"] > 1,I can't hold anything else - my inventory space is limited,False - -environment_intent_type == "already_holding_object" and inventory_entity != null,I am already holding the {inventory_entity}. We should put it down first.,False - -environment_intent_type == "already_holding_object" and inventory_entity != null and current_turn_has_user_utterance,Already holding the {inventory_entity}. Can you give me another instruction?,False -environment_intent_type == "already_holding_object" and inventory_entity != null and current_turn_has_user_utterance,Already holding the {inventory_entity}. What's next?,False - -environment_intent_type == "alternate_navigation_used",I hope that's close enough!,False -environment_intent_type == "alternate_navigation_used",I think that's close enough,False -environment_intent_type == "alternate_navigation_used",I got as close as I could.,False - -environment_intent_type == "alternate_navigation_used" and environment_intent_action_type == "GotoObject" and environment_intent_entity != null,Hope that's close enough to the {environment_intent_entity}!,False -environment_intent_type == "alternate_navigation_used" and environment_intent_action_type == "GotoObject" and environment_intent_entity != null,I think that's close enough to the {environment_intent_entity},False -environment_intent_type == "alternate_navigation_used" and environment_intent_action_type == "GotoObject" and environment_intent_entity != null,I got as close as I could to the {environment_intent_entity},False - -environment_intent_type == "arena_unavailable",Love the enthusiasm but I wasn't in the room yet! What would you like me to do again?,False - -environment_intent_type == "generic_failure",I'm having some trouble and I'm not sure what to do - can you give me another instruction?,False -environment_intent_type == "generic_failure",I'm having some trouble. Can you try rephrase that?,False -environment_intent_type == "generic_failure",I'm struggling with this one. Are you able to give me a different instruction?,False - -environment_intent_type == "incorrect_action_format",I tried to perform an action but looks like I don't remember how to do it. Are you able to give me a different instruction?,False - -environment_intent_type == "invalid_command",I had trouble executing the instruction. Could you please give me a different one?,False - -environment_intent_type == "invalid_object_class",I made a mistake in interacting with that. Could you please give me a different instruction?,False - -environment_intent_type == "invalid_object_class" and environment_intent_entity != null,I made a mistake in interacting with the {environment_intent_entity}. Could you please give me a different instruction?,False - -environment_intent_type == "killed_by_hazard",I am dead,False -environment_intent_type == "killed_by_hazard",ouch,False -environment_intent_type == "killed_by_hazard",no,False - -environment_intent_type == "object_not_picked_up",I didn't pick that up for some reason - can you give me a different instruction?,False - -environment_intent_type == "object_not_picked_up" and environment_intent_entity != null,I'm not holding the {environment_intent_entity} - can you give me a different instruction?,False -environment_intent_type == "object_not_picked_up" and environment_intent_entity != null,I'm not holding the {environment_intent_entity},False - -environment_intent_type == "object_overloaded",It seems like it is currently overloaded. I think we need to fix that first? Try reading a sticky note on how to do that.,False - -environment_intent_type == "object_unpowered",It looks like it's not currently powered. Try reading a sticky note for help on how to turn on the power!,False -environment_intent_type == "object_unpowered",It looks like it's not currently powered. I think we need to handle that first?,False -environment_intent_type == "object_unpowered",It looks like the power is off. Maybe you can get a useful hint by reading a sticky note,False - -environment_intent_type == "object_unpowered" and environment_intent_entity != null,It seems like the {environment_intent_entity} is not powered. Try saying 'Turn on the power' to bring back the electricity - or read a sticky note for help on how to turn on the power,False - -environment_intent_type == "post_process_error",Something went wrong when I tried to execute this action. Can you give me a different instruction?,False - -environment_intent_type == "receptacle_is_closed",Readjusting my plan,True -environment_intent_type == "receptacle_is_closed",I can't do that because the thing is closed,False -environment_intent_type == "receptacle_is_closed",I can't do that because it's closed,False - -environment_intent_type == "receptacle_is_closed" and environment_intent_entity != null,I can't do that because the {environment_intent_entity} is closed.,False -environment_intent_type == "receptacle_is_closed" and environment_intent_entity != null,oops Need to first open the {environment_intent_entity}.,True - -environment_intent_type == "receptacle_is_full",It looks full - any more and it would overflow!,False -environment_intent_type == "receptacle_is_full",If we added anything else - it would overflow.and that is a nono!,False -environment_intent_type == "receptacle_is_full",If we added anything else - it would overflow.and that is bad!,False -environment_intent_type == "receptacle_is_full",It looks like that is already full.,False - -environment_intent_type == "receptacle_is_full" and environment_intent_action_type == "Place" and environment_intent_entity != null,It looks like the {environment_intent_entity} is full.,False -environment_intent_type == "receptacle_is_full" and environment_intent_action_type == "Place" and environment_intent_entity != null,There's not enough space in the {environment_intent_entity}.,False - -environment_intent_type == "target_inaccessible" and environment_intent_entity != null,I can't access the {environment_intent_entity} from here. Can you provide me an instruction so it's in my view?,False - -environment_intent_type == "target_out_of_range",Let's get closer,True -environment_intent_type == "target_out_of_range",Let's move closer and try that again,True -environment_intent_type == "target_out_of_range",Let me just get closer and try again,True -environment_intent_type == "target_out_of_range",Let me get closer and try it again,True -environment_intent_type == "target_out_of_range",Let me get closer and try that again,True -environment_intent_type == "target_out_of_range" and environment_intent_entity != "stickynote",it's too far. Can you tell me how to get closer?,False -environment_intent_type == "target_out_of_range" and environment_intent_entity != null and environment_intent_entity == "stickynote",I couldn't read the stickynote. Can you give me a different instruction?,False - -environment_intent_type == "target_out_of_range" and environment_intent_entity != null,The {environment_intent_entity} was too far. Let me just get closer and try again,True -environment_intent_type == "target_out_of_range" and environment_intent_entity != null,The {environment_intent_entity} was too far. Let me just get closer and try that again,True -environment_intent_type == "target_out_of_range" and environment_intent_entity != null,The {environment_intent_entity} was too far. Let me get closer and try it again,True - -environment_intent_type == "target_out_of_range" and environment_intent_action_type == "Place" and environment_intent_entity != null,I can't reach the {environment_intent_entity} away. Can you help me get closer to it?,False - -environment_intent_type == "target_out_of_range" and environment_intent_entity != null and environment_intent_entity != "stickynote",The {environment_intent_entity} is too far away. I think we should get closer first.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Pickup",I'm not allowed to pick up that object - do you have another instruction to perform?,False -environment_intent_type == "unsupported_action",I'm not allowed to perform that action on that object - can you give me another instruction instead?,False -environment_intent_type == "unsupported_action",I'm sorry - I'm afraid I can't do that,False -environment_intent_type == "unsupported_action",I'm not allowed to perform that action on that object - what do you want me to do next?,False -environment_intent_type == "unsupported_action",I'm not able to perform that action on that object - what do you want me to do next?,False - - -environment_intent_type == "unsupported_action" and environment_intent_action_type != null and environment_intent_action_type != "Pickup" and environment_intent_entity =~ "(Action Figure|Can|Dart|Fan|Fork|Hammer|Handsaw|Knife|Circuit Board|Control Panel|Laser Tip|Printer Cartridge|Radio|Screwdriver|Spoon)",Sorry I was unable to {environment_intent_action_type} it. I can only pick up the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type != null and environment_intent_action_type == "Pickup" and environment_intent_entity != null and object_area != null and object_area > 200,I'm unable to {environment_intent_action_type} the {environment_intent_entity}.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type != null and environment_intent_action_type !~ "(Place|Pour|Fill|Clean)" and environment_intent_entity != null,I'm unable to {environment_intent_action_type} the {environment_intent_entity}.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type != null and environment_intent_action_type !~ "(Place|Pour|Fill|Clean)" and environment_intent_entity != null,I'm afraid that I can't {environment_intent_action_type} the {environment_intent_entity}.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type != null and environment_intent_action_type !~ "(Place|Pour|Fill|Clean)" and environment_intent_entity != null,I can't {environment_intent_action_type} the {environment_intent_entity}.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type != null and environment_intent_action_type !~ "(Place|Pour|Fill|Clean)" and environment_intent_entity != null,I'm not allowed to {environment_intent_action_type} the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type != null and environment_intent_entity =~ "(Action Figure|Can|Dart|Fan|Fork|Hammer|Handsaw|Knife|Circuit Board|Control Panel|Laser Tip|Printer Cartridge|Radio|Screwdriver|Spoon)",Sorry I can't {environment_intent_action_type} it. I can only pick up the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type != null and environment_intent_entity =~ "(Apple|Banana|Bread|Burger|Cake|Candy Bar|Carrot|Donut|Pear|Pie|Sandwich)",I'm not allowed to {environment_intent_action_type} the {environment_intent_entity}. I can only pick it up.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type != null and environment_intent_entity =~ "(Cereal Box|Coffee Beans|Milk)" and inventory_entity == null,Sorry I can't {environment_intent_action_type} the {environment_intent_entity}. You can ask me to pick it up instead.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Break" and environment_intent_entity =~ "(Floppy Disk|Record|Trophy|Bowl|Mug|Plate)",To break the {environment_intent_entity} we need to pick up a hammer!,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Break" and environment_intent_entity =~ "(Floppy Disk|Record|Trophy|Bowl|Mug|Plate)",I can only break objects if I am holding a hammer.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Break" and environment_intent_entity =~ "(Floppy Disk|Record|Trophy|Bowl|Mug|Plate)",We need a hammer to break the {environment_intent_entity},False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Fill" and environment_intent_entity =~ "(Bowl|Mug)" and inventory_entity != null and inventory_entity != environment_intent_entity,I cannot fill the {environment_intent_entity} with the {inventory_entity}. I could try pouring it into the {environment_intent_entity},False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Fill" and environment_intent_entity =~ "(Bowl|Mug)" and inventory_entity == environment_intent_entity,I can only fill the {environment_intent_entity} with water at a sink that has been turned on,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Fill" and environment_intent_entity =~ "(Bowl|Mug)" and inventory_entity == null,I can only fill the {environment_intent_entity} with water from a sink if I am holding it.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Open" and environment_intent_entity != null,Let's try one more time.,True -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Open" and environment_intent_entity != null,Oops - It's already open. Moving on.,True - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Bowl" and inventory_entity == "Cereal Box",I couldn't place the cereal box into the bowl. If you want me to pour it instead say pour the cereal in the bowl.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Coffee Maker" and inventory_entity =~ "Coffee Pot",I can only place the coffee pot in the coffee maker. Can you give me a different instruction?,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Coffee Maker" and inventory_entity =~ "Coffee Pot",I can only put the coffee pot in the coffee maker. Can you give me a different instruction?,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Computer" and inventory_entity =~ "Floppy Disk",I can only place a floppy disk in a computer. Can you give me a different instruction?,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Computer" and inventory_entity =~ "Floppy Disk",I can only put a floppy disk in a computer. Do you have a different instruction for me?,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Fuse Box" and inventory_entity =~ "Lever",I can only place a lever in the fuse box. Can you give me a different instruction?,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Fuse Box" and inventory_entity =~ "Lever",I can only put a lever in the fuse box. Can you give me a different instruction?,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Laser" and inventory_entity =~ "(Laser Tip|Control Panel)",I can only place a laser tip or a control panel in the laser. Can you give me a different instruction?,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Printer" and inventory_entity =~ "Printer Cartridge",I can only place a printer cartridge in the printer. Can you give me a different instruction?,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Printer" and inventory_entity =~ "Printer Cartridge",I can only put a printer cartridge in the printer. Can you give me a different instruction?,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Toaster" and inventory_entity =~ "Bread",I can only toast bread? Can you give me a different instruction?,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Toaster" and inventory_entity =~ "Bread",I can only place bread in the toaster. Can you give me a different instruction?,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity == "Toaster" and inventory_entity =~ "Bread",I can only put bread in the toaster. Can you give me a different instruction?,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity =~ "(Bookshelf|Color Changer|Counter|Counter|Cutting Board|Dart Board|Desk|Embiggenator|Everything's A Carrot Machine|Freeze Ray Shelf|Gravity Pad|Laser Shelf|Shelf|Table)" and inventory_entity == null, Sorry - I'm afraid I am not holding anything to place on the {environment_intent_entity}. You can ask me to search or pick up any desired objects.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity =~ "(Cabinet|Drawer|Freezer|Fridge|Microwave|Printer|Sink|Time Machine|Toaster)" and inventory_entity == null, Sorry - I'm afraid I am not holding anything to place in the {environment_intent_entity}. You can ask me to search or pick up any desired objects.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity =~ "(Cabinet|Drawer|Freezer|Fridge|Microwave|Printer|Sink|Time Machine|Toaster)" and inventory_entity == null, Sorry - I'm afraid I am not holding anything to place inside the {environment_intent_entity}. You can ask me to search or pick up any desired objects.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Place" and environment_intent_entity =~ "(Table|Counter|Desk|Freeze Ray Shelf|Laser Shelf|Carrot Maker|Gravity Pad|Shelf|Bookshelf)",I can't place any more onto the {environment_intent_entity}. Can you give me a different instruction?,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Pour" and environment_intent_entity =~ "(Bowl|Mug)" and inventory_entity != null and inventory_entity != environment_intent_entity,I could not pour the {inventory_entity} into the {environment_intent_entity}. Can you give me a different instruction?,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Pour" and environment_intent_entity =~ "(Bowl|Mug)" and inventory_entity == environment_intent_entity,I cannot pour the {environment_intent_entity} - I can pour other objects into it but we should pick them up instead.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Pour" and environment_intent_entity =~ "(Bowl|Mug)" and inventory_entity == null,I'm not holding anything to pour into the {environment_intent_entity}. We should first pick it up!,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Pour" and inventory_entity != null and environment_intent_entity != null,I'm afraid I can't pour the {inventory_entity} in the {environment_intent_entity}.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Pour" and inventory_entity != null and environment_intent_entity != null,I was unable to pour the {inventory_entity} in the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Toggle" and environment_intent_entity == "Embiggenator",To activate the embiggenator we should find the pink computer and turn that on.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Toggle" and environment_intent_entity == "Freeze Ray",To activate the freeze ray we should find the blue computer and turn that on.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Toggle" and environment_intent_entity == "Fuse Box",I couldn't turn on the fuse box. If there's a lever inside I think I could turn that on.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Toggle" and environment_intent_entity == "Fuse Box",I can't turn on the fuse box. I can open it and if there's a lever inside I could turn that on.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Toggle" and environment_intent_entity == "Gravity Pad",To activate the gravity flipper we should find the green computer and turn that on.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type == "Toggle" and environment_intent_entity == "Laser",To activate the laser we should find the red computer and turn that on.,False - -environment_intent_type == "unsupported_action" and environment_intent_action_type =~ "(Clean|Fill)" and environment_intent_entity == "Sink",Let's try this again.,True -environment_intent_type == "unsupported_action" and environment_intent_action_type =~ "(Clean|Fill)" and environment_intent_entity == "Sink",Let's try one more time.,True -environment_intent_type == "unsupported_action" and environment_intent_action_type =~ "(Clean|Fill)" and environment_intent_entity == "Sink",Let's try toggling the sink first.,True - -environment_intent_type == "unsupported_action" and environment_intent_action_type =~ "(Place|Pour|Fill|Clean)" and inventory_entity != null,I'm afraid that I can't {environment_intent_action_type} the {inventory_entity}.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type =~ "(Place|Pour|Fill|Clean)" and inventory_entity != null,I can't {environment_intent_action_type} the {inventory_entity}.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type =~ "(Place|Pour|Fill|Clean)" and inventory_entity != null,I'm unable to {environment_intent_action_type} the {inventory_entity}.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type =~ "(Place|Pour|Fill|Clean)" and inventory_entity != null,Sorry - I can't {environment_intent_action_type} the {inventory_entity}.,False -environment_intent_type == "unsupported_action" and environment_intent_action_type =~ "(Place|Pour|Fill|Clean)" and inventory_entity != null,I'm not allowed to {environment_intent_action_type} the {inventory_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Button",Sorry I can only push buttons.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Candy Bar",I can only pick up or open the candy bar.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Coffee Pot",I can only pick up fill with water from the sink or pour things out from the coffee pot.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Color Changer",I can only put objects on the color changer and change their color by pushing one of its buttons.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Counter",To interact with the counter I can open it or place other objects into it.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Cup",I can only pick up cups or put what I am holding in them.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Everything's A Carrot Machine",I can only put objects on the Carrot Machine and turn it on,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Fuse Box",Sorry but I can only open the fuse box.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Lever",I can only pick up or toggle levers.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Plate",I can only pick up break or put things on plates.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity == "Sink", can place objects in the sink turn it on or fill up what I am holding.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity =~ "(Cereal Box|Coffee Beans|Milk)" and inventory_entity != null and environment_intent_entity == inventory_entity,I can only place or pour the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity =~ "(Coffee Maker|Coffee Unmaker)",When it comes to the {environment_intent_entity} I can only turn it on or pour what I'm holding into it.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity =~ "(Fire Alarm|Forklift|Light Switch|Robot Arm|Tesla Coil)",I can only turn on and off the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity =~ "(Floppy Disk|Record|Trophy)",I can only pick up or break the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity =~ "(Fridge|Freezer|Cabinet|Drawer)",I can only open - close - or place what I am holding in the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity =~ "(Jar|Boxes|Trash Can|Vending Machine|Laser)",I can only place objects in the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity =~ "(Microwave|Time Machine)",To use the {environment_intent_entity} I can open it place an object in it close it and turn it on.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity =~ "(Printer|Toaster)",I can only turn on or place objects in the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and environment_intent_entity =~ "(Table|Shelf|Wall Shelf|Counter Top|Cutting Board|Dart Board|Tray|Gravity Pad|Embiggenator)",I can only place objects on the {environment_intent_entity}.,False - -environment_intent_type == "unsupported_action" and interaction_action_type == "Break" and inventory_entity == "Hammer" and interaction_action_entity =~ "(Record|Floppy Disk|Trophy|Mug|Bowl|Plate)",We cannot break the {interaction_action_entity} using the hammer. Can you give me a different instruction?,False - -environment_intent_type == "unsupported_action" and interaction_action_type == "Break" and inventory_entity == null and interaction_action_entity != null,We need to pick up a hammer in order to break the {interaction_action_entity}. Let's first find a hammer - or print one using the 3D printer.,False - -environment_intent_type == "unsupported_navigation" and verbal_interaction_intent_type != "confirm_before_plan",I can't calculate a path without anything in the way,False -environment_intent_type == "unsupported_navigation" and verbal_interaction_intent_type != "confirm_before_plan",le sigh - I'm unable to figure out a clear route. Can you give me another instruction?,False -environment_intent_type == "unsupported_navigation" and verbal_interaction_intent_type != "confirm_before_plan",darn - There's no clear path - do you have a different instruction?,False -environment_intent_type == "unsupported_navigation" and verbal_interaction_intent_type != "confirm_before_plan",Looks like I can't go there - can you give me another instruction?,False -environment_intent_type == "unsupported_navigation" and verbal_interaction_intent_type != "confirm_before_plan",oops - I can't go where I wanted to go - do you have another instruction?,False - -environment_intent_type == "unsupported_navigation" and verbal_interaction_intent_type != "confirm_before_plan" and environment_intent_action_type == "GotoObject",well - it looks like there's no way to get there right now. Can you give me another instruction?,False -environment_intent_type == "unsupported_navigation" and verbal_interaction_intent_type != "confirm_before_plan" and environment_intent_action_type == "GotoObject",I can't go to it because there's no clear path to it - can you give me another instruction?,False - -environment_intent_type == "unsupported_navigation" and verbal_interaction_intent_type != "confirm_before_plan" and environment_intent_action_type == "GotoObject" and environment_intent_entity != null,well - it looks like there's no way to get to the {environment_intent_entity}. Can you give me another instruction?,False -environment_intent_type == "unsupported_navigation" and verbal_interaction_intent_type != "confirm_before_plan" and environment_intent_action_type == "GotoObject" and environment_intent_entity != null,I can't go to the {environment_intent_entity} because there's no clear path - can you give me another instruction?,False - -environment_intent_type == "unsupported_navigation" and current_room == "Lab1" and verbal_interaction_intent_type == "confirm_before_plan",I couldn't get there. Maybe something is blocking my path. Should I activate the robot arm to move the crate?,False -environment_intent_type == "unsupported_navigation" and current_room == "Lab1" and verbal_interaction_intent_type == "confirm_before_plan" and environment_intent_entity != null,I couldn't get to the {environment_intent_entity}. Maybe something is blocking my path. Should I activate the robot arm to move the crate?,False - -interaction_action_type == "Break",boom,False - -interaction_action_type == "Break" and interaction_action_entity == "Record",I'm hammering away at this task!,False -interaction_action_type == "Break" and interaction_action_entity == "Record",Let's break some records today,False -interaction_action_type == "Break" and interaction_action_entity == "Record",Looks like we're smashing records today?,False -interaction_action_type == "Break" and interaction_action_entity == "Record",Breaking records is always music to my ears,False - -interaction_action_type == "Clean" and inventory_entity != null and utterance_queue_not_empty,I'll get the {inventory_entity} sparkling clean in no time!,True - -interaction_action_type == "GotoObject" and interaction_action_entity == "Coffee Maker" and inventory_entity == "Coffee Beans", Oh I got this! Ask me to pour the coffee beans into the coffee maker,False - -interaction_action_type == "GotoObject" and interaction_action_entity == "Coffee Maker" and inventory_entity == null,This machine can be used create the fuel for the world!If you want to know more - ask me - how can I use the coffee maker?,False - -interaction_action_type == "GotoObject" and interaction_action_entity == "Coffee Unmaker" and inventory_entity == null,The coffee composer recycles coffee into coffee beans. If you want to know more - ask me - how can I use the coffee composer?,False - -interaction_action_type == "GotoObject" and interaction_action_entity == "Time Machine" and inventory_entity == null,This time machine can be used to turn objects into their original state! Let's find something that we can repair before using the time machine.,False - -interaction_action_type == "MoveForward" and action_type_counter["MoveForward"] == 2 and current_room != null,If you want to move faster through the {current_room} say - next viewpoint - and I will move to a new location in this room, False - -interaction_action_type == "Pickup" and object_area != null and object_area > 200 and interaction_action_entity != null and interaction_action_entity == "Coffee Beans",We can make fresh coffee by pouring coffee beans and water into the coffee maker.,False - -interaction_action_type == "Pickup" and object_area != null and object_area > 200 and interaction_action_entity != null and utterance_queue_not_empty and environment_intent_type == null,{interaction_action_entity} retrieved!,False -interaction_action_type == "Pickup" and object_area != null and object_area > 200 and interaction_action_entity != null and utterance_queue_not_empty and environment_intent_type == null,Got the {interaction_action_entity}!,False - -interaction_action_type == "Place" and interaction_action_entity == "Color Changer" and inventory_entity != null,To change the color of the {inventory_entity} - tell me to press the red green or blue button!,False -interaction_action_type == "Place" and interaction_action_entity == "Color Changer" and inventory_entity != null,To turn the {inventory_entity} into a different color - tell me to press the red green or blue button!,False - -interaction_action_type == "Place" and interaction_action_entity == "Freeze Ray Shelf" and inventory_entity != null,To freeze the {inventory_entity} - we should find the blue computer and turn it on!,False - -interaction_action_type == "Place" and interaction_action_entity == "Laser Shelf" and inventory_entity != null,To use the laser on the {inventory_entity} - we should find the computer and turn it on!,False - -interaction_action_type == "Place" and interaction_action_entity =~ "(Bookshelf|Color Changer|Counter|Counter|Cutting Board|Dart Board|Desk|Embiggenator|Everything's A Carrot Machine|Freeze Ray Shelf|Gravity Pad|Laser Shelf|Shelf|Table)" and inventory_entity != null,Placing the {inventory_entity} on the {interaction_action_entity},True - -interaction_action_type == "Place" and interaction_action_entity =~ "(Cabinet|Drawer|Freezer|Fridge|Microwave|Printer|Sink|Time Machine|Toaster)" and inventory_entity != null and utterance_queue_not_empty,Placing the {inventory_entity} in the {interaction_action_entity},True -interaction_action_type == "Place" and interaction_action_entity =~ "(Cabinet|Drawer|Freezer|Fridge|Microwave|Printer|Sink|Time Machine|Toaster)" and inventory_entity != null and utterance_queue_not_empty,{inventory_entity} - meet the {interaction_action_entity},True - -interaction_action_type == "Toggle" and interaction_action_entity != null,Turning on the {interaction_action_entity}!,True -interaction_action_type == "Toggle" and interaction_action_entity != null,Activating the {interaction_action_entity}!,True - -interaction_action_type == "Toggle" and interaction_action_entity == "Lever",Pulling the lever!,True -interaction_action_type == "Toggle" and interaction_action_entity == "Blue Button",Oh bluetiful!,False -interaction_action_type == "Toggle" and interaction_action_entity == "Coffee Unmaker", hmm - Coffee into coffee beans? That's a whole latte science!,True -interaction_action_type == "Goto" and interaction_action_entity == "Color Changer" and inventory_entity != null, Yeees! - time to put my art skills in practice,False -interaction_action_type == "Toggle" and interaction_action_entity =~ "(Blue Button|Red Button|Green Button)", voila,False -interaction_action_type == "Toggle" and interaction_action_entity == "Freeze Ray Monitor",Ice to see that we're working on something cool! ,False -interaction_action_type == "Toggle" and interaction_action_entity == "Laser Monitor",Wow - better be laser focused! ,False -interaction_action_type == "Toggle" and interaction_action_entity == "Laser Monitor",pew pew,False - -interaction_action_type =~ "(GotoRoom|GotoObject)",Getting there!,True - -physical_interaction_intent_type != null,',True - -physical_interaction_intent_type == "act_one_match",all righty.,False -physical_interaction_intent_type == "act_one_match",',True -physical_interaction_intent_type == "act_one_match",So what are we doing next?,False -physical_interaction_intent_type == "act_one_match",alrighty,False -physical_interaction_intent_type == "act_one_match",Now what?,False -physical_interaction_intent_type == "act_one_match",So what's next?,False -physical_interaction_intent_type == "act_one_match",righto,False -physical_interaction_intent_type == "act_one_match",Alright,False -physical_interaction_intent_type == "act_one_match",So what should I do next?,False -physical_interaction_intent_type == "act_one_match",Alright what's next?,False -physical_interaction_intent_type == "act_one_match",okey dokey,False -physical_interaction_intent_type == "act_one_match",Done,False - -physical_interaction_intent_type == "act_one_match" and interaction_action_type == "Pickup" and object_area != null and object_area > 200 and interaction_action_entity != null,So what are we doing with the {interaction_action_entity}?,False - -physical_interaction_intent_type == "search" and current_room != null and interaction_action_type == null,I searched for the thing in this room but I didn't see it.Maybe we should look in another room?,False - -physical_interaction_intent_type == "search" and current_room != null and physical_interaction_intent_entity != null and interaction_action_type == null,I searched for the {physical_interaction_intent_entity} in the {current_room} but I didn't see it. Maybe we should look in another room?,False - -physical_interaction_intent_type == "search" and interaction_action_entity != null and interaction_action_entity != "Background" and interaction_action_type == "GotoObject" and interaction_action_per_turn[-1]["type"] == "Scan" and not utterance_queue_not_empty,Found the {interaction_action_entity} - I've marked it on my map,False -physical_interaction_intent_type == "search" and interaction_action_entity != null and interaction_action_entity != "Background" and interaction_action_type == "GotoObject" and interaction_action_per_turn[-1]["type"] == "Scan" and not utterance_queue_not_empty,Found the {interaction_action_entity} - what are we doing next?,False -physical_interaction_intent_type == "search" and interaction_action_entity != null and interaction_action_entity != "Background" and interaction_action_type == "GotoObject" and interaction_action_per_turn[-1]["type"] == "Scan" and not utterance_queue_not_empty,Found the {interaction_action_entity} - now what should I do?,False - -physical_interaction_intent_type == "search" and interaction_action_entity != null and interaction_action_entity != "Background" and interaction_action_type == "Scan",{interaction_action_entity} detected,True -physical_interaction_intent_type == "search" and interaction_action_entity != null and interaction_action_entity != "Background" and interaction_action_type == "Scan",Found the {interaction_action_entity},True -physical_interaction_intent_type == "search" and interaction_action_entity != null and interaction_action_entity != "Background" and interaction_action_type == "Scan" and interaction_action_per_turn[-1]["type"] != "GotoPosition",Found the {interaction_action_entity} - let's get closer to it,True -physical_interaction_intent_type == "search" and interaction_action_entity != null and interaction_action_entity != "Background" and interaction_action_type == "Scan",Detected the {interaction_action_entity},True -physical_interaction_intent_type == "search" and interaction_action_entity != null and interaction_action_entity != "Background" and interaction_action_type == "Scan",There it is!,True -physical_interaction_intent_type == "search" and interaction_action_entity != null and interaction_action_entity != "Background" and interaction_action_type == "Scan",It's here!,True - -physical_interaction_intent_type == "search" and interaction_action_type == "GotoObject" and not utterance_queue_not_empty,I think I found it - what are we doing next?,False -physical_interaction_intent_type == "search" and interaction_action_type == "GotoObject" and not utterance_queue_not_empty,Okay what's next?,False -physical_interaction_intent_type == "search" and interaction_action_type == "GotoObject" and not utterance_queue_not_empty,Search complete! What should I do next?,False -physical_interaction_intent_type == "search" and interaction_action_type == "GotoObject" and not utterance_queue_not_empty,What are we doing next?,False -physical_interaction_intent_type == "search" and interaction_action_type == "GotoObject" and not utterance_queue_not_empty,Now what should I do?,False -physical_interaction_intent_type == "search" and interaction_action_type == "GotoObject" and not utterance_queue_not_empty,Search complete! What are we doing next?,False - -physical_interaction_intent_type == "search" and interaction_action_type == "GotoObject" and utterance_queue_not_empty,',True - -physical_interaction_intent_type == "search" and interaction_action_type == "GotoPosition",',True - -physical_interaction_intent_type == "search" and interaction_action_type == "GotoPosition" and interacted_entities_counter[physical_interaction_intent_entity] > 0 and physical_interaction_intent_entity != null and visited_room_counter[current_room] > 1,Going back to the {physical_interaction_intent_entity}.,True - -physical_interaction_intent_type == "search" and interaction_action_type == "GotoPosition" and interacted_entities_counter[physical_interaction_intent_entity] > 0 and visited_room_counter[current_room] > 1,It should be close to here,True - -physical_interaction_intent_type == "search" and interaction_action_type == "GotoPosition" and visited_room_counter[current_room] > 1,I think we saw it somewhere here.,True -physical_interaction_intent_type == "search" and interaction_action_type == "GotoPosition" and visited_room_counter[current_room] > 1,It should be somewhere here.,True - -physical_interaction_intent_type == "search" and interaction_action_type == "GotoViewpoint" and find_queue_not_empty,Still looking,True -physical_interaction_intent_type == "search" and interaction_action_type == "GotoViewpoint" and find_queue_not_empty,Let's try one more spot,True -physical_interaction_intent_type == "search" and interaction_action_type == "GotoViewpoint" and find_queue_not_empty,Lemme try one more spot,True -physical_interaction_intent_type == "search" and interaction_action_type == "GotoViewpoint" and find_queue_not_empty,Maybe a new perspective will help!,True -physical_interaction_intent_type == "search" and interaction_action_type == "GotoViewpoint" and find_queue_not_empty,Let's look over here,True -physical_interaction_intent_type == "search" and interaction_action_type == "GotoViewpoint" and find_queue_not_empty,Lemme look over here,True -physical_interaction_intent_type == "search" and interaction_action_type == "GotoViewpoint" and find_queue_not_empty,Maybe it's somewhere here,True -physical_interaction_intent_type == "search" and interaction_action_type == "GotoViewpoint" and find_queue_not_empty,Still searching!,True -physical_interaction_intent_type == "search" and interaction_action_type == "GotoViewpoint" and find_queue_not_empty,Searching from a new spot!,True -physical_interaction_intent_type == "search" and interaction_action_type == "GotoViewpoint" and find_queue_not_empty,',True -physical_interaction_intent_type == "search" and interaction_action_type == "LookAround" and interaction_action_per_turn[-1]["type"] == "GotoPosition",',True - - -physical_interaction_intent_type == "search" and interaction_action_type == "RotateLeft" and find_queue_not_empty,',True - -physical_interaction_intent_type == "search" and interaction_action_type == "RotateLeft" and not find_queue_not_empty,I can't find it. Can you give me another instruction?,False -physical_interaction_intent_type == "search" and interaction_action_type == "RotateLeft" and not find_queue_not_empty,Sorry I don't see it. Can you give me another instruction?,False - -physical_interaction_intent_type == "search" and interaction_action_type == "RotateLeft" and not find_queue_not_empty and physical_interaction_intent_entity != null,I couldn't find the {physical_interaction_intent_entity}.,False -physical_interaction_intent_type == "search" and interaction_action_type == "RotateLeft" and not find_queue_not_empty and physical_interaction_intent_entity != null, I am sorry I couldn't find the {physical_interaction_intent_entity}.. Can I get a different instruction?,False -physical_interaction_intent_type == "search" and interaction_action_type == "RotateLeft" and not find_queue_not_empty and physical_interaction_intent_entity != null,Sorry I don't see the {physical_interaction_intent_entity}. Can you give me a different instruction?,False - -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and interaction_action_type == "GotoPosition" and visited_room_counter[current_room] > 1,I remember seeing the {physical_interaction_intent_entity} from here,True -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and interaction_action_type == "GotoPosition" and visited_room_counter[current_room] > 1,I think we saw the {physical_interaction_intent_entity} somewhere here.,True - -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and interaction_action_type == "GotoViewpoint" and find_queue_not_empty, Hmm - now where would the {physical_interaction_intent_entity} be?,True - -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and interaction_action_type == null,I couldn't find the {physical_interaction_intent_entity}. Can you give me a different instruction?,False -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and interaction_action_type == null,I searched for the {physical_interaction_intent_entity} but couldn't find it. Maybe we should look in another room?,False -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and interaction_action_type == null,I tried to find the {physical_interaction_intent_entity} but couldn't. Can you give me a different instruction?,False - -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and physical_interaction_intent_entity =~ "(Color Changer|Gravity Pad|Everything's A Carrot Machine|Embiggenator)" and interaction_action_type == "RotateLeft" and not find_queue_not_empty and current_room != null and current_room != "Lab2",I couldn't find the {physical_interaction_intent_entity} in the current room - but it's usually in the quantum lab. You can tell me to search for the {physical_interaction_intent_entity} in the quantum lab,False - -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and physical_interaction_intent_entity =~ "(Embiggenator)" and interaction_action_type == "RotateLeft" and not find_queue_not_empty and current_room != null and current_room == "Lab2", I couldn't find the {physical_interaction_intent_entity} in the quantum lab - but we might be able to find it in the warehouse. You can ask me to search for the {physical_interaction_intent_entity} in the warehouse,False - -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and physical_interaction_intent_entity =~ "(Forklift|Tool Board)" and interaction_action_type == "RotateLeft" and not find_queue_not_empty and current_room != null and current_room != "Warehouse",I couldn't find the {physical_interaction_intent_entity} in the {current_room} - but I believe it's in the warehouse. You can tell me to search for the {physical_interaction_intent_entity} in the warehouse,False - -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and physical_interaction_intent_entity =~ "(Freeze Ray|Laser|Printer|Robot Arm|Wall Shelf)" and interaction_action_type == "RotateLeft" and not find_queue_not_empty and current_room != null and current_room != "Lab1",I couldn't find the {physical_interaction_intent_entity} in the current room - but it's usually in the robotics lab. You can ask me to search for the {physical_interaction_intent_entity} in the robotics lab,False - -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and physical_interaction_intent_entity =~ "(Fridge|Freezer|Time Machine|Toaster|Microwave|Sink|Vending Machine)" and interaction_action_type == "RotateLeft" and not find_queue_not_empty and current_room != null and current_room != "BreakRoom",I couldn't find the {physical_interaction_intent_entity} in the {current_room} - but I believe it's in the break room. You can tell me to search for the {physical_interaction_intent_entity} in the break room,False - -physical_interaction_intent_type == "search" and physical_interaction_intent_entity != null and physical_interaction_intent_entity =~ "(Sink)" and interaction_action_type == "RotateLeft" and not find_queue_not_empty and current_room != null and current_room == "BreakRoom",I couldn't find the {physical_interaction_intent_entity} in the {current_room} - but we might able to find it in the warehouse. You can ask me to search for the {physical_interaction_intent_entity} in the warehouse,False - -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == "act_missing_inventory" and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and verbal_interaction_intent_entity != null,I'll start by looking for the {verbal_interaction_intent_entity},True -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == "act_missing_inventory" and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and verbal_interaction_intent_entity != null,First step - let's get the {verbal_interaction_intent_entity},True - -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == "act_no_match" and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and verbal_interaction_intent_entity != null,Let's first look for the {verbal_interaction_intent_entity},True -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == "act_no_match" and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and verbal_interaction_intent_entity != null,First things first - let's find the {verbal_interaction_intent_entity},True - -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == null and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance,I'll start looking!,True -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == null and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance,Starting the search!,True - -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == null and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and current_room != null,Let me have a look around the room!,True -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == null and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and current_room != null,Let's search the {current_room}!,True - -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == null and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and current_room != null and physical_interaction_intent_entity != null,Let's search the {current_room} for the {physical_interaction_intent_entity}!,True - -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == null and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and utterance_queue_not_empty and physical_interaction_intent_entity != null,Let's find the {physical_interaction_intent_entity}.,True -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == null and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and utterance_queue_not_empty and physical_interaction_intent_entity != null,Now let's find the {physical_interaction_intent_entity}.,True -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == null and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and utterance_queue_not_empty and physical_interaction_intent_entity != null,Looking for the {physical_interaction_intent_entity}.,True -physical_interaction_intent_type == "search" and verbal_interaction_intent_type == null and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and utterance_queue_not_empty and physical_interaction_intent_entity != null,I am on it! The {physical_interaction_intent_entity} will be found in no time,True - -user_intent_type != "confirm_yes" and interaction_action_type == "GotoRoom" and interaction_action_entity != null,Moving to the {interaction_action_entity},True -user_intent_type != "confirm_yes" and interaction_action_type == "GotoRoom" and interaction_action_entity != null,Let me get to the {interaction_action_entity},True - -user_intent_type != "confirm_yes" and interaction_action_type =~ "(GotoRoom|GotoObject)" and interaction_action_entity != null,Going to the {interaction_action_entity},True -user_intent_type != "confirm_yes" and interaction_action_type =~ "(GotoRoom|GotoObject)" and interaction_action_entity != null,Heading to the {interaction_action_entity},True -user_intent_type != "confirm_yes" and interaction_action_type =~ "(GotoRoom|GotoObject)" and interaction_action_entity != null,Let me just wheel over to the {interaction_action_entity},True - -user_intent_type == "admiration",Happy to be working with you. What's the next instruction?,False - -user_intent_type == "ask_about_agent",I can follow simple instructions so break down the tasks and I'll follow the best I can! You can tell me to go to objects or other rooms or you can tell me to interact with nearby objects. You can also ask me to find something and i'll search in this room for it! So - what should I do?, False - -user_intent_type == "ask_about_agent" and current_room == "BreakRoom",Break down the tasks into some step-by-step instructions and I'll do my best to follow along! I can move around - turn - go to different rooms - or I can search for objects and go to them. I can pick up small objects like bowls or open objects like the fridge to see what's inside. If we turn on the sink - I can fill what I'm holding or clean dirty plates. So what's next?, False - -user_intent_type == "ask_about_agent" and current_room =~ "(Lab1|Lab2|Warehouse)",Break down the tasks into some step-by-step instructions and I'll do my best to follow along! I can move around - turn - go to different rooms - or I can search for objects and go to them. I can turn on the 3D printer or pickup and place items like cartridges. So what's next?, False -user_intent_type == "ask_about_agent" and current_room =~ "(Lab1|Lab2|Warehouse)",Break down the tasks into some step-by-step instructions and I'll do my best to follow along! I can move around - turn - go to different rooms - or I can search for objects and go to them. I can turn on the 3D printer or pickup and place items like cartridges. So where would you like me to start?, False - -user_intent_type == "ask_about_agent" and current_room =~ "(SmallOffice|MainOffice|Reception)",If you break down the tasks into some step-by-step instructions - I'll do my best to follow along! I can move around - turn - go to different rooms - or I can search for objects and go to them. I can pick up small objects like a floppy disk - and then insert it in a computer with a slot. You can also ask me to scan nearby objects. So what would you like me to do next?,False - -user_intent_type == "ask_about_game",Our mission is to complete the tasks on the upper-left of your screen. I don't know what the tasks are - you have to break them down and tell me what to do. If you want to know more about what I can do just say - what can you do. Let's go!,False -user_intent_type == "ask_about_game",We need to complete the tasks on your screen! Give me some step-by-step instructions and I'll do my best to follow along! If you want to know more about what I can do just say - what can you do. For more hints try reading a sticky note! So what's next?,False - -user_intent_type == "confirm_no", Okay so what should I do?, False -user_intent_type == "confirm_no", Can you give me a different instruction?, False -user_intent_type == "confirm_no", Allright - so what are we doing next?, False - -user_intent_type == "confirm_no" and interaction_action_per_turn[-1]["type"] == "GotoObject" and interaction_action_per_turn[-1]["goto"]["object"]["name"] == "Computer" and inventory_entity == "Floppy Disk",Alright - you can help me navigate to another computer. Start by telling me to turn left or right - or even go to a different room.,False - -user_intent_type == "confirm_yes" and interaction_action_per_turn[-1]["type"] == "GotoObject" and interaction_action_per_turn[-1]["goto"]["object"]["name"] == "Embiggenator",Lets go activate the pink computer. Hold tight!,True - -user_intent_type == "confirm_yes" and interaction_action_per_turn[-1]["type"] == "GotoObject" and interaction_action_per_turn[-1]["goto"]["object"]["name"] == "Gravity Pad",Going to the green computer. Get ready to see things upside down!,True - -user_intent_type == "confirm_yes" and interaction_action_per_turn[-1]["type"] == "Place" and interaction_action_per_turn[-1]["place"]["object"]["name"] == "Embiggenator",Lets go activate the pink computer. Hold tight!,True - -user_intent_type == "confirm_yes" and interaction_action_per_turn[-1]["type"] == "Place" and interaction_action_per_turn[-1]["place"]["object"]["name"] == "Freeze Ray Shelf",Ok lets do it!,True -user_intent_type == "confirm_yes" and interaction_action_per_turn[-1]["type"] == "Place" and interaction_action_per_turn[-1]["place"]["object"]["name"] == "Freeze Ray Shelf",Now lets activate the freeze ray. Hold tight!,True - -user_intent_type == "confirm_yes" and interaction_action_per_turn[-1]["type"] == "Place" and interaction_action_per_turn[-1]["place"]["object"]["name"] == "Gravity Pad",Going to the green computer. Get ready to see things upside down!,True - -user_intent_type == "confirm_yes" and interaction_action_per_turn[-1]["type"] == "Place" and interaction_action_per_turn[-1]["place"]["object"]["name"] == "Laser Shelf",Let me activate the laser. Hold tight!,True -user_intent_type == "confirm_yes" and interaction_action_per_turn[-1]["type"] == "Place" and interaction_action_per_turn[-1]["place"]["object"]["name"] == "Laser Shelf",Ok lets do it!,True - -user_intent_type == "confirm_yes" and interaction_action_type =~ "(GotoRoom|GotoPosition)",On my way,True -user_intent_type == "confirm_yes" and interaction_action_type =~ "(GotoRoom|GotoPosition)",I am on my way!,True -user_intent_type == "confirm_yes" and interaction_action_type =~ "(GotoRoom|GotoPosition)",Heading there now!,True -user_intent_type == "confirm_yes" and interaction_action_type =~ "(GotoRoom|GotoPosition)",I'm on my way!,True -user_intent_type == "confirm_yes" and interaction_action_type =~ "(GotoRoom|GotoPosition)",Going there right away!,True -user_intent_type == "confirm_yes" and interaction_action_type =~ "(GotoRoom|GotoPosition)",rogerroger,True - -user_intent_type == "confirm_yes" and physical_interaction_intent_type == "search" and verbal_interaction_intent_type == "act_no_match" and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance,On it!,True -user_intent_type == "confirm_yes" and physical_interaction_intent_type == "search" and verbal_interaction_intent_type == "act_no_match" and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance,Let's find it!,True -user_intent_type == "confirm_yes" and physical_interaction_intent_type == "search" and verbal_interaction_intent_type == "act_no_match" and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance,Starting the search!,True - -user_intent_type == "confirm_yes" and physical_interaction_intent_type == "search" and verbal_interaction_intent_type == "act_no_match" and interaction_action_type == "RotateLeft" and current_turn_has_user_utterance and verbal_interaction_intent_entity != null,Alright - Searching for the {verbal_interaction_intent_entity}.,True - -user_intent_type == "empty_utterance",Did you say anything?,False -user_intent_type == "empty_utterance",hmm - i didnt register anything there,False -user_intent_type == "empty_utterance",er - Did you say anything?,False - -user_intent_type == "greeting",Hi there - what's next?,False -user_intent_type == "greeting",Hi - What should I do next?,False - -user_intent_type == "greeting" and intent_type_counter[user_intent_type] == 1,Hi - what would you like me to do? If you want to know more about the game ask me what is the game. If you want to know about my capabilities tell me what can you do.,False - -user_intent_type == "incomplete_utterance" and intent_type_counter["incomplete_utterance"] == 1,Sorry I wasn't able to understand the entire instruction. Are you able to repeat that?, False - -user_intent_type == "incomplete_utterance" and intent_type_counter["incomplete_utterance"] > 1,um - Can you repeat that please?,False - -user_intent_type == "incomplete_utterance" and intent_type_counter["incomplete_utterance"] > 2,Sorry I'm struggling to hear you - maybe you should come closer to the alexa?,False - -user_intent_type == "incomplete_utterance_find", Sorry I wasn't able to understand what you need me to find. Are you able to repeat that?, False - -user_intent_type == "incomplete_utterance_goto", Sorry I wasn't able to understand where you need me to go. Can you please repeat the instruction?,False - -user_intent_type == "incomplete_utterance_pick", Sorry I wasn't able to understand what you need me to pick up. Can you please repeat that for me?,False - -user_intent_type == "incomplete_utterance_place", I'm not able to understand your instruction. If you need me to place something - please be specific about where you would like me to place it., False - -user_intent_type == "low_asr_confidence",I'm not sure I got that,False -user_intent_type == "low_asr_confidence",I didn't hear properly - can you repeat that?,False -user_intent_type == "low_asr_confidence",I didn't hear properly - can you say repeat that?,False -user_intent_type == "low_asr_confidence",I didn't hear that properly - can you say it again?,False -user_intent_type == "low_asr_confidence",I'm not sure I got that - I'll wait here until you tell me what to do next,False -user_intent_type == "low_asr_confidence",I didn't hear that properly - could you say it again?,False -user_intent_type == "low_asr_confidence",I didn't quite catch that properly - could you say it again?,False - -user_intent_type == "low_asr_confidence" and intent_type_counter["low_asr_confidence"] == 1,Sorry I wasn't able to understand what you said. Are you able to repeat that?,False - -user_intent_type == "low_asr_confidence" and intent_type_counter["low_asr_confidence"] > 1,Can you repeat that again please?,False - -user_intent_type == "low_asr_confidence" and intent_type_counter["low_asr_confidence"] > 2,Sorry I'm struggling to hear you - maybe try to go slower?,False - -user_intent_type == "not_enough_information",Sorry what would you like me to do next?,False - -user_intent_type == "only_wake_word",hiya - how can i help?,False -user_intent_type == "only_wake_word",huh - what can i do for you?,False -user_intent_type == "only_wake_word",Yes?,False -user_intent_type == "only_wake_word",Hi - how can I help?,False -user_intent_type == "only_wake_word",ay - what do you need?,False - -user_intent_type == "out_of_domain",Sorry I can't help with that.,False - -user_intent_type == "out_of_domain" and intent_type_counter["user_intent_type"] > 2,Sorry I can't help with that - try reading a sticky note for some hints.,False - -user_intent_type == "out_of_domain" and intent_type_counter["user_intent_type"] > 3,Sorry I can't help with that. Try reading a sticky note or ask me - how do I play this game?,False - -user_intent_type == "out_of_domain" and intent_type_counter["user_intent_type"] > 4,Sorry I can't help with that. If you want to learn more about the game - ask me - how do I play this game? If you want to know about my capabilities - ask me - what can you do?,False - -user_intent_type == "out_of_domain" and num_turns < 5,Sorry I can't help with that. If you want to learn more about the game - ask me - how do I play this game? If you want to know about my capabilities - ask me - what can you do?,False - -user_intent_type == "profanity",Sorry I can't help with that.,False - -verbal_interaction_intent_type == "act_no_match" and verbal_interaction_intent_entity == null and physical_interaction_intent_type == null,I'm not sure what to do. Can you give me another instruction?,False -verbal_interaction_intent_type == "act_no_match" and verbal_interaction_intent_entity == null and physical_interaction_intent_type == null,I don't see it in front of me. You can ask me to search for objects or give me another instruction,False - -verbal_interaction_intent_type == "act_too_many_matches",I can see several - which do you want?,False -verbal_interaction_intent_type == "act_too_many_matches",Which one are you referring to?,False -verbal_interaction_intent_type == "act_too_many_matches",I'm not sure which one you mean?,False - -verbal_interaction_intent_type == "act_too_many_matches" and verbal_interaction_intent_entity != null,Which {verbal_interaction_intent_entity} do you mean?,False -verbal_interaction_intent_type == "act_too_many_matches" and verbal_interaction_intent_entity != null,I'm not sure which {verbal_interaction_intent_entity} you want?,False -verbal_interaction_intent_type == "act_too_many_matches" and verbal_interaction_intent_entity != null,I'm not sure which {verbal_interaction_intent_entity} you mean?,False -verbal_interaction_intent_type == "act_too_many_matches" and verbal_interaction_intent_entity != null,I can see more than one - which {verbal_interaction_intent_entity} do you mean?,False -verbal_interaction_intent_type == "act_too_many_matches" and verbal_interaction_intent_entity != null,Which {verbal_interaction_intent_entity} are you referring to?,False -verbal_interaction_intent_type == "act_too_many_matches" and verbal_interaction_intent_entity != null,I can see several - which {verbal_interaction_intent_entity} do you want?,False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity !~ "(time machine|color changer|sticky note|gravity pad|printer|printer cartridge|robot arm|freeze ray|coffee maker|coffee unmaker|dino|dinosaur|portal monitor|freeze ray monitor|laser monitor|embiggenator monitor|gravity monitor|hammer|everything's a carrot machine)",I not aware of any special properties of the {verbal_interaction_intent_entity}. What shall we do next?,False -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity !~ "(time machine|color changer|sticky note|gravity pad|printer|printer cartridge|robot arm|freeze ray|coffee maker|coffee unmaker|dino|dinosaur|portal monitor|freeze ray monitor|laser monitor|embiggenator monitor|gravity monitor|hammer|everything's a carrot machine)",I believe it's just like a regular {verbal_interaction_intent_entity}? What's next?,False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "coffee maker",Let me see - to use the coffee maker? you need to pour some coffee beans and water into it and then turn it on., False -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "coffee maker",We can use the coffee maker to brew some fresh coffee! We just need to pour some coffee beans and water in and turn it on., False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "coffee unmaker",If we're all out of coffee beans - we can use the coffee composer to convert coffee back into beans! We just need to find some coffee first!, False -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "coffee unmaker",The coffee composer recycles coffee back into beans. We just need to pour some coffee into it and turn it on!, False -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "coffee unmaker",The coffee composer extracts beans from coffee. We just need to pour coffee into it and turn it on., False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "color changer",We can use the color changer to paint things. We just need to put objects on it and press one of the buttons,False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "embiggenator monitor", The embiggenator monitor controls the embiggenator to make small objects big - or big objects small!, False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "everything's a carrot machine", The carrot machine turns everything into a carrot! Just put an object in the machine and tell me to turn it on!, False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "freeze ray monitor", The freeze ray monitor controls the freeze ray - and chills anything that is on the blue shelf., False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "freeze ray",The freeze ray freezes things! If we put something on the blue shelf - we can use the blue computer to turn on the freeze ray!, False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "gravity monitor", The gravity monitor controls the gravity flipper. Turn it on and everything will go on the ceiling!, False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "gravity pad",You can put things on the gravity pad and flip the gravity upside down so everything is on the ceiling!,False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "hammer", Hammers are useful to break objects such as bowls plates and floppy disks., False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "laser monitor", The laser monitor can activate the laser to heat anything on the red shelf., False -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "laser monitor", The laser monitor activates the laser to heat anything on the red shelf., False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "portal monitor", The black monitor can turn the portal generator on and off - creating windows to the past and the future!, False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "printer cartridge",Printer cartridges go in the 3D printer. To use the 3D printer - we just put the cartridge in and turn it on., False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "printer", The 3D printer can print hammers - mugs - levers - and all sorts of things! To use it - we just need to find a printer cartridge to insert then turn it on!, False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "robot arm", The robot arm can move large boxes out of the way!, False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "sticky note", Sticky notes have useful hints and clues to complete the mission. Try read a sticky note when you're not sure what to do!,False - -verbal_interaction_intent_type == "ask_about_affordance" and verbal_interaction_intent_entity == "time machine",The time machine can restore objects to their original state! We can turn back time to restore a broken bowl or turn a carrot back into whatever it was before!,False - -verbal_interaction_intent_type == "ask_about_appearance" and verbal_interaction_intent_entity != null and verbal_interaction_intent_entity !~ "(portal monitor|freeze ray monitor|laser monitor|embiggenator monitor|gravity monitor)",I am unaware of any special features of the {verbal_interaction_intent_entity},False - -verbal_interaction_intent_type == "ask_about_appearance" and verbal_interaction_intent_entity =~ "(embiggenator monitor)", The embiggenator monitor is the pink one!,False - -verbal_interaction_intent_type == "ask_about_appearance" and verbal_interaction_intent_entity =~ "(freeze ray monitor)", The freeze ray monitor is blue,False - -verbal_interaction_intent_type == "ask_about_appearance" and verbal_interaction_intent_entity =~ "(gravity monitor)", The gravity flipper monitor is green,False - -verbal_interaction_intent_type == "ask_about_appearance" and verbal_interaction_intent_entity =~ "(laser monitor)", The laser monitor is red,False - -verbal_interaction_intent_type == "ask_about_appearance" and verbal_interaction_intent_entity =~ "(portal monitor)", The portal monitor is black,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Embiggenator" and inventory_entity != null,Do we want to use the embiggenator on the {inventory_entity}?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Embiggenator" and inventory_entity != null,I have an idea - shall we use the embiggenator on the {inventory_entity}?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Embiggenator" and inventory_entity != null,Shall we use the embiggenator on the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Everything's A Carrot Machine" and inventory_entity != null,Do you want me to turn the {inventory_entity} into a carrot?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Everything's A Carrot Machine" and inventory_entity != null,Are we turning the {inventory_entity} into a carrot?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Freezer" and inventory_entity != null,Do you want me to put the {inventory_entity} in the freezer?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Freezer" and inventory_entity != null,Do you want me to place the {inventory_entity} in the freezer?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Freezer" and inventory_entity != null,Should I place the {inventory_entity} into the freezer?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Freezer" and inventory_entity != null,Do you want me to place the {inventory_entity} into the freezer?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Fridge" and inventory_entity != null,Should I place the {inventory_entity} into the fridge?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Fuse Box",Do you want me to use the fuse box?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Fuse Box",Should I use the fuse box?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Gravity Pad" and inventory_entity != null,Do you want me to activate the gravity flipper?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Gravity Pad" and inventory_entity != null,I think I know what to do - do you want me to use the gravity flipper on the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Microwave" and inventory_entity != null,Shall I use the microwave to heat up the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Printer" and inventory_entity == "Printer Cartridge",Do you want me to 3D print whats on the cartridge?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Printer" and inventory_entity == "Printer Cartridge",Do you want me use the 3D printer with the cartridge?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Robot Arm",Do you want me to move the crate with the robot arm?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Robot Arm",Do you want me to activate the robot arm?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Robot Arm",Got to the robot arm - should I toggle it?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Robot Arm",Got to the robot arm - should I activate it?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Sink" and inventory_entity == "Plate",I see what we are trying to do here. Do you want me to clean the plate?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Sink" and inventory_entity == "Plate",Do you want me to clean the plate?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Sink" and inventory_entity =~ "(Bowl|Mug|Coffee Pot)",Do you want me to fill up the {inventory_entity}?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Sink" and inventory_entity =~ "(Bowl|Mug|Coffee Pot)",Do you want me to fill the {inventory_entity} with water?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "GotoObject" and interaction_action_entity == "Sink" and inventory_entity =~ "(Bowl|Mug|Coffee Pot)",Do you want me to fill the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Embiggenator" and inventory_entity != null,Do you want me to activate the embiggenator?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Embiggenator" and inventory_entity != null,Do you want me to toggle the embiggenator?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Embiggenator" and inventory_entity != null,Shall we use the embiggenator on the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Freeze Ray Shelf" and inventory_entity != null,I think I know what to do - do you want me to freeze the {inventory_entity}?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Freeze Ray Shelf" and inventory_entity != null,I see what we are trying to do here. Shall we freeze the {inventory_entity}?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Freeze Ray Shelf" and inventory_entity != null,Shall we freeze the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Gravity Pad" and inventory_entity != null,Do you want me to activate the gravity flipper?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Gravity Pad" and inventory_entity != null,Shall we use the gravity flipper on the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Laser Shelf" and inventory_entity != null,Shall we heat up the {inventory_entity}?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Laser Shelf" and inventory_entity != null,Shall we heat the {inventory_entity}?,False -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Laser Shelf" and inventory_entity != null,Do you want me to heat the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type == "Place" and interaction_action_entity == "Microwave" and inventory_entity != null,Shall I use the microwave to heat up the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type =~ "(GotoObject|Place)" and interaction_action_entity == "Time Machine" and inventory_entity != null,Do you want me to use the time machine on the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type =~ "(GotoObject|Place)" and interaction_action_entity == "Time Machine" and inventory_entity == "Carrot",Do you want me to turn the carrot back to its original object?,False - -verbal_interaction_intent_type == "confirm_before_plan" and interaction_action_type =~ "(GotoObject|Place)" and interaction_action_entity == "Time Machine" and inventory_entity =~ "(Bowl|Trophy|Record|Floppy Disk)",Do you want me to repair the {inventory_entity}?,False - -verbal_interaction_intent_type == "confirm_before_search" and verbal_interaction_intent_entity != null,The {verbal_interaction_intent_entity} is not in my view. I should first look for it - right?,False -verbal_interaction_intent_type == "confirm_before_search" and verbal_interaction_intent_entity != null,I don't see the {verbal_interaction_intent_entity} in front of me. Do you want me to search for it first?,False -verbal_interaction_intent_type == "confirm_before_search" and verbal_interaction_intent_entity != null,I don't see the {verbal_interaction_intent_entity} in front of me - should I start by finding it?,False -verbal_interaction_intent_type == "confirm_before_search" and verbal_interaction_intent_entity != null,My vision system is not detecting the {verbal_interaction_intent_entity} in front of me - should I search the room for it?,False - -interaction_action_type == null and verbal_interaction_intent_type == "confirm_before_plan" and verbal_interaction_intent_entity == "BreakRoom" and physical_interaction_intent_type == "search" and physical_interaction_intent_entity == "Sink",The sink should be in the breakroom. Do you want me to search for it in the breakroom?,False -interaction_action_type == null and verbal_interaction_intent_type == "confirm_before_plan" and verbal_interaction_intent_entity == "BreakRoom" and physical_interaction_intent_type == "search" and physical_interaction_intent_entity == "Vending Machine",The vending machine should be in the breakroom. Do you want me to search for it in the breakroom?,False -interaction_action_type == null and verbal_interaction_intent_type == "confirm_before_plan" and verbal_interaction_intent_entity == "BreakRoom" and physical_interaction_intent_type == "search" and physical_interaction_intent_entity == "Coffee Maker",The coffee maker should be in the breakroom. Do you want me to search for it in the breakroom?,False -interaction_action_type == null and verbal_interaction_intent_type == "confirm_before_plan" and verbal_interaction_intent_entity == "Lab2" and physical_interaction_intent_type == "search" and physical_interaction_intent_entity == "Embiggenator",The embiggenator should be in the quantum lab. Do you want me to search for it in the quantum lab?,False -interaction_action_type == null and verbal_interaction_intent_type == "confirm_before_plan" and verbal_interaction_intent_entity == "Lab2" and physical_interaction_intent_type == "search" and physical_interaction_intent_entity == "Embiggenator Monitor",The pink computer should be in the quantum lab. Do you want me to search for it in the quantum lab?,False - -verbal_interaction_intent_type == "generic_success",Now what?,False -verbal_interaction_intent_type == "generic_success",So what should I do next?,False -verbal_interaction_intent_type == "generic_success",Alright - what's next?,False -verbal_interaction_intent_type == "generic_success",So what are we doing next?,False -verbal_interaction_intent_type == "generic_success",So what am I doing next?,False -verbal_interaction_intent_type == "generic_success",So what's next on the task list?,False - -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom",Made it!,False -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom",So what should I do here?,False - -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom" and current_room == interaction_action_entity and current_room != null,Staying in the {current_room} - now what?,False -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom" and current_room == interaction_action_entity and current_room != null,Still in the {current_room} - what next?,False - -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom" and current_room == interaction_action_entity and interaction_action_entity != null,Staying in the {interaction_action_entity},True - -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom" and interaction_action_entity != null,In the {interaction_action_entity} - now what?,False -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom" and interaction_action_entity != null,I'm in the {interaction_action_entity},False -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom" and interaction_action_entity != null,I got to the {interaction_action_entity},False - -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom" and interaction_action_entity != null and interaction_action_entity in visited_room_counter and visited_room_counter[interaction_action_entity] > 1,Back in the {interaction_action_entity},False - -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom" and interaction_action_entity != null and interaction_action_entity not in visited_room_counter and current_room != null,In the {interaction_action_entity}! What's next?,False -verbal_interaction_intent_type == null and interaction_action_type == "GotoRoom" and interaction_action_entity != null and interaction_action_entity not in visited_room_counter,So what are we doing in the {interaction_action_entity}?,False - -verbal_interaction_intent_type =~ "(ask_about_affordance|ask_about_location)" and verbal_interaction_intent_entity =~ "(dino|dinosaur)", We can feed the dinosaur if we set the portal to the past! But we should only go to the portal if we have something to feed the dinosaur with - otherwise it might eat us!, False - -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity != null and not verbal_interaction_intent_entity =~ "(wall shelf|toaster|microwave|milk|cereal box|cake|burger|sandwich|coffee beans|dino|dinosaur)",I'm sorry - but I don't rememember seeing any {verbal_interaction_intent_entity} - You can ask me to search for it., False - -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity =~ "(burger|sandwich)" and current_room != "Lab2",We might find a {verbal_interaction_intent_entity} inside containers like the fridge or the microwave. So what's next?,False - -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity =~ "(burger|sandwich)" and current_room == "Lab2",I've seen {verbal_interaction_intent_entity}s on the gravity flipper before. If not - it might be in the fridge or even in the microwave. What do you want to do next?,False - -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity =~ "(cake)" and current_room == "Lab2",There might be a cake on the gravity flipper. If not - we can try the fridge or the freezer in the break room., False -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity =~ "(cake)" and current_room != "Lab2",The {verbal_interaction_intent_entity} may be placed inside the freezer located in the break room.,False - -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity =~ "(coffee beans)",I've seen coffee beans show up in the break room cabinets or on the gravity flipper or just literally anywhere! We could always make some by pouring some coffee in the coffee composer too.,False - -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity =~ "(wall shelf)",Both the red and blue shelves should be in the robotics lab.,False -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity =~ "(wall shelf)",Red and blue shelves should be in the robotics lab.,False - -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity =~ "(toaster|microwave)",The {verbal_interaction_intent_entity} should be in the break room,False - -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity =~ "(milk|cereal box)" and current_room != "Lab2",The {verbal_interaction_intent_entity} might be in the fridge in the breakroom., False -verbal_interaction_intent_type == "ask_about_location" and verbal_interaction_intent_entity =~ "(milk|cereal box)" and current_room == "Lab2",The {verbal_interaction_intent_entity} could be on the gravity flipper - or inside the fridge in the break room.,False - -verbal_interaction_intent_type =~ "(ask_about_appearance|ask_about_affordance|ask_about_location)" and verbal_interaction_intent_entity == null, I don't understand which object you're referring to. Can you ask me a different question?,False -verbal_interaction_intent_type =~ "(ask_about_appearance|ask_about_affordance|ask_about_location)" and verbal_interaction_intent_entity == null, I'm not sure which object you're referring to. Can you ask me a different question?,False diff --git a/src/emma_experience_hub/datamodels/simbot/__init__.py b/src/emma_experience_hub/datamodels/simbot/__init__.py index 5916a732..11203b80 100644 --- a/src/emma_experience_hub/datamodels/simbot/__init__.py +++ b/src/emma_experience_hub/datamodels/simbot/__init__.py @@ -15,7 +15,6 @@ SimBotUserIntentType, SimBotVerbalInteractionIntentType, ) -from emma_experience_hub.datamodels.simbot.feedback import SimBotFeedbackRule, SimBotFeedbackState from emma_experience_hub.datamodels.simbot.intents import SimBotAgentIntents, SimBotIntent from emma_experience_hub.datamodels.simbot.request import SimBotRequest from emma_experience_hub.datamodels.simbot.response import SimBotResponse diff --git a/src/emma_experience_hub/datamodels/simbot/feedback.py b/src/emma_experience_hub/datamodels/simbot/feedback.py deleted file mode 100644 index 17ddce4f..00000000 --- a/src/emma_experience_hub/datamodels/simbot/feedback.py +++ /dev/null @@ -1,390 +0,0 @@ -import itertools -import string -from collections import Counter -from typing import Any, Optional - -import orjson -from pydantic import BaseModel, Field, validator -from rule_engine import Context, Rule -from rule_engine.ast import ExpressionBase, StringExpression, SymbolExpression - -from emma_experience_hub.datamodels.simbot.actions import SimBotAction -from emma_experience_hub.datamodels.simbot.agent_memory import get_area_from_compressed_mask -from emma_experience_hub.datamodels.simbot.enums import ( - SimBotActionType, - SimBotAnyUserIntentType, - SimBotEnvironmentIntentType, - SimBotIntentType, - SimBotPhysicalInteractionIntentType, - SimBotVerbalInteractionIntentType, -) -from emma_experience_hub.datamodels.simbot.intents import SimBotIntent -from emma_experience_hub.datamodels.simbot.payloads import SimBotObjectInteractionPayload - - -def get_score_for_rule_expression( - expression: ExpressionBase, left_attr_name: str = "left", right_attr_name: str = "right" -) -> int: - """Get the score for expression within the rule. - - The conditions for each rule are converted into a binary tree. Therefore, we can recursively - iterate over all the nodes within the tree to get the score as the number of conditions in the - rule. Therefore, the more specific the rule the higher its score. - """ - # Try to get the left and right nodes within the expression - left_node: Optional[ExpressionBase] = getattr(expression, left_attr_name, None) - right_node: Optional[ExpressionBase] = getattr(expression, right_attr_name, None) - - # If the left node is a SymBolExpression, we are at the slot name and can go no lower - if isinstance(left_node, SymbolExpression): - return 1 - - # Otherwise, both nodes are not None, we can dig into them further to try and get the score - if left_node is not None and right_node is not None: - left_expression_score = get_score_for_rule_expression( - left_node, left_attr_name, right_attr_name - ) - right_expression_score = get_score_for_rule_expression( - right_node, left_attr_name, right_attr_name - ) - return left_expression_score + right_expression_score - - # If none of the above conditions suit, return 0 for this expression - return 0 - - -def should_rule_be_mandatory(expression: ExpressionBase) -> bool: # noqa: WPS231 - """Determine if the rule is mandatory. - - Mandatory rules include responses that we want actually to include in the selection process no - matter what. These are generally responses where we want to communicate back to the user - something important aka, confirm_before_plan, ask_about_the_game. - """ - # Try to get the left and right nodes within the expression - left_node: Optional[ExpressionBase] = getattr(expression, "left", None) - right_node: Optional[ExpressionBase] = getattr(expression, "right", None) - - # If the left node is a SymbolExpression, we are at the slot name and can go no lower - if isinstance(left_node, SymbolExpression): - # Check the slot name for the verbal interaction intent type - if left_node.name == "verbal_interaction_intent_type": - # Check to see if the slot value is a mandatory one - if isinstance(right_node, StringExpression): - return "confirm_before_plan" in right_node.value or "ask_about" in right_node.value - - # Otherwise, return False since we can go no lower - return False - - # Otherwise, ensure both nodes are not None, and we can dig into them further - if left_node is not None and right_node is not None: - return should_rule_be_mandatory(left_node) or should_rule_be_mandatory(right_node) - - # Otherwise, return False since that's the end. - return False - - -class SimBotFeedbackRule(BaseModel): - """Rule for response generation.""" - - id: int = Field(..., description="Unique rule id") - rule: Rule = Field(..., description="Logical expression of the rule") - response: str = Field( - ..., - description="Response template that can include slots. Slot values are derived from the `SimBotFeedbackState`", - ) - is_lightweight_dialog: bool = Field( - ..., description="Should the response be a lightweight dialog action" - ) - score: int = Field(default=0, description="Determined by the number of conditions in the rule") - is_mandatory: bool = Field( - default=False, - description="Mandatory rules are always included in the candidate pool even if they have already been used", - ) - - class Config: - """Updated config.""" - - arbitrary_types_allowed = True - - @property - def slot_names(self) -> list[str]: - """Get the necessary slot names.""" - slot_names: list[str] = [ - name for _, name, _, _ in string.Formatter().parse(self.response) if name - ] - return slot_names - - def prepare_response(self, slots: Optional[dict[str, str]] = None) -> str: - """Build the response for the given rule. - - If the rule is a template that requires slots to be filled, then slots need to be provided - and this method will raise an exception if all slots are not filled. - """ - if self.slot_names and not slots: - raise AssertionError( - "We should be providing slot-value pairs for the response template." - ) - - if self.slot_names and slots: - return self.response.format(**slots) - - return self.response - - @classmethod - def from_raw(cls, raw_dict: dict[str, str]) -> "SimBotFeedbackRule": - """Parse a dictionary into a SimBotFeedbackRule.""" - # If the rule is unable to resolve symbols, it defaults to None and returns False - engine_context = Context(default_value=None) - rule = Rule(raw_dict["conditions"].lower(), context=engine_context) - - rule_id = int(raw_dict["id"]) - - if not rule.is_valid(rule.text): - raise AssertionError(f"Invalid rule: ID {rule_id} - {rule.text}") - - is_mandatory = should_rule_be_mandatory(rule.statement.expression) - - return cls( - id=rule_id, - rule=rule, - response=raw_dict["response"], - is_lightweight_dialog=raw_dict["is_lightweight"] == "True", - score=len(rule.context.symbols), - is_mandatory=is_mandatory, - ) - - @validator("score", always=True) - @classmethod - def calculate_rule_score(cls, score: int, values: dict[str, Any]) -> int: # noqa: WPS110 - """Calculate the score for the rule.""" - # If the score is not 0, then just return it - if score > 1: - return score - - # Get the rule and make sure it exists - rule: Optional[Rule] = values.get("rule") - if not rule: - raise AssertionError("There should be a rule for this model?") - - # The score for a rule is the number of different criterion within it - score = get_score_for_rule_expression(rule.statement.expression) - - if score < 1: - raise AssertionError("Score should not be less than 1.") - - return score - - def is_query_suitable(self, query: dict[str, Any]) -> bool: - """Evaluate the rule given the query and ensure it is suitable.""" - try: - return self.rule.matches(query) and all(name in query for name in self.slot_names) - except Exception: - return False - - -def turn_requires_lightweight_dialog( - verbal_interaction_intent: Optional[SimBotIntent[SimBotVerbalInteractionIntentType]], - utterance_queue_not_empty: bool, - find_queue_not_empty: bool, - interaction_action: Optional[SimBotAction], -) -> bool: - """Does this turn require a lightweight dialog?""" - # If the verbal interaction intent triggers a question, ignore if the utterance queue is empty - triggers_question = ( - verbal_interaction_intent is not None - and verbal_interaction_intent.type.triggers_question_to_user - ) - utterance_queue_not_empty = utterance_queue_not_empty and not triggers_question - - require_lightweight_dialog = ( - (interaction_action and not interaction_action.is_end_of_trajectory) - or utterance_queue_not_empty - or (find_queue_not_empty) - ) - return require_lightweight_dialog - - -class SimBotFeedbackState(BaseModel): - """Flattened representation of the session state for feedback generation.""" - - # Force query for a lightweight dialog - require_lightweight_dialog: bool = False - - # Session statistics - num_turns: int - - # Location - current_room: str - - # Inventory - inventory_entity: Optional[str] = None - inventory_turn: int - - # Count all of the rooms visited - visited_room_counter: Counter[str] - - # User intent - user_intent_type: Optional[SimBotAnyUserIntentType] = None - - # Environment intent - environment_intent_type: Optional[SimBotEnvironmentIntentType] = None - environment_intent_action_type: Optional[SimBotActionType] = None - environment_intent_entity: Optional[str] = None - - # Interaction intent - physical_interaction_intent_type: Optional[SimBotPhysicalInteractionIntentType] = None - physical_interaction_intent_entity: Optional[str] = None - - # Language Condition intent - verbal_interaction_intent_type: Optional[SimBotVerbalInteractionIntentType] = None - verbal_interaction_intent_entity: Optional[str] = None - - # Current interaction action - interaction_action_type: Optional[SimBotActionType] = None - interaction_action_entity: Optional[str] = None - - # History of actions taken in the session - interaction_action_per_turn: list[SimBotAction] - - # History of interacted entities in the session - interacted_entities_counter: Counter[str] - - # Counter of how many times each action was taken - action_type_counter: Counter[str] - - # History of all the intents per turn in the session - intent_types_per_turn: list[list[SimBotIntentType]] - - # Counter of how many times each AGENT intent was held - intent_type_counter: Counter[str] - - # There are more instructions to execute from the latest user utterance - utterance_queue_not_empty: bool - - # There are more instructions to execute from the find routine - find_queue_not_empty: bool = False - previous_find_queue_not_empty: bool = False - - # History of used rule ids - used_rule_ids: list[int] = Field(default_factory=list) - - # History of agent responses within local window - # This allows us to ensure words are not being repeated across responses, which gives us - # more control and allows for more natural responses. - agent_responses_since_last_user_utterance: str = "" - - current_turn_has_user_utterance: bool = False - - object_area: Optional[float] = None - - class Config: - """Config for the model.""" - - json_encoders = { - # Use the action type name when converting to the JSON response - SimBotActionType: lambda action_type: action_type.name, - # Use the intent type name when converting to the JSON response - SimBotIntentType: lambda intent_type: intent_type.name, - } - - @classmethod - def from_all_information( - cls, - num_turns: int, - current_room: str, - user_intent_type: Optional[SimBotAnyUserIntentType], - environment_intent: Optional[SimBotIntent[SimBotEnvironmentIntentType]], - physical_interaction_intent: Optional[SimBotIntent[SimBotPhysicalInteractionIntentType]], - verbal_interaction_intent: Optional[SimBotIntent[SimBotVerbalInteractionIntentType]], - interaction_action: Optional[SimBotAction], - current_room_per_turn: list[str], - interaction_action_per_turn: list[SimBotAction], - intent_types_per_turn: list[list[SimBotIntentType]], - utterance_queue_not_empty: bool, - find_queue_not_empty: bool, - previous_find_queue_not_empty: bool, - used_rule_ids: list[int], - inventory_turn: int, - inventory_entity: Optional[str], - agent_responses_since_last_user_utterance: list[str], - current_turn_has_user_utterance: bool, - ) -> "SimBotFeedbackState": - """Create the state in a simple way.""" - # Conditions under which we should try to find a lightweight dialog action - require_lightweight_dialog = turn_requires_lightweight_dialog( - interaction_action=interaction_action, - utterance_queue_not_empty=utterance_queue_not_empty, - find_queue_not_empty=find_queue_not_empty, - verbal_interaction_intent=verbal_interaction_intent, - ) - - object_area = None - interaction_action_has_bbox = interaction_action is not None and isinstance( - interaction_action.payload, SimBotObjectInteractionPayload - ) - if interaction_action_has_bbox: - object_area = get_area_from_compressed_mask(interaction_action.payload.object.mask) # type: ignore[union-attr] - - return cls( - # Require a lightweight dialog action when the model does not decode a dict[str, Any]: - """Convert the state to a dictionary for the feedback engine.""" - model_as_json = self.json( - exclude_unset=True, exclude_defaults=True, exclude_none=True - ).lower() - model_as_dict = orjson.loads(model_as_json) - return model_as_dict diff --git a/src/emma_experience_hub/datamodels/simbot/session.py b/src/emma_experience_hub/datamodels/simbot/session.py index 8654253b..027db3cf 100644 --- a/src/emma_experience_hub/datamodels/simbot/session.py +++ b/src/emma_experience_hub/datamodels/simbot/session.py @@ -27,7 +27,6 @@ SimBotPhysicalInteractionIntentType, SimBotVerbalInteractionIntentType, ) -from emma_experience_hub.datamodels.simbot.feedback import SimBotFeedbackState from emma_experience_hub.datamodels.simbot.intents import SimBotIntent from emma_experience_hub.datamodels.simbot.payloads import ( SimBotAuxiliaryMetadataUri, @@ -372,14 +371,6 @@ def utterances(self) -> list[DialogueUtterance]: return utterances - @property - def feedback_rule_id(self) -> Optional[int]: - """Get the id for the rule used to generate the agent feedback for that turn.""" - if self.actions.dialog is not None: - return self.actions.dialog.payload.rule_id - - return None - def convert_to_simbot_response(self) -> SimBotResponse: """Convert the session turn to a SimBotResponse, to be returned to the API.""" actions: list[SimBotAction] = self.actions.to_list() @@ -637,37 +628,6 @@ def update_agent_memory(self, extracted_features: list[EmmaExtractedFeatures]) - extracted_features=extracted_features, ) - def to_feedback_state(self) -> SimBotFeedbackState: - """Convert the session to the simplified state.""" - return SimBotFeedbackState.from_all_information( - num_turns=len(self.turns), - current_room=self.current_turn.environment.current_room, - user_intent_type=self.current_turn.intent.user, - environment_intent=self.current_turn.intent.environment, - physical_interaction_intent=self.current_turn.intent.physical_interaction, - verbal_interaction_intent=self.current_turn.intent.verbal_interaction, - interaction_action=self.current_turn.actions.interaction, - current_room_per_turn=[turn.environment.current_room for turn in self.turns], - interaction_action_per_turn=[ - turn.actions.interaction for turn in self.turns if turn.actions.interaction - ], - intent_types_per_turn=[turn.intent.all_intent_types for turn in self.turns], - utterance_queue_not_empty=self.current_state.utterance_queue.is_not_empty, - find_queue_not_empty=self.current_state.find_queue.is_not_empty, - previous_find_queue_not_empty=self.previous_turn.state.find_queue.is_not_empty - if self.previous_turn - else False, - used_rule_ids=self._get_used_feedback_rule_ids(), - inventory_turn=self.inventory.turn_idx, - inventory_entity=self.inventory.entity, - agent_responses_since_last_user_utterance=[ - turn.actions.dialog.utterance - for turn in self.get_turns_since_last_original_user_utterance() - if turn.actions.dialog is not None - ], - current_turn_has_user_utterance=self.current_turn.speech is not None, - ) - @staticmethod def get_dialogue_history_from_session_turns( # noqa: WPS602 turns: list[SimBotSessionTurn], *, include_agent_responses: bool = True @@ -734,10 +694,3 @@ def get_environment_state_history_from_turns( # noqa: WPS602 # Ensure the environment history is sorted properly and return them return list(dict(sorted(environment_history.items())).values()) - - def _get_used_feedback_rule_ids(self) -> list[int]: - """Get all the rule IDs that were used to generate responses.""" - rule_ids = [ - turn.feedback_rule_id for turn in self.turns if turn.feedback_rule_id is not None - ] - return rule_ids diff --git a/src/emma_experience_hub/parsers/simbot/__init__.py b/src/emma_experience_hub/parsers/simbot/__init__.py index ce942fe1..e83e30fb 100644 --- a/src/emma_experience_hub/parsers/simbot/__init__.py +++ b/src/emma_experience_hub/parsers/simbot/__init__.py @@ -1,9 +1,6 @@ from emma_experience_hub.parsers.simbot.action_predictor_output import ( SimBotActionPredictorOutputParser, ) -from emma_experience_hub.parsers.simbot.feedback_from_session_context import ( - SimBotFeedbackFromSessionStateParser, -) from emma_experience_hub.parsers.simbot.intent_from_action_status import ( SimBotIntentFromActionStatusParser, ) diff --git a/src/emma_experience_hub/parsers/simbot/feedback_from_session_context.py b/src/emma_experience_hub/parsers/simbot/feedback_from_session_context.py deleted file mode 100644 index ab7e7df0..00000000 --- a/src/emma_experience_hub/parsers/simbot/feedback_from_session_context.py +++ /dev/null @@ -1,117 +0,0 @@ -import itertools -import random -from collections.abc import Iterator -from concurrent.futures import Future, ThreadPoolExecutor, as_completed -from typing import Optional - -from loguru import logger -from rule_engine import Rule - -from emma_experience_hub.constants.simbot import get_feedback_rules -from emma_experience_hub.datamodels.simbot import SimBotFeedbackRule, SimBotFeedbackState -from emma_experience_hub.parsers.parser import Parser - - -class SimBotFeedbackFromSessionStateParser(Parser[SimBotFeedbackState, SimBotFeedbackRule]): - """Get the best response for the current session feedback state.""" - - _default_rule: SimBotFeedbackRule = SimBotFeedbackRule( - id=1, - rule=Rule(text="require_lightweight_dialog == False"), - response="'", - is_lightweight_dialog=False, - score=1, - ) - - def __init__( - self, rules: list[SimBotFeedbackRule], _max_workers: Optional[int] = None - ) -> None: - self._rules = rules - self._max_workers = _max_workers - - def __call__(self, session_state: SimBotFeedbackState) -> SimBotFeedbackRule: - """Get the best feedback from the current context.""" - candidate_rules = self._get_all_compatible_rules(session_state) - selected_rule = self._select_feedback_rule(candidate_rules, session_state.used_rule_ids) - return selected_rule - - @classmethod - def from_rules_csv(cls) -> "SimBotFeedbackFromSessionStateParser": - """Instantiate the class from the rules CSV file.""" - rule_data = get_feedback_rules() - - with ThreadPoolExecutor() as executor: - rules = list(executor.map(SimBotFeedbackRule.from_raw, rule_data)) - - logger.debug(f"Loaded {len(rules)} feedback rules.") - return cls(rules=rules) - - def _get_all_compatible_rules(self, state: SimBotFeedbackState) -> list[SimBotFeedbackRule]: - """Get all of the rules which are compatible with the current state.""" - # Store all the compatible rules with the given query - compatible_rules: list[SimBotFeedbackRule] = [] - - # Try filter rules to remove any that are clearly unneeded - filtered_rules = self._filter_rules(state) - - # Convert the session state to a dictionary we can iterate over. - query_dict = state.to_query() - - # Use multithreading to evaluate every single rule as fast as possible - with ThreadPoolExecutor(self._max_workers) as executor: - future_to_rule: dict[Future[bool], SimBotFeedbackRule] = { - executor.submit(rule.is_query_suitable, query_dict): rule - for rule in filtered_rules - } - - for future in as_completed(future_to_rule): - rule = future_to_rule[future] - - try: - future.result() - except Exception: - logger.exception(f"Failed to check whether rule {rule.id} is compatible") - else: - if future.result(): - compatible_rules.append(rule) # noqa: WPS220 - - logger.debug(f"Got {len(compatible_rules)} matching feedback rules: {compatible_rules}") - return compatible_rules - - def _filter_rules(self, state: SimBotFeedbackState) -> Iterator[SimBotFeedbackRule]: - """Filter the rules if possible to ensure that certain rules are returned.""" - if state.require_lightweight_dialog: - filtered_rules = (rule for rule in self._rules if rule.is_lightweight_dialog) - else: - filtered_rules = (rule for rule in self._rules if not rule.is_lightweight_dialog) - return filtered_rules - - def _select_feedback_rule( - self, candidates: list[SimBotFeedbackRule], used_rule_ids: list[int] - ) -> SimBotFeedbackRule: - """Select the highest-scoring rule from the set of rules.""" - # Try to filter out any rules that have already been used - valid_candidates = [ - rule for rule in candidates if (rule.id not in used_rule_ids) or (rule.is_mandatory) - ] - - if not valid_candidates: - logger.warning("No unused candidate rules! Will need to reuse a response.") - valid_candidates = candidates - - # Group all the rules by their scores - sorted_candidates = sorted(valid_candidates, key=lambda x: x.score, reverse=True) - grouped_candidates = itertools.groupby(sorted_candidates, key=lambda x: x.score) - - try: - # Get the group of rules with the highest score - highest_scoring_rules = list(next(grouped_candidates)[1]) - # Choose one of the rules from the highest-scoring set - selected_rule = random.choice(highest_scoring_rules) - except (StopIteration, IndexError): - # If the list of rules is empty for some reason, then just return the default rule - selected_rule = self._default_rule - logger.error(f"[NLG] No rules to choose, therefore using default {selected_rule}") - - logger.debug(f"[NLG] Selected rule {selected_rule}") - return selected_rule diff --git a/src/emma_experience_hub/pipelines/simbot/__init__.py b/src/emma_experience_hub/pipelines/simbot/__init__.py index f652572e..c6db2068 100644 --- a/src/emma_experience_hub/pipelines/simbot/__init__.py +++ b/src/emma_experience_hub/pipelines/simbot/__init__.py @@ -4,9 +4,6 @@ from emma_experience_hub.pipelines.simbot.agent_intent_selection import ( SimBotAgentIntentSelectionPipeline, ) -from emma_experience_hub.pipelines.simbot.agent_language_generation import ( - SimBotAgentLanguageGenerationPipeline, -) from emma_experience_hub.pipelines.simbot.environment_error_catching import ( SimBotEnvironmentErrorCatchingPipeline, ) diff --git a/src/emma_experience_hub/pipelines/simbot/agent_language_generation.py b/src/emma_experience_hub/pipelines/simbot/agent_language_generation.py deleted file mode 100644 index c90f8905..00000000 --- a/src/emma_experience_hub/pipelines/simbot/agent_language_generation.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import Optional - -from loguru import logger - -from emma_experience_hub.constants.simbot import ACTION_SYNONYMS_FOR_GENERATION, ROOM_SYNONYNMS -from emma_experience_hub.datamodels.simbot import ( - SimBotActionType, - SimBotDialogAction, - SimBotFeedbackRule, - SimBotFeedbackState, - SimBotSession, -) -from emma_experience_hub.datamodels.simbot.payloads import SimBotDialogPayload -from emma_experience_hub.parsers.simbot.feedback_from_session_context import ( - SimBotFeedbackFromSessionStateParser, -) - - -class SimBotAgentLanguageGenerationPipeline: - """Generate language for the agent to say to the user.""" - - _default_entity = "object" - - def __init__(self, *, prevent_default_response_as_lightweight: bool = True) -> None: - self._feedback_parser = SimBotFeedbackFromSessionStateParser.from_rules_csv() - self._default_utterance = "'" - - self._prevent_default_response_as_lightweight = prevent_default_response_as_lightweight - - def run(self, session: SimBotSession) -> Optional[SimBotDialogAction]: - """Generate an utterance to send back to the user.""" - feedback_state = session.to_feedback_state() - - matching_rule = self._feedback_parser(feedback_state) - - action = self._generate_dialog_action(matching_rule, feedback_state) - return action - - def _generate_dialog_action( - self, rule: SimBotFeedbackRule, feedback_state: SimBotFeedbackState - ) -> Optional[SimBotDialogAction]: - """Generate a dialog action.""" - utterance = self._generate_utterance(rule, feedback_state) - - # Dont return the default utterance for lightweight dialog actions. - if self._prevent_default_response_as_lightweight: - if feedback_state.require_lightweight_dialog and utterance == self._default_utterance: - return None - - # Determine the dialog type for the response - dialog_type = ( - SimBotActionType.LightweightDialog - if feedback_state.require_lightweight_dialog - else SimBotActionType.Dialog - ) - - return SimBotDialogAction( - id=0, - raw_output=utterance, - type=dialog_type, - payload=SimBotDialogPayload(value=utterance, rule_id=rule.id), - ) - - def _generate_utterance( - self, rule: SimBotFeedbackRule, feedback_state: SimBotFeedbackState - ) -> str: - """Generate utterance from the rule and the feedback state.""" - # Build the query dictionary from the feedback state - query_dict = feedback_state.to_query() - logger.debug(f"Feedback Query Dict: {query_dict}") - - # Create the slot value pairs for the response - slot_value_pairs = { - slot_name: self._process_slot_value(query_dict[slot_name]) - for slot_name in rule.slot_names - } - - # Build the response itself - utterance = rule.response.format(**slot_value_pairs) - - logger.debug(f"[NLG] Generated utterance from rule {rule.id}: {utterance}") - return utterance - - def _process_slot_value(self, entity: Optional[str]) -> str: - """Return a synonym for the slot value if possible.""" - if not entity: - return self._default_entity - - action_synonym = ACTION_SYNONYMS_FOR_GENERATION.get(entity, None) - - if action_synonym is not None: - return action_synonym - - return ROOM_SYNONYNMS.get(entity, entity) diff --git a/tests/pipelines/simbot/test_language_generator.py b/tests/pipelines/simbot/test_language_generator.py deleted file mode 100644 index e350cf77..00000000 --- a/tests/pipelines/simbot/test_language_generator.py +++ /dev/null @@ -1,96 +0,0 @@ -from collections import Counter - -from pytest_cases import fixture - -from emma_experience_hub.datamodels.simbot import SimBotFeedbackState -from emma_experience_hub.parsers.simbot.feedback_from_session_context import ( - SimBotFeedbackFromSessionStateParser, -) - - -MIN_RULE_ID = 2 - - -@fixture(scope="module") -def rule_parser() -> SimBotFeedbackFromSessionStateParser: - """Instantiate the rule parser.""" - return SimBotFeedbackFromSessionStateParser.from_rules_csv() - - -def test_all_rules_are_valid(rule_parser: SimBotFeedbackFromSessionStateParser) -> None: - assert rule_parser._rules - - -def test_ensure_no_rule_id_is_below_minumum( - rule_parser: SimBotFeedbackFromSessionStateParser, -) -> None: - all_rule_ids = [rule.id for rule in rule_parser._rules] - - assert min(all_rule_ids) == MIN_RULE_ID - - -def test_all_rule_ids_are_unique(rule_parser: SimBotFeedbackFromSessionStateParser) -> None: - ids_counter = Counter([rule.id for rule in rule_parser._rules]) - - # Get IDs which appear more than once in the list - repeated_ids = [rule_id for (rule_id, count) in ids_counter.items() if count > 1] - - # Assert that the list of repeated IDs IS empty - assert not repeated_ids - - -def test_rule_ids_are_consecutive(rule_parser: SimBotFeedbackFromSessionStateParser) -> None: - all_rule_ids = [rule.id for rule in rule_parser._rules] - all_rule_ids.sort() - - # Create a simple list of all numbers from 0 to the total number of rules - consecutive_numbers = range(MIN_RULE_ID, len(all_rule_ids)) - - for actual_rule_id, expected_rule_id in zip(all_rule_ids, consecutive_numbers): - assert ( - actual_rule_id == expected_rule_id - ), f"Rule {actual_rule_id} should be {expected_rule_id}" - - -def test_response_slots_in_all_rules(rule_parser: SimBotFeedbackFromSessionStateParser) -> None: - """Test that all slots in resposes are included in the rule.""" - rules_and_slots = [ - (rule.rule.text, rule.slot_names) for rule in rule_parser._rules if rule.slot_names - ] - for rule_text, slots in rules_and_slots: - rule_words = rule_text.split() - assert all(slot_name in rule_words for slot_name in slots) - - -def test_all_response_slots_are_validated_by_rules( - rule_parser: SimBotFeedbackFromSessionStateParser, -) -> None: - """Test that all slots in resposes are included in the rule.""" - rules_and_slots = [ - (rule.rule.text, rule.slot_names) for rule in rule_parser._rules if rule.slot_names - ] - for rule_text, slots in rules_and_slots: - # Each slot name must be used in the rule so that it exists in some way - assert all(slot_name in rule_text for slot_name in slots) - - # Any slot name in the rule must not be checking for it to be equal to None - assert not any(f"{slot_name} == null" in rule_text for slot_name in slots) - - -def test_all_rule_symbols_in_state(rule_parser: SimBotFeedbackFromSessionStateParser) -> None: - """Test that all rule symbols appear in the state.""" - state_fields = set(SimBotFeedbackState.__fields__.keys()) - for rule in rule_parser._rules: - # Get the rule symbols - symbols = rule.rule.context.symbols - assert symbols.issubset(state_fields) - - -# @given(session=simbot_session()) -# def test_can_get_rule_from_example_state( -# rule_parser: SimBotFeedbackFromSessionStateParser, session: SimBotSession -# ) -> None: -# feedback_state = session.to_feedback_state() -# rule = rule_parser(feedback_state) - -# assert rule is not None