diff --git a/.gitignore b/.gitignore index d21dae37e..490e48bfb 100644 --- a/.gitignore +++ b/.gitignore @@ -19,10 +19,11 @@ uv.lock # Misc trajectory_logs/ +/fle/data/replays/stdout_logs/ **/.claude/settings.local.json *.mp4 *.jsonl *.db #fle -.fle/ \ No newline at end of file +.fle/ diff --git a/fle/data/replays/action_converter.py b/fle/data/replays/action_converter.py new file mode 100644 index 000000000..7c5c45347 --- /dev/null +++ b/fle/data/replays/action_converter.py @@ -0,0 +1,416 @@ +""" +Handles conversion between different action argument formats. +""" + +import json +import re +import ast +from typing import Dict, Any, List + +from fle.env.entities import Position, Direction, PlaceholderEntity +from fle.env.game_types import prototype_by_name, Technology + + +class ActionConverter: + """Handles conversion between different action argument formats.""" + + @staticmethod + def parse_function_call(call_string: str) -> tuple[str, Dict[str, Any]]: + """Parse a function call string into function name and arguments.""" + # Extract function name and arguments using regex + match = re.match(r"(\w+)\((.*)\)", call_string) + if not match: + raise ValueError(f"Invalid function call format: {call_string}") + + func_name = match.group(1) + args_string = match.group(2) + + # Parse arguments + args = {} + if args_string.strip(): + # Split arguments by comma, but handle nested structures + arg_pairs = [] + paren_count = 0 + bracket_count = 0 + brace_count = 0 + current_arg = "" + + for char in args_string: + if char in "([{": + if char == "(": + paren_count += 1 + elif char == "[": + bracket_count += 1 + elif char == "{": + brace_count += 1 + elif char in ")]}": + if char == ")": + paren_count -= 1 + elif char == "]": + bracket_count -= 1 + elif char == "}": + brace_count -= 1 + elif ( + char == "," + and paren_count == 0 + and bracket_count == 0 + and brace_count == 0 + ): + arg_pairs.append(current_arg.strip()) + current_arg = "" + continue + + current_arg += char + + if current_arg.strip(): + arg_pairs.append(current_arg.strip()) + + # Parse each argument pair + for arg_pair in arg_pairs: + if "=" in arg_pair: + key, value = arg_pair.split("=", 1) + key = key.strip() + value = value.strip() + + # Try to evaluate the value safely + try: + # Handle string literals + if value.startswith("'") and value.endswith("'"): + args[key] = value[1:-1] + elif value.startswith('"') and value.endswith('"'): + args[key] = value[1:-1] + # Handle lists and dicts (for items parameter) + elif value.startswith("[") or value.startswith("{"): + args[key] = ast.literal_eval(value) + # Handle numbers + elif value.replace(".", "").replace("-", "").isdigit(): + if "." in value: + args[key] = float(value) + else: + args[key] = int(value) + else: + # Default to string + args[key] = value + except (ValueError, SyntaxError): + # If evaluation fails, keep as string + args[key] = value + + return func_name, args + + @staticmethod + def convert_legacy_args_to_tool_args( + func_name: str, args: Dict[str, Any] + ) -> Dict[str, Any]: + """Convert legacy run_actions.py argument format to tool argument format.""" + + if func_name == "harvest_resource": + return { + "position": Position(args["x"], args["y"]), + "quantity": args.get("quantity", 1), + } + + elif func_name == "move_to": + return {"position": Position(args["end_x"], args["end_y"])} + + elif func_name == "extract_item": + # Parse the items JSON and get the first item + items_str = args["items"].replace("'", '"') + items_list = json.loads(items_str) + if items_list: + item_info = items_list[0] + entity = ActionConverter._get_prototype(item_info["item"]) + + return { + "entity": entity, + "source": Position(args["entity_x"], args["entity_y"]), + "quantity": item_info.get("count", 1), + } + + elif func_name == "place_entity": + # For place_entity, convert item to entity and create Position + item_name = args["item"] + entity = ActionConverter._get_prototype(item_name) + + result = {"entity": entity, "position": Position(args["x"], args["y"])} + + if "direction" in args: + result["direction"] = Direction.from_int(int(args["direction"])) + + return result + + elif func_name == "craft_item": + result = {} + if "recipe" in args: + result["entity"] = args["recipe"] + if "count" in args: + result["quantity"] = args["count"] + return result + + elif func_name == "insert_item": + # Parse the items JSON and get the first item + items_str = args["items"].replace("'", '"') + items_list = json.loads(items_str) + if items_list: + item_info = items_list[0] + entity = ActionConverter._get_prototype(item_info["item"]) + + return { + "entity": entity, + "target": Position(args["entity_x"], args["entity_y"]), + "quantity": item_info.get("count", 1), + } + + elif func_name == "pickup_entity": + entity_name = args["entity"] + entity = ActionConverter._get_prototype(entity_name) + + return {"entity": entity, "position": Position(args["x"], args["y"])} + + elif func_name == "set_research": + technology_name = args["technology"] + + try: + if hasattr(technology_name, "value"): + technology = technology_name + else: + technology = Technology(technology_name) + except (ValueError, AttributeError): + technology = technology_name + + return {"technology": technology} + + elif func_name == "set_entity_recipe": + # Convert entity name and position to PlaceholderEntity, and recipe to RecipeName + entity_name = args["entity"] + recipe_name = args["new_recipe"] + position = Position(args["x"], args["y"]) + + # Create a placeholder entity at the given position + entity = PlaceholderEntity(name=entity_name, position=position) + + # Convert recipe name to appropriate type - try Prototype first, then RecipeName + from fle.env.game_types import RecipeName, Prototype + + prototype = None + + # First try to find it in Prototype enum (most recipes are here) + try: + for proto in Prototype: + if proto.value[0] == recipe_name: + prototype = proto + break + except Exception: + pass + + # If not found in Prototype, try RecipeName enum (for fluid recipes) + if prototype is None: + try: + prototype = RecipeName(recipe_name) + except (ValueError, AttributeError): + # Fall back to string if neither enum works + prototype = recipe_name + + return {"entity": entity, "prototype": prototype} + + # Default: return args as-is + return args + + @staticmethod + def execute_tool_call_in_batch( + namespace, func_name: str, args: Dict[str, Any], tick: int + ) -> Any: + """Execute a tool call with proper argument conversion.""" + if not hasattr(namespace, func_name): + print(f"Warning: Function '{func_name}' not found in namespace") + return None + + func = getattr(namespace, func_name) + filtered_args = { + k: v for k, v in args.items() if k not in ["start_tick", "end_tick", "tick"] + } + + # Delegate to specific converters + converter_map = { + "move_to": ActionConverter._convert_move_to, + "harvest_resource": ActionConverter._convert_harvest_resource, + "place_entity": ActionConverter._convert_place_entity, + "craft_item": ActionConverter._convert_craft_item, + "insert_item": ActionConverter._convert_insert_item, + "extract_item": ActionConverter._convert_extract_item, + "pickup_entity": ActionConverter._convert_pickup_entity, + "set_research": ActionConverter._convert_set_research, + "inspect_inventory": ActionConverter._convert_inspect_inventory, + "set_entity_recipe": ActionConverter._convert_set_entity_recipe, + } + + converter = converter_map.get(func_name) + if converter: + return converter(func, filtered_args, tick) + + print(f"Warning: Unhandled function '{func_name}' with args {filtered_args}") + return None + + @staticmethod + def _convert_move_to(func, args: Dict[str, Any], tick: int): + if "end_x" in args and "end_y" in args: + return func(position=Position(args["end_x"], args["end_y"]), tick=tick) + + @staticmethod + def _convert_harvest_resource(func, args: Dict[str, Any], tick: int): + if "x" in args and "y" in args: + return func(position=Position(args["x"], args["y"]), tick=tick) + + @staticmethod + def _convert_place_entity(func, args: Dict[str, Any], tick: int): + if "item" in args and "x" in args and "y" in args: + entity = ActionConverter._get_prototype(args["item"]) + position = Position(args["x"], args["y"]) + kwargs = {"entity": entity, "position": position, "tick": tick} + + if "direction" in args: + kwargs["direction"] = Direction.from_int(int(args["direction"])) + + return func(**kwargs) + + @staticmethod + def _convert_craft_item(func, args: Dict[str, Any], tick: int): + entity = args.get("recipe") + quantity = args.get("count", 1) + return func(entity=entity, quantity=quantity, tick=tick) + + @staticmethod + def _convert_insert_item(func, args: Dict[str, Any], tick: int): + if "items" in args and "entity_x" in args and "entity_y" in args: + items_data = ActionConverter._parse_items_string(args["items"]) + if not items_data: + return None + + item_info = items_data[0] # For now, only handle the first item + entity = ActionConverter._get_prototype(item_info["item"]) + quantity = item_info.get("count", 1) + + target_position = Position(args["entity_x"], args["entity_y"]) + target_entity_name = args.get("entity") + + target = ( + PlaceholderEntity(name=target_entity_name, position=target_position) + if target_entity_name + else target_position + ) + + return func(entity=entity, target=target, quantity=quantity, tick=tick) + + @staticmethod + def _convert_extract_item(func, args: Dict[str, Any], tick: int): + if "items" in args and "entity_x" in args and "entity_y" in args: + items_data = ActionConverter._parse_items_string(args["items"]) + if not items_data: + return None + + if len(items_data) > 1: + print( + f"Warning: Extracting multiple items is not supported, missing: {items_data[1:]}" + ) + + item_info = items_data[0] # For now, only handle the first item + entity = ActionConverter._get_prototype(item_info["item"]) + quantity = item_info.get("count", 1) + source_position = Position(args["entity_x"], args["entity_y"]) + + return func( + entity=entity, source=source_position, quantity=quantity, tick=tick + ) + + @staticmethod + def _convert_pickup_entity(func, args: Dict[str, Any], tick: int): + if "entity" in args and "x" in args and "y" in args: + entity_name = args["entity"] + # Skip if entity is blank or empty + if not entity_name or not entity_name.strip(): + print( + f"Warning: Skipping pickup_entity with blank entity at tick {tick}" + ) + return None + + entity = ActionConverter._get_prototype(entity_name) + position = Position(args["x"], args["y"]) + return func(entity=entity, position=position, tick=tick) + + @staticmethod + def _convert_set_research(func, args: Dict[str, Any], tick: int): + technology_name = args.get("technology") or args.get("research") + if technology_name: + try: + technology = ( + technology_name + if hasattr(technology_name, "value") + else Technology(technology_name) + ) + except (ValueError, AttributeError): + print( + f"Warning: No Technology enum found for '{technology_name}', using string" + ) + technology = technology_name + return func(technology=technology, tick=tick) + + @staticmethod + def _convert_inspect_inventory(func, args: Dict[str, Any], tick: int): + return func(tick=tick) + + @staticmethod + def _convert_set_entity_recipe(func, args: Dict[str, Any], tick: int): + entity_name = args.get("entity") + recipe_name = args.get("new_recipe") + x = args.get("x") + y = args.get("y") + + if entity_name and recipe_name and x is not None and y is not None: + position = Position(x, y) + + # Create a placeholder entity at the given position + entity = PlaceholderEntity(name=entity_name, position=position) + + # Convert recipe name to appropriate type - try Prototype first, then RecipeName + from fle.env.game_types import RecipeName, Prototype + + prototype = None + + # First try to find it in Prototype enum (most recipes are here) + try: + for proto in Prototype: + if proto.value[0] == recipe_name: + prototype = proto + break + except Exception: + pass + + # If not found in Prototype, try RecipeName enum (for fluid recipes) + if prototype is None: + try: + prototype = RecipeName(recipe_name) + except (ValueError, AttributeError): + # Fall back to string if neither enum works + prototype = recipe_name + + return func(entity=entity, prototype=prototype, tick=tick) + + @staticmethod + def _get_prototype(item_name: str): + """Convert item name to Prototype enum instance.""" + if item_name in prototype_by_name: + return prototype_by_name[item_name] + else: + print(f"Warning: No Prototype found for '{item_name}', using string") + return item_name + + @staticmethod + def _parse_items_string(items_str: str) -> List[Dict]: + """Parse items string to list of item dictionaries.""" + if not items_str or not items_str.strip(): + return [] + + try: + items_str = items_str.replace("'", '"') + return json.loads(items_str) + except json.JSONDecodeError: + return [] diff --git a/fle/data/replays/extract.py b/fle/data/replays/extract.py new file mode 100644 index 000000000..dca822404 --- /dev/null +++ b/fle/data/replays/extract.py @@ -0,0 +1,657 @@ +import json +import math +from pathlib import Path +from typing import List, Dict, Any, Union + + +def get_time_from_record(record: Dict[str, Any]) -> int: + """ + Extract time from a record, checking both 't' and 'tick' fields. + Returns the maximum of the two if both exist. + + Args: + record: Dictionary that should contain time information + + Returns: + Time value as integer + + Raises: + ValueError: If neither 't' nor 'tick' fields exist or are valid + """ + t_val = record.get("t") + tick_val = record.get("tick") + + # Convert to int if they exist and are not None + valid_times = [] + if t_val is not None: + try: + valid_times.append(int(t_val)) + except (ValueError, TypeError): + pass + + if tick_val is not None: + try: + valid_times.append(int(tick_val)) + except (ValueError, TypeError): + pass + + if not valid_times: + raise ValueError(f"Record has no valid time field ('t' or 'tick'): {record}") + + return max(valid_times) + + +def is_standard_factorio_resource(entity_name: str) -> bool: + """ + Check if an entity name represents a standard Factorio resource that can be harvested. + + Args: + entity_name: The name of the entity + + Returns: + True if it's a standard harvestable resource, False otherwise + """ + # Standard resources + standard_resources = { + "coal", + "iron-ore", + "copper-ore", + "stone", + "uranium-ore", + "crude-oil", + } + + # Trees (various types) + tree_prefixes = ["tree-", "dead-tree-"] + + # Rocks (various types) + rock_names = {"rock-big", "rock-huge", "sand-rock-big"} + + # Check exact matches for standard resources and rocks + if entity_name in standard_resources or entity_name in rock_names: + return True + + # Check tree prefixes + if any(entity_name.startswith(prefix) for prefix in tree_prefixes): + return True + + # Check for generic tree names + if entity_name == "tree" or "tree" in entity_name: + return True + + return False + + +def decompose_move_to_call( + start_tick: int, + end_tick: int, + start_x: float, + start_y: float, + end_x: float, + end_y: float, +) -> List[Dict[str, Any]]: + """ + Decompose a long move_to call into smaller chunks based on character movement speed. + Character moves at 8.9 tiles per second (60 ticks per second). + + Args: + start_tick: Starting tick + end_tick: Ending tick + start_x, start_y: Starting position + end_x, end_y: Ending position + + Returns: + List of move_to call dictionaries, each representing a 0.5-second (30 tick) movement + """ + MAX_DISTANCE_PER_HALF_SECOND = 4.45 # tiles per 0.5 seconds (8.9 / 2) + TICKS_PER_HALF_SECOND = 30 + + # Convert coordinates to float to handle string inputs + start_x = round(float(start_x), 1) + start_y = round(float(start_y), 1) + end_x = round(float(end_x), 1) + end_y = round(float(end_y), 1) + start_tick = int(start_tick) + end_tick = int(end_tick) + + # Calculate total distance + total_distance = math.sqrt((end_x - start_x) ** 2 + (end_y - start_y) ** 2) + + # If distance is small enough for one move, return single call + if total_distance <= MAX_DISTANCE_PER_HALF_SECOND: + call = f"move_to(start_tick={start_tick}, end_tick={end_tick}, start_x={start_x}, start_y={start_y}, end_x={end_x}, end_y={end_y})" + return [{"call": call, "sort_tick": start_tick}] + + # Calculate how many segments we need + num_segments = math.ceil(total_distance / MAX_DISTANCE_PER_HALF_SECOND) + + # Calculate direction vector + dx = end_x - start_x + dy = end_y - start_y + + # Normalize to get unit vector + unit_dx = dx / total_distance + unit_dy = dy / total_distance + + calls = [] + current_x = start_x + current_y = start_y + current_tick = start_tick + + for i in range(num_segments): + # Calculate next position + if i == num_segments - 1: + # Last segment - go to exact end position + next_x = end_x + next_y = end_y + next_tick = min(current_tick + TICKS_PER_HALF_SECOND, end_tick) + else: + # Intermediate segment - move MAX_DISTANCE_PER_HALF_SECOND + next_x = round(current_x + (unit_dx * MAX_DISTANCE_PER_HALF_SECOND), 1) + next_y = round(current_y + (unit_dy * MAX_DISTANCE_PER_HALF_SECOND), 1) + next_tick = current_tick + TICKS_PER_HALF_SECOND + + # Create the call + call = f"move_to(start_tick={current_tick}, end_tick={next_tick}, start_x={current_x}, start_y={current_y}, end_x={next_x}, end_y={next_y})" + calls.append({"call": call, "sort_tick": current_tick}) + + # Update for next iteration + current_x = next_x + current_y = next_y + current_tick = next_tick + + # If we've reached the end position or end tick, stop + if ( + abs(current_x - end_x) < 0.1 and abs(current_y - end_y) < 0.1 + ) or current_tick >= end_tick: + break + + return calls + + +def handle_action_based_record(record: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Handle records that use the new action-based schema. + + Args: + record: The log record dictionary + + Returns: + List of dictionaries with 'call' and 'sort_tick' fields, or empty list if not an action-based record + """ + action = record.get("action") + + if action == "craft_item": + timing = record.get("timing", {}) + crafting = record.get("crafting", {}) + + recipe = crafting.get("recipe", "") + craft_timings = crafting.get("craft_timings", []) + + # If we have detailed craft timings, create individual calls for each completed item + if craft_timings: + calls = [] + for craft_timing in craft_timings: + if craft_timing.get("status") == "completed": + queue_tick = craft_timing.get("queue_tick", 0) + completion_tick = craft_timing.get("completion_tick", 0) + + call = f"craft_item(start_tick={queue_tick}, end_tick={completion_tick}, recipe='{recipe}', count=1)" + calls.append({"call": call, "sort_tick": queue_tick}) + return calls + else: + # Fall back to the original behavior if craft_timings is not available + start_tick = timing.get("start_tick", 0) + end_tick = timing.get("end_tick", 0) + count = crafting.get("total_crafted", 0) + + call = f"craft_item(start_tick={start_tick}, end_tick={end_tick}, recipe='{recipe}', count={count})" + return [{"call": call, "sort_tick": start_tick}] + + elif action == "move_to_direction": + player = record.get("player", {}) + start_movement = player.get("start_movement", {}) + end_movement = player.get("end_movement", {}) + + start_tick = start_movement.get("tick", 0) + end_tick = end_movement.get("tick", 0) + start_x = start_movement.get("x", 0) + start_y = start_movement.get("y", 0) + end_x = end_movement.get("x", 0) + end_y = end_movement.get("y", 0) + + # Return the move data for further processing (decompose vs single call) + return [ + { + "action": "move_to_direction", + "start_tick": start_tick, + "end_tick": end_tick, + "start_x": start_x, + "start_y": start_y, + "end_x": end_x, + "end_y": end_y, + } + ] + + elif action == "pickup_entity": + tick = get_time_from_record(record) + selected_entity = record.get("selected_entity", {}) + + entity = selected_entity.get("name", "") + x = selected_entity.get("x", 0) + y = selected_entity.get("y", 0) + + call = f"pickup_entity(tick={tick}, entity='{entity}', x={x}, y={y})" + return [{"call": call, "sort_tick": tick}] + + elif action == "place_entity": + tick = get_time_from_record(record) + item = record.get("item", {}) + entity = record.get("entity", {}) + + item_name = item.get("name", "") + x = entity.get("x", 0) + y = entity.get("y", 0) + direction = entity.get("direction", {}).get("value", 0) + + call = f"place_entity(tick={tick}, item='{item_name}', x={x}, y={y}, direction={direction})" + return [{"call": call, "sort_tick": tick}] + + elif action == "extract_item": + tick = get_time_from_record(record) + entity = record.get("entity", {}) + items = record.get("items", []) + + entity_name = entity.get("name", "") + entity_x = entity.get("x", 0) + entity_y = entity.get("y", 0) + items_str = str(items) if items else "" + + call = f"extract_item(tick={tick}, entity='{entity_name}', entity_x={entity_x}, entity_y={entity_y}, items='{items_str}')" + return [{"call": call, "sort_tick": tick}] + + elif action == "insert_item": + tick = get_time_from_record(record) + entity = record.get("entity", {}) + items = record.get("items", []) + + entity_name = entity.get("name", "") + entity_x = entity.get("x", 0) + entity_y = entity.get("y", 0) + items_str = str(items) if items else "" + + call = f"insert_item(tick={tick}, entity='{entity_name}', entity_x={entity_x}, entity_y={entity_y}, items='{items_str}')" + return [{"call": call, "sort_tick": tick}] + + elif action == "rotate_entity": + tick = get_time_from_record(record) + entity = record.get("entity", {}) + direction = entity.get("direction", {}) + + entity_name = entity.get("name", "") + x = entity.get("x", 0) + y = entity.get("y", 0) + old_direction = direction.get("previous", {}).get("value", 0) + new_direction = direction.get("new", {}).get("value", 0) + + call = f"rotate_entity(tick={tick}, entity='{entity_name}', x={x}, y={y}, old_direction={old_direction}, new_direction={new_direction})" + return [{"call": call, "sort_tick": tick}] + + elif action == "set_entity_recipe": + tick = get_time_from_record(record) + entity = record.get("entity", {}) + player = record.get("player", {}) + + entity_name = entity.get("name", "") + new_recipe = entity.get("new_recipe", "") + x = player.get("x", 0) + y = player.get("y", 0) + + call = f"set_entity_recipe(tick={tick}, entity='{entity_name}', new_recipe='{new_recipe}', x={x}, y={y})" + return [{"call": call, "sort_tick": tick}] + + elif action == "research_started": + tick = get_time_from_record(record) + research = record.get("research", "") + + call = f"set_research(tick={tick}, research='{research}')" + return [{"call": call, "sort_tick": tick}] + + # Not an action-based record + return [] + + +def transform_record_to_python_call( + record: Dict[str, Any], source_file: str +) -> List[Dict[str, Any]]: + """ + Transform a log record into Python function call strings based on the source file type. + + Args: + record: The log record dictionary + source_file: Name of the source file (without extension) + + Returns: + List of dictionaries with 'call' and 'sort_tick' fields, or empty list if record should be ignored + """ + # Get the base filename without extension + file_type = source_file.replace(".jsonl", "") + + # Skip core-meta files entirely + if file_type == "core-meta": + return [] + + # Handle new action-based schema + action_result = handle_action_based_record(record) + if action_result: + # Special handling for move_to_direction - needs decomposition + if action_result[0].get("action") == "move_to_direction": + move_data = action_result[0] + return decompose_move_to_call( + move_data["start_tick"], + move_data["end_tick"], + move_data["start_x"], + move_data["start_y"], + move_data["end_x"], + move_data["end_y"], + ) + return action_result + + tick = get_time_from_record(record) + + if file_type == "harvest_resource_collated": + start_tick = tick - record.get("duration_ticks", 0) + entity = record.get("entity", "") + x = record.get("x", 0) + y = record.get("y", 0) + + # Check if this is a standard Factorio resource + if is_standard_factorio_resource(entity): + call = f"harvest_resource(start_tick={start_tick}, end_tick={tick}, entity='{entity}', x={x}, y={y})" + return [{"call": call, "sort_tick": start_tick}] + else: + # For non-standard resources, treat as pickup_entity + call = f"pickup_entity(tick={tick}, entity='{entity}', x={x}, y={y})" + return [{"call": call, "sort_tick": tick}] + + # Unknown file type, skip + return [] + + +def transform_record_to_python_call_no_decompose( + record: Dict[str, Any], source_file: str +) -> List[Dict[str, Any]]: + """ + Transform a log record into Python function call strings based on the source file type. + This version does NOT decompose move_to calls. + + Args: + record: The log record dictionary + source_file: Name of the source file (without extension) + + Returns: + List of dictionaries with 'call' and 'sort_tick' fields, or empty list if record should be ignored + """ + # Get the base filename without extension + file_type = source_file.replace(".jsonl", "") + + # Skip core-meta files entirely + if file_type == "core-meta": + return [] + + # Handle new action-based schema + action_result = handle_action_based_record(record) + if action_result: + # Special handling for move_to_direction - create single call instead of decomposing + if action_result[0].get("action") == "move_to_direction": + move_data = action_result[0] + call = f"move_to(start_tick={move_data['start_tick']}, end_tick={move_data['end_tick']}, start_x={move_data['start_x']}, start_y={move_data['start_y']}, end_x={move_data['end_x']}, end_y={move_data['end_y']})" + return [{"call": call, "sort_tick": move_data["start_tick"]}] + return action_result + + tick = get_time_from_record(record) + + if file_type == "harvest_resource_collated": + start_tick = tick - record.get("duration_ticks", 0) + entity = record.get("entity", "") + x = record.get("x", 0) + y = record.get("y", 0) + + # Check if this is a standard Factorio resource + if is_standard_factorio_resource(entity): + call = f"harvest_resource(start_tick={start_tick}, end_tick={tick}, entity='{entity}', x={x}, y={y})" + else: + # For non-standard resources, treat as pickup_entity + call = f"pickup_entity(tick={tick}, entity='{entity}', x={x}, y={y})" + + return [{"call": call, "sort_tick": start_tick}] + + # Unknown file type, skip + return [] + + +def read_and_combine_jsonls( + folder_path: Union[str, Path], max_time: int = None +) -> List[Dict[str, Any]]: + """ + Read all JSONL files from a folder, combine them, sort by time (t or tick field), + and optionally filter up to a certain time. + + Args: + folder_path: Path to the folder containing JSONL files + max_time: Maximum time threshold (inclusive). If None, no filtering is applied. + + Returns: + List of dictionaries sorted by time field in ascending order + """ + folder_path = Path(folder_path) + + if not folder_path.exists(): + raise FileNotFoundError(f"Folder not found: {folder_path}") + + if not folder_path.is_dir(): + raise ValueError(f"Path is not a directory: {folder_path}") + + all_records = [] + crash_site_filtered = 0 + + # Find all JSONL files in the folder + jsonl_files = list(folder_path.glob("*.jsonl")) + + if not jsonl_files: + print(f"Warning: No JSONL files found in {folder_path}") + return all_records + + # Read each JSONL file + for jsonl_file in jsonl_files: + try: + with open(jsonl_file, "r", encoding="utf-8") as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if line: # Skip empty lines + try: + record = json.loads(line) + + # Skip records containing 'crash-site' anywhere in the data + record_str = json.dumps(record, default=str).lower() + if "crash-site" in record_str: + crash_site_filtered += 1 + continue + + # Validate that record has a valid time field + try: + get_time_from_record(record) + except ValueError as e: + print( + f"Warning: Skipping record on line {line_num} in {jsonl_file.name}: {e}" + ) + continue + + # Add source file information for debugging + record["_source_file"] = jsonl_file.name + all_records.append(record) + except json.JSONDecodeError as e: + print( + f"Warning: Invalid JSON on line {line_num} in {jsonl_file.name}: {e}" + ) + continue + except Exception as e: + print(f"Error reading file {jsonl_file.name}: {e}") + continue + + # Filter by time if max_time is specified + if max_time is not None: + all_records = [ + record for record in all_records if get_time_from_record(record) <= max_time + ] + + # Sort by time field + all_records.sort(key=get_time_from_record) + + print(f"Loaded {len(all_records)} records from {len(jsonl_files)} JSONL files") + if crash_site_filtered > 0: + print(f"Filtered out {crash_site_filtered} records containing 'crash-site'") + if max_time is not None: + print(f"Filtered to records with time <= {max_time}") + + return all_records + + +def save_python_calls( + records: List[Dict[str, Any]], output_path: Union[str, Path], decompose: bool = True +) -> None: + """ + Transform records to Python function calls and save to JSONL file. + + Args: + records: List of record dictionaries + output_path: Path to the output JSONL file + decompose: Whether to decompose move_to calls into smaller chunks + """ + output_path = Path(output_path) + + python_calls = [] + skipped_count = 0 + + # Choose which transform function to use + transform_func = ( + transform_record_to_python_call + if decompose + else transform_record_to_python_call_no_decompose + ) + + def get_function_priority(call_string: str) -> int: + """Get priority for function calls to ensure proper ordering on same tick.""" + if call_string.startswith("pickup_entity("): + return 0 # Highest priority - execute first + elif call_string.startswith("place_entity("): + return 1 # Lower priority - execute after pickup + else: + return 2 # Default priority for other functions + + for record in records: + source_file = record.get("_source_file", "") + call_data_list = transform_func(record, source_file) + + if call_data_list: # Check if call_data_list is not empty + for call_data in call_data_list: + python_calls.append( + {"tick": call_data["sort_tick"], "call": call_data["call"]} + ) + else: + skipped_count += 1 + + # Sort by tick first, then by function priority to ensure pickup_entity comes before place_entity + python_calls.sort(key=lambda x: (x["tick"], get_function_priority(x["call"]))) + + # Create output directory if it doesn't exist + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_path, "w", encoding="utf-8") as f: + for call_record in python_calls: + f.write(json.dumps(call_record) + "\n") + + decompose_text = ( + "with decomposed move_to" if decompose else "without decomposed move_to" + ) + print( + f"Saved {len(python_calls)} Python function calls ({decompose_text}) to {output_path}" + ) + print(f"Skipped {skipped_count} records (ignored file types or filtered events)") + + +def get_time_range(records: List[Dict[str, Any]]) -> tuple: + """ + Get the time range (min, max) from a list of records. + + Args: + records: List of dictionaries with time fields + + Returns: + Tuple of (min_time, max_time) + """ + if not records: + return (0, 0) + + times = [get_time_from_record(record) for record in records] + return (min(times), max(times)) + + +def save_combined_jsonl( + records: List[Dict[str, Any]], output_path: Union[str, Path] +) -> None: + """ + Save a list of records to a JSONL file. + + Args: + records: List of dictionaries to save + output_path: Path to the output JSONL file + """ + output_path = Path(output_path) + + with open(output_path, "w", encoding="utf-8") as f: + for record in records: + # Remove the source file metadata before saving + record_copy = record.copy() + record_copy.pop("_source_file", None) + f.write(json.dumps(record_copy) + "\n") + + print(f"Saved {len(records)} records to {output_path}") + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--max-time", type=int, required=True, help="Maximum tick to process" + ) + args = parser.parse_args() + + # Example usage + folder_path = "/Users/neel/Desktop/Work/factorio-data-collector/factorio_replays/factorio_replay_20250723_110424" + max_time = args.max_time + + # Read and combine all JSONL files + combined_records = read_and_combine_jsonls(folder_path, max_time) + + # Create output directories + runnable_dir = Path("_runnable_actions") + extracted_dir = Path("_extracted") + runnable_dir.mkdir(exist_ok=True) + extracted_dir.mkdir(exist_ok=True) + + # Save decomposed version to _runnable_actions (for execution) + runnable_output = runnable_dir / f"combined_events_py_{max_time}.jsonl" + save_python_calls(combined_records, runnable_output, decompose=True) + + # Save non-decomposed version to _extracted (for analysis) + extracted_output = extracted_dir / f"combined_events_py_{max_time}.jsonl" + save_python_calls(combined_records, extracted_output, decompose=False) + + # Print time range + min_time, max_time_actual = get_time_range(combined_records) + print(f"Time range: {min_time} to {max_time_actual}") + + # Optionally save the original combined results + save_combined_jsonl(combined_records, "combined_events.jsonl") diff --git a/fle/data/replays/periodic_logger.py b/fle/data/replays/periodic_logger.py new file mode 100644 index 000000000..a3e6b4edd --- /dev/null +++ b/fle/data/replays/periodic_logger.py @@ -0,0 +1,171 @@ +""" +Handles periodic data logging functionality. +""" + +import json +from typing import Any, List, Optional +from datetime import datetime + +from fle.data.replays.action_converter import ActionConverter + + +class PeriodicLogger: + """Handles periodic data logging functionality.""" + + def __init__(self, log_file_path: Optional[str], interval: int): + self.log_file_path = log_file_path + self.interval = interval + self.last_logged_tick = 0 + self.enabled = log_file_path is not None and interval > 0 + + if self.enabled: + # Clear any existing log file + with open(self.log_file_path, "w"): + pass + + def should_log_at_tick(self, tick: int, min_tick: int = 0) -> bool: + """Check if we should log at the given tick.""" + if not self.enabled or tick < min_tick: + return False + + return tick >= self.last_logged_tick + self.interval + + def add_periodic_commands( + self, batch_info: List, namespace, min_tick: int, max_tick: int + ): + """Add periodic logging commands to the batch.""" + if not self.enabled: + return {} + + periodic_commands = {} + current_log_tick = self.last_logged_tick + self.interval + + while current_log_tick <= max_tick: + if current_log_tick >= min_tick: + try: + ActionConverter.execute_tool_call_in_batch( + namespace, "inspect_inventory", {}, current_log_tick + ) + command_index = len(batch_info) + periodic_commands[command_index] = { + "tick": current_log_tick, + "type": "inventory", + } + batch_info.append( + { + "index": command_index, + "func_name": "inspect_inventory", + "tick": current_log_tick, + "args": {}, + "is_periodic": True, + } + ) + except Exception as e: + print( + f"Error adding periodic logging at tick {current_log_tick}: {e}" + ) + + current_log_tick += self.interval + + return periodic_commands + + def generate_periodic_commands(self, namespace, min_tick: int, max_tick: int): + """Generate periodic logging commands without modifying batch_info.""" + if not self.enabled: + return [] + + periodic_commands = [] + current_log_tick = self.last_logged_tick + self.interval + + while current_log_tick <= max_tick: + if current_log_tick >= min_tick: + try: + ActionConverter.execute_tool_call_in_batch( + namespace, "inspect_inventory", {}, current_log_tick + ) + periodic_commands.append( + { + "func_name": "inspect_inventory", + "tick": current_log_tick, + "args": {}, + "is_periodic": True, + "periodic_type": "inventory", + } + ) + except Exception as e: + print( + f"Error preparing periodic logging at tick {current_log_tick}: {e}" + ) + + current_log_tick += self.interval + + return periodic_commands + + def log_result(self, tick: int, result_data: Any) -> bool: + """Log a periodic result and return success status.""" + if not self.enabled: + return False + + try: + # Create a serializable version of the inventory data + def make_serializable(obj, visited=None, depth=0): + if visited is None: + visited = set() + if depth > 10: # Prevent infinite recursion + return str(obj) + + obj_id = id(obj) + if obj_id in visited: + return f"" + + visited.add(obj_id) + + try: + if hasattr(obj, "to_dict"): + return make_serializable(obj.to_dict(), visited, depth + 1) + elif hasattr(obj, "__dict__"): + result = {} + for key, value in obj.__dict__.items(): + if not key.startswith("_"): + try: + result[key] = make_serializable( + value, visited, depth + 1 + ) + except: + result[key] = str(value) + return result + elif isinstance(obj, (list, tuple)): + return [ + make_serializable(item, visited, depth + 1) for item in obj + ] + elif isinstance(obj, dict): + return { + k: make_serializable(v, visited, depth + 1) + for k, v in obj.items() + } + elif hasattr(obj, "value"): + return obj.value + else: + return obj + except: + return str(obj) + finally: + visited.discard(obj_id) + + serializable_data = make_serializable(result_data) + + log_entry = { + "timestamp": datetime.now().isoformat(), + "tick": tick, + "inventory": serializable_data, + "entities": [], + } + + with open(self.log_file_path, "a") as f: + f.write(json.dumps(log_entry) + "\n") + + self.last_logged_tick = tick + return True + except Exception as e: + print(f"Warning: Failed to save periodic data at tick {tick}: {e}") + return False diff --git a/fle/data/replays/processors.py b/fle/data/replays/processors.py new file mode 100644 index 000000000..579cafc96 --- /dev/null +++ b/fle/data/replays/processors.py @@ -0,0 +1,701 @@ +""" +Factorio batch processors with different execution strategies. + +Classes: +- BatchProcessor: Base class for batch processing logic +- SequentialProcessor: Sequential batch processing implementation +- PipelineProcessor: Pipeline batch processing implementation +""" + +import time +from pathlib import Path +from typing import Dict, Any, List, Tuple, Optional +from dataclasses import dataclass + +from fle.data.replays.run_actions_utils import ( + load_events, + create_factorio_instance, + parse_function_call, +) +from fle.data.replays.action_converter import ActionConverter +from fle.data.replays.periodic_logger import PeriodicLogger + + +@dataclass +class ProcessingConfig: + """Configuration for batch processing.""" + + events_file_path: str + enable_logging: bool = False + speed: float = 1.0 + batch_size: int = 500 + max_concurrent_batches: int = 1 + enable_periodic_logging: Optional[bool] = None + periodic_log_interval: int = 0 + + def __post_init__(self): + if self.enable_periodic_logging is None: + self.enable_periodic_logging = self.periodic_log_interval > 0 + + +class BatchProcessor: + """Base class for batch processing logic.""" + + def __init__(self, config: ProcessingConfig): + self.config = config + self.instance = None + self.periodic_logger = None + + def setup(self): + """Initialize the Factorio instance and logging.""" + self.instance = create_factorio_instance(self.config.max_concurrent_batches) + self.instance.reset() + self.instance.speed(self.config.speed) + print("Factorio instance created and reset") + + # Setup logging + if self.config.enable_logging: + print( + "Note: Global logging functions have been removed. Use PeriodicLogger instead." + ) + else: + print("Logging disabled - no data will be saved") + + # Setup periodic logging + periodic_log_file = None + if self.config.enable_periodic_logging: + events_path = Path(self.config.events_file_path) + periodic_log_file = ( + events_path.parent / f"{events_path.stem}_periodic_data.jsonl" + ) + print( + f"Periodic logging enabled - data will be saved to {periodic_log_file} every {self.config.periodic_log_interval} ticks" + ) + + self.periodic_logger = PeriodicLogger( + str(periodic_log_file) if periodic_log_file else None, + self.config.periodic_log_interval, + ) + + def cleanup(self): + """Clean up resources.""" + try: + if self.instance: + # Emergency cleanup for all batch managers + print("🧹 Performing final server cleanup...") + for manager in self.instance.batch_managers: + try: + manager.emergency_cleanup() + except Exception as e: + print( + f"Warning: Manager {manager.manager_id} cleanup failed: {e}" + ) + + # Final server cleanup + self.instance.begin_transaction() + self.instance.add_command( + "/sc global.actions.reset_sequence()", raw=True + ) + self.instance.add_command( + "/sc global.actions.clear_batch_results()", raw=True + ) + self.instance.execute_transaction() + print("✅ Final server cleanup completed") + + self.instance.cleanup() + except Exception as e: + print(f"Warning: Final cleanup failed: {e}") + + print("Instance cleaned up") + + def load_and_prepare_events(self) -> List[Dict]: + """Load events from file and prepare them for processing.""" + events = load_events(self.config.events_file_path) + events.sort(key=lambda x: x.get("tick", 0)) + + batch_actions = [] + for event in events: + tick = event.get("tick", 0) + call = event.get("call", "") + + try: + func_name, args = parse_function_call(call) + batch_actions.append( + {"tick": tick, "func_name": func_name, "args": args} + ) + except Exception as e: + print(f"Warning: Failed to parse event {call}: {e}") + + return batch_actions + + def submit_batch_to_server( + self, batch: List[Dict], start_tick: int + ) -> Tuple[List[Dict], float]: + """Submit a batch of actions to the server and handle results.""" + namespace = self.instance.namespace + print(f"Processing batch of {len(batch)} actions starting at tick {start_tick}") + + tool_execution_start = time.time() + + # Calculate tick range + min_tick = batch[0]["tick"] if batch else start_tick + max_tick = max(action["tick"] for action in batch) if batch else start_tick + + # Collect all commands (regular + periodic) first + all_commands = [] + + # Add regular commands (collect metadata first, don't execute yet) + for action in batch: + tick = action["tick"] + func_name = action["func_name"] + args = action["args"] + + all_commands.append( + { + "func_name": func_name, + "tick": tick, + "args": args, + "is_periodic": False, + } + ) + + # Add periodic commands + periodic_commands = self.periodic_logger.generate_periodic_commands( + namespace, min_tick, max_tick + ) + all_commands.extend(periodic_commands) + + # Sort ALL commands by tick, then assign sequential indices + all_commands.sort(key=lambda x: x["tick"]) + + # Assign indices and create final batch_info + batch_info = [] + periodic_indices = set() + + for i, cmd in enumerate(all_commands): + cmd["index"] = i + batch_info.append(cmd) + + if cmd["is_periodic"]: + periodic_indices.add(i) + + # CRITICAL: Execute tools in sorted order to ensure batch manager receives commands + # in the same order as batch_info indices. This prevents command_index mismatches + # between Python (0-based batch_info indices) and Lua (1-based scheduled_commands indices). + for cmd in all_commands: + if not cmd["is_periodic"]: + # Execute the tool call to add it to batch manager in correct order + try: + result = ActionConverter.execute_tool_call_in_batch( + namespace, cmd["func_name"], cmd["args"], cmd["tick"] + ) + cmd["batch_result"] = result + except Exception as e: + print( + f"Error adding {cmd['func_name']} to batch at tick {cmd['tick']}: {e}" + ) + cmd["error"] = str(e) + + tool_execution_time = time.time() - tool_execution_start + + # Submit and stream results + print( + f"Submitting batch of {len(batch)} regular actions + {len(periodic_commands)} periodic commands = {len(batch_info)} total commands (sorted by tick)" + ) + + submission_start_time = time.time() + results = [] + result_count = 0 + + for result in self.instance.batch_manager.submit_batch_and_stream( + timeout_seconds=600, poll_interval=2 + ): + # Measure time to first result if this is the first one + if result_count == 0: + first_result_time = time.time() - submission_start_time + print(f" First result received after {first_result_time:.3f}s") + + result_count += 1 + command_index = result["command_index"] + + # Handle periodic logging using the index set + if command_index in periodic_indices: + if result.get("success") and result.get("result"): + executed_tick = result["tick"] + success = self.periodic_logger.log_result( + executed_tick, result["result"] + ) + if success: + print(f" 📊 Periodic data logged at tick {executed_tick}") + # Add periodic results to results list for proper tracking + results.append(result) + continue + + # Handle regular commands + executed_tick = result["tick"] + planned_tick = result.get("planned_tick", "?") + success = result["success"] + + tick_info = ( + f"tick {executed_tick} (planned {planned_tick}) ⚠️" + if planned_tick != "?" and executed_tick != planned_tick + else f"tick {executed_tick}" + ) + + status = "✓" if success else "❌" + print(f" {status} {result['command']} at {tick_info}") + if not success: + print(f" Error: {result['result']}") + + results.append(result) + + # Debug: Check which commands are missing results + print( + f" 📊 Finished streaming results: received {result_count} results, expected {len(batch_info)} total commands" + ) + regular_results = [ + r for r in results if r["command_index"] not in periodic_indices + ] + print( + f" 📊 Regular command results: {len(regular_results)} out of {len(batch)} actions" + ) + + # Additional debug info + print(f" 📊 Periodic command indices: {sorted(periodic_indices)}") + print( + f" 📊 Total expected commands: {len(batch_info)} (regular: {len(batch_info) - len(periodic_indices)}, periodic: {len(periodic_indices)})" + ) + + if len(regular_results) < len(batch): + # Get the indices of ALL received results (both periodic and regular) + all_received_indices = {r["command_index"] for r in results} + + # Get the expected indices for regular commands only + expected_regular_indices = { + i for i in range(len(batch_info)) if i not in periodic_indices + } + + # Find which regular command indices are missing from ALL received results + missing_indices = expected_regular_indices - all_received_indices + + print(f" 📊 All received indices: {sorted(all_received_indices)}") + print( + f" 📊 Expected regular indices: {sorted(expected_regular_indices)}" + ) + print( + f" ⚠️ Missing results for command indices: {sorted(missing_indices)}" + ) + + if missing_indices: + # Show which specific commands are missing (using batch_info which preserves order) + for idx in missing_indices: + if idx < len(batch_info): + cmd = batch_info[idx] + print( + f" Missing: {cmd['func_name']} at tick {cmd['tick']}" + ) + else: + # Fallback: try to get info from batch manager metadata + if hasattr(self.instance.batch_manager, "last_batch_metadata"): + metadata = self.instance.batch_manager.last_batch_metadata + if idx < len(metadata["commands"]): + cmd = metadata["commands"][idx] + print( + f" Missing: {cmd['command']} at tick {cmd['tick']} (from batch metadata)" + ) + else: + print( + f" Missing: command index {idx} (no metadata available)" + ) + else: + print( + f" Missing: command index {idx} (no metadata available)" + ) + else: + print(" ✅ All regular commands received successfully!") + else: + print(" ✅ All expected regular commands received!") + + # Final verification + periodic_results = [ + r for r in results if r["command_index"] in periodic_indices + ] + print( + f" 📊 Final count verification: {len(results)} total results ({len(regular_results)} regular + {len(periodic_results)} periodic)" + ) + + if len(results) != len(batch_info): + print( + f" ⚠️ Total result count mismatch: expected {len(batch_info)}, got {len(results)}" + ) + else: + print(" ✅ Total result count matches expected!") + + return results, tool_execution_time + + +class SequentialProcessor(BatchProcessor): + """Sequential batch processing implementation.""" + + def execute(self): + """Execute events using sequential batch processing.""" + try: + events = self.load_and_prepare_events() + if not events: + print("No events to process") + return + + print( + f"Starting sequential batch execution with tick interval batch size {self.config.batch_size}" + ) + print(f"Total events to process: {len(events)}") + + min_tick = events[0]["tick"] + max_tick = events[-1]["tick"] + print(f"Event tick range: {min_tick} to {max_tick}") + + total_results = [] + batch_num = 0 + current_tick_start = ( + min_tick // self.config.batch_size + ) * self.config.batch_size + + while current_tick_start <= max_tick: + current_tick_end = current_tick_start + self.config.batch_size + batch_num += 1 + + batch_events = [ + event + for event in events + if current_tick_start <= event["tick"] < current_tick_end + ] + + if not batch_events: + print( + f"\n\033[96m=== Batch {batch_num} (ticks {current_tick_start}-{current_tick_end - 1}): No events ===\033[0m" + ) + current_tick_start = current_tick_end + continue + + print( + f"\n\033[96m=== Batch {batch_num} (ticks {current_tick_start}-{current_tick_end - 1}): {len(batch_events)} events ===\033[0m" + ) + + # Show tick distribution in this batch + batch_ticks = [event["tick"] for event in batch_events] + print( + f"\033[36m Tick range in batch: {min(batch_ticks)} to {max(batch_ticks)}\033[0m" + ) + + # Process batch + self.instance.batch_manager.activate() + batch_results, tool_time = self.submit_batch_to_server( + batch_events, batch_events[0]["tick"] + ) + self.instance.batch_manager.deactivate() + + total_results.extend(batch_results) + + # Display tool execution timing + avg_time_per_action = ( + tool_time / len(batch_events) if batch_events else 0 + ) + print( + f" Actions processed in {tool_time:.3f}s (avg {avg_time_per_action * 1000:.2f}ms per action)" + ) + + # Periodic memory cleanup + if batch_num % 10 == 0: + try: + print( + f" 🧹 Performing periodic server memory cleanup (batch {batch_num})" + ) + self.instance.begin_transaction() + self.instance.add_command( + "/sc global.actions.clear_batch_results()", raw=True + ) + self.instance.execute_transaction() + except Exception as e: + print(f"Warning: Failed to perform server memory cleanup: {e}") + + current_tick_start = current_tick_end + + # Print final statistics + successful_results = sum( + 1 for r in total_results if r.get("success", False) + ) + failed_results = len(total_results) - successful_results + print( + f"\nCompleted batch execution of {len(events)} events across {batch_num} tick interval batches" + ) + print(f"Total results collected: {len(total_results)}") + print(f"Successful commands: {successful_results}") + print(f"Failed commands: {failed_results}") + + if self.config.enable_periodic_logging and self.periodic_logger.enabled: + print(f"Periodic data logged to: {self.periodic_logger.log_file_path}") + + except KeyboardInterrupt: + print("\n🛑 Execution interrupted by user - cleaning up queued actions...") + self._emergency_cleanup() + except Exception as e: + print(f"Error during batch execution: {e}") + self._emergency_cleanup() + raise + + def _emergency_cleanup(self): + """Emergency cleanup for interrupts.""" + try: + print(" Clearing queued actions from server...") + for manager in self.instance.batch_managers: + print(f" Processing manager {manager.manager_id}...") + cleanup_results = manager.emergency_cleanup() + + # Report results + for operation, result in cleanup_results.items(): + if result == "success": + print(f" ✅ {operation}") + else: + print(f" ❌ {operation}: {result}") + + print(" ✅ Server cleanup completed") + except Exception as e: + print(f" ⚠️ Warning: Failed to clear queued actions: {e}") + + try: + self.instance.batch_manager.deactivate() + except: + pass + + +class PipelineProcessor(BatchProcessor): + """Pipeline batch processing implementation.""" + + def __init__(self, config: ProcessingConfig): + super().__init__(config) + self.completed_batches = [] + self.batch_stats = {} + self.start_time = None + + def execute(self): + """Execute events using pipeline batch processing.""" + try: + events = self.load_and_prepare_events() + if not events: + print("No events to process") + return + + print( + f"Starting pipeline execution with tick interval batch size {self.config.batch_size}" + ) + print(f"Max concurrent batches: {self.config.max_concurrent_batches}") + print(f"Total events to process: {len(events)}") + + min_tick = events[0]["tick"] + max_tick = events[-1]["tick"] + print(f"Event tick range: {min_tick} to {max_tick}") + + self.start_time = time.time() + + # Process batches sequentially for simplicity but with pipeline organization + batch_count = 0 + current_tick_start = ( + min_tick // self.config.batch_size + ) * self.config.batch_size + all_results = [] + + while current_tick_start <= max_tick: + current_tick_end = current_tick_start + self.config.batch_size + + batch_events = [ + event + for event in events + if current_tick_start <= event["tick"] < current_tick_end + ] + + if batch_events: + tick_range = f"(ticks {current_tick_start}-{current_tick_end - 1})" + print( + f"\n📦 Processing batch {batch_count} {tick_range}: {len(batch_events)} events" + ) + + # Process this batch + batch_start_time = time.time() + self.instance.batch_manager.activate() + batch_results, tool_time = self.submit_batch_to_server( + batch_events, batch_events[0]["tick"] + ) + self.instance.batch_manager.deactivate() + + batch_duration = time.time() - batch_start_time + + # Store batch statistics + self.batch_stats[batch_count] = { + "start_time": batch_start_time, + "duration": batch_duration, + "command_count": len(batch_events), + "result_count": len(batch_results), + "tick_range": tick_range, + "tool_execution_time": tool_time, + } + + all_results.extend(batch_results) + batch_count += 1 + + # Display timing info + avg_time_per_action = ( + tool_time / len(batch_events) if batch_events else 0 + ) + print( + f" ✅ Completed in {batch_duration:.2f}s, tool time: {tool_time:.3f}s (avg {avg_time_per_action * 1000:.2f}ms per action)" + ) + + # Periodic memory cleanup for pipeline mode + if batch_count % 20 == 0: # Every 20 batches for pipeline mode + try: + print( + f" 🧹 Performing periodic server memory cleanup (batch {batch_count})" + ) + self.instance.begin_transaction() + self.instance.add_command( + "/sc global.actions.clear_batch_results()", raw=True + ) + self.instance.execute_transaction() + except Exception as e: + print( + f"Warning: Failed to perform server memory cleanup: {e}" + ) + + current_tick_start = current_tick_end + + print(f"\n📦 All {batch_count} batches completed") + + # Calculate and print final statistics + total_time = time.time() - self.start_time + total_results = len(all_results) + successful_results = sum(1 for r in all_results if r.get("success", False)) + failed_results = total_results - successful_results + + print("\n📊 Final Statistics:") + print(f" Total events processed: {len(events)}") + print(f" Total batches: {batch_count}") + print(f" Total commands: {total_results}") + print(f" Successful commands: {successful_results}") + print(f" Failed commands: {failed_results}") + print(f" Total time: {total_time:.2f}s") + + if total_time > 0: + print(f" Commands per second: {total_results / total_time:.1f}") + + # Print detailed timing analysis + self.print_timing_summary() + + if self.config.enable_periodic_logging and self.periodic_logger.enabled: + print(f"Periodic data logged to: {self.periodic_logger.log_file_path}") + + except KeyboardInterrupt: + print("\n🛑 Execution interrupted by user - cleaning up queued actions...") + self._emergency_cleanup() + except Exception as e: + print(f"Error during pipeline execution: {e}") + self._emergency_cleanup() + raise + + def get_statistics(self) -> Dict[str, Any]: + """Get processing statistics.""" + if not self.start_time: + return {} + + elapsed = time.time() - self.start_time + total_commands = sum( + stats["result_count"] for stats in self.batch_stats.values() + ) + + return { + "total_batches_completed": len(self.batch_stats), + "total_commands_processed": total_commands, + "elapsed_time": elapsed, + "commands_per_second": total_commands / elapsed if elapsed > 0 else 0, + "batch_stats": self.batch_stats.copy(), + } + + def get_all_results(self) -> Dict[int, List[Dict]]: + """Get all results organized by batch_id.""" + # For this simplified version, return empty dict as results are handled inline + return {} + + def print_timing_summary(self): + """Print detailed timing analysis for bottleneck identification.""" + if not self.batch_stats: + print("No timing data available") + return + + print("\n" + "=" * 60) + print("📊 PIPELINE PROCESSING TIMING ANALYSIS") + print("=" * 60) + + durations = [stats["duration"] for stats in self.batch_stats.values()] + tool_times = [ + stats["tool_execution_time"] for stats in self.batch_stats.values() + ] + command_counts = [stats["command_count"] for stats in self.batch_stats.values()] + + avg_duration = sum(durations) / len(durations) + max_duration = max(durations) + avg_tool_time = sum(tool_times) / len(tool_times) + max_tool_time = max(tool_times) + avg_commands = sum(command_counts) / len(command_counts) + + print(f"Batches Processed: {len(self.batch_stats)}") + print() + print("⏱️ TIMING BREAKDOWN:") + print( + f" Batch Duration: avg={avg_duration:.3f}s max={max_duration:.3f}s" + ) + print( + f" Tool Execution Time: avg={avg_tool_time:.3f}s max={max_tool_time:.3f}s" + ) + print(f" Commands per Batch: avg={avg_commands:.1f}") + + if avg_commands > 0: + avg_time_per_command = avg_tool_time / avg_commands + print(f" Time per Command: avg={avg_time_per_command * 1000:.2f}ms") + + print() + print("🔍 PERFORMANCE ANALYSIS:") + if avg_duration > 5.0: + print(" • High batch duration - consider smaller batch sizes") + if avg_tool_time / avg_duration > 0.8: + print( + " • Tool execution is majority of time - well optimized server communication" + ) + else: + print( + " • Significant overhead beyond tool execution - check server responsiveness" + ) + + print("=" * 60) + + def _emergency_cleanup(self): + """Emergency cleanup for interrupts.""" + try: + print(" Clearing queued actions from server...") + for manager in self.instance.batch_managers: + print(f" Processing manager {manager.manager_id}...") + cleanup_results = manager.emergency_cleanup() + + # Report results + for operation, result in cleanup_results.items(): + if result == "success": + print(f" ✅ {operation}") + else: + print(f" ❌ {operation}: {result}") + + print(" ✅ Server cleanup completed") + except Exception as e: + print(f" ⚠️ Warning: Failed to clear queued actions: {e}") + + try: + self.instance.batch_manager.deactivate() + except: + pass diff --git a/fle/data/replays/run_actions_batched.py b/fle/data/replays/run_actions_batched.py new file mode 100644 index 000000000..92e8b62bf --- /dev/null +++ b/fle/data/replays/run_actions_batched.py @@ -0,0 +1,114 @@ +""" +Refactored Factorio batch action processor with clean class architecture. + +Main entry point that uses the processor classes defined in separate modules. +""" + +from pathlib import Path + +from fle.data.replays.processors import ( + SequentialProcessor, + PipelineProcessor, + ProcessingConfig, +) + + +def main(): + """Main entry point with argument parsing.""" + import argparse + + parser = argparse.ArgumentParser( + description="Execute Factorio events using batch processing" + ) + parser.add_argument( + "--max-time", + type=int, + default=10000, + help="Maximum tick time to include in events file", + ) + parser.add_argument( + "--analyze-logs", + action="store_true", + help="Analyze existing log files instead of running simulation", + ) + parser.add_argument( + "--enable-logging", + action="store_true", + help="Enable logging of entities and inventory data to files", + ) + parser.add_argument( + "--speed", + type=float, + default=1.0, + help="Game speed multiplier (default: 1.0, higher = faster)", + ) + parser.add_argument( + "--batch-size", + type=int, + default=500, + help="Tick interval for batching events (default: 500)", + ) + parser.add_argument( + "--max-concurrent", + type=int, + default=1, + help="Maximum number of concurrent batch preparation threads", + ) + parser.add_argument( + "--pipeline", + action="store_true", + help="Use non-blocking pipeline processing instead of sequential batches", + ) + parser.add_argument( + "--periodic-log-interval", + type=int, + default=0, + help="Tick interval for periodic logging (default: 0, disabled)", + ) + + args = parser.parse_args() + + if args.analyze_logs: + from log_analyzer import analyze_logs_example + + analyze_logs_example() + return + + # Determine events file + current_dir = Path(__file__).parent + events_file = ( + current_dir / "_runnable_actions" / f"combined_events_py_{args.max_time}.jsonl" + ) + + if not events_file.exists(): + print(f"Events file not found: {events_file}") + print("Please provide the path to your events JSONL file") + return + + # Create configuration + config = ProcessingConfig( + events_file_path=str(events_file), + enable_logging=args.enable_logging, + speed=args.speed, + batch_size=args.batch_size, + max_concurrent_batches=args.max_concurrent, + periodic_log_interval=args.periodic_log_interval, + ) + + # Create and run processor + if args.pipeline: + processor = PipelineProcessor(config) + print("🚀 Using pipeline batch processing (non-blocking)") + else: + processor = SequentialProcessor(config) + print("📦 Using sequential batch processing (blocking)") + + processor.setup() + try: + processor.execute() + finally: + processor.cleanup() + + +if __name__ == "__main__": + main() diff --git a/fle/data/replays/run_actions_utils.py b/fle/data/replays/run_actions_utils.py new file mode 100644 index 000000000..8724e144b --- /dev/null +++ b/fle/data/replays/run_actions_utils.py @@ -0,0 +1,59 @@ +import json +from typing import Dict, Any, List +from fle.env import FactorioInstance + +# Import moved functionality from specialized modules +from fle.data.replays.action_converter import ActionConverter + + +def load_events(file_path: str) -> List[Dict[str, Any]]: + """Load events from a JSONL file.""" + events = [] + with open(file_path, "r") as f: + for line in f: + line = line.strip() + if line: + events.append(json.loads(line)) + return events + + +def create_factorio_instance(max_concurrent_batches=1): + """Create and return a FactorioInstance with support for concurrent batch processing.""" + return FactorioInstance( + address="localhost", + bounding_box=200, + tcp_port=27000, + cache_scripts=True, + fast=True, + regenerate="map", + inventory={ + "iron-plate": 8, + "stone-furnace": 1, + "burner-mining-drill": 1, + "wood": 1, + "iron-ore": 1, + "stone": 5, + }, + max_concurrent_batches=max_concurrent_batches, + ) + + +# Legacy function aliases for backward compatibility +def parse_function_call(call_string: str) -> tuple[str, Dict[str, Any]]: + """Parse a function call string into function name and arguments. + + This function has been moved to ActionConverter.parse_function_call(). + This alias is provided for backward compatibility. + """ + return ActionConverter.parse_function_call(call_string) + + +def convert_run_actions_args_to_tool_args( + func_name: str, args: Dict[str, Any] +) -> Dict[str, Any]: + """Convert run_actions.py argument format to tool argument format. + + This function has been moved to ActionConverter.convert_legacy_args_to_tool_args(). + This alias is provided for backward compatibility. + """ + return ActionConverter.convert_legacy_args_to_tool_args(func_name, args) diff --git a/fle/data/replays/scripts/compare_inventories.py b/fle/data/replays/scripts/compare_inventories.py new file mode 100644 index 000000000..21d152890 --- /dev/null +++ b/fle/data/replays/scripts/compare_inventories.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +""" +Compare inventories between fle_actions_events and replay_inventory datasets at common tick times. +""" + +import json +import re +from typing import Dict + + +def parse_lua_inventory(lua_string: str) -> Dict[str, int]: + """Parse Lua table string into Python dictionary.""" + if not lua_string or lua_string.strip() == "": + return {} + + # Remove outer braces and split by commas + content = lua_string.strip() + if content.startswith("{") and content.endswith("}"): + content = content[1:-1] + + items = {} + # Use regex to find [key] = value pairs + pattern = r'\["([^"]+)"\]\s*=\s*(\d+)' + matches = re.findall(pattern, content) + + for item_name, quantity in matches: + items[item_name] = int(quantity) + + return items + + +def load_fle_actions_events(filename: str) -> Dict[int, Dict[str, int]]: + """Load fle_actions_events data and extract tick -> inventory mapping.""" + tick_to_inventory = {} + + with open(filename, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + + data = json.loads(line) + tick = data.get("tick") + inventory = data.get("inventory", {}) + + if tick is not None and inventory: + tick_to_inventory[tick] = inventory + + return tick_to_inventory + + +def load_replay_inventory(filename: str) -> Dict[int, Dict[str, int]]: + """Load replay_inventory data and extract tick -> inventory mapping.""" + tick_to_inventory = {} + + with open(filename, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + + data = json.loads(line) + tick = data.get("t") + + # Skip if there's an inventory error or no inventory_raw + if "inventory_error" in data or "inventory_raw" not in data: + continue + + inventory_raw = data.get("inventory_raw", "") + inventory = parse_lua_inventory(inventory_raw) + + if tick is not None and inventory: + tick_to_inventory[tick] = inventory + + return tick_to_inventory + + +def compare_inventories( + fle_actions_inv: Dict[str, int], replay_inv: Dict[str, int] +) -> Dict[str, tuple[str, str]]: + """Compare two inventories and return differences. + + Returns: + dict: {item_name: (fle_actions_display, replay_display)} for items that differ + """ + differences = {} + all_items = set(fle_actions_inv.keys()) | set(replay_inv.keys()) + + for item in all_items: + fle_actions_qty = fle_actions_inv.get(item, 0) + replay_qty = replay_inv.get(item, 0) + + if fle_actions_qty != replay_qty: + if fle_actions_qty > replay_qty: + fle_actions_display = f"+{fle_actions_qty - replay_qty}" + replay_display = "." + else: + fle_actions_display = "." + replay_display = f"+{replay_qty - fle_actions_qty}" + + differences[item] = (fle_actions_display, replay_display) + + return differences + + +def main(): + # Load both datasets + print("Loading fle_actions_events data...") + fle_actions_events = load_fle_actions_events( + "_runnable_actions/combined_events_py_10027_periodic_data.jsonl" + ) + + print("Loading replay_inventory data...") + replay_inventory = load_replay_inventory( + "replay_observations/inspect_inventory.jsonl" + ) + + # Find common ticks + common_ticks = set(fle_actions_events.keys()) & set(replay_inventory.keys()) + common_ticks = sorted(common_ticks) + + print(f"Found {len(common_ticks)} common tick times") + print(f"Common ticks: {common_ticks}") + + # Compare inventories at common ticks and group consecutive identical differences + differences_found = 0 + current_differences = None + current_tick_range = [] + + def print_differences_group(tick_range, differences): + """Print a group of ticks with the same differences.""" + nonlocal differences_found + differences_found += 1 + + if len(tick_range) == 1: + print(f"\n=== DIFFERENCES AT TICK {tick_range[0]} ===") + else: + print( + f"\n=== DIFFERENCES AT TICK(S) {tick_range[0]} - {tick_range[-1]} ===" + ) + + print("Item FLE_Actions Replay") + print("-" * 45) + + for item in sorted(differences.keys()): + fle_actions_display, replay_display = differences[item] + print(f"{item:<20} {fle_actions_display:>11} {replay_display:>6}") + + for tick in common_ticks: + fle_actions_inv = fle_actions_events[tick] + replay_inv = replay_inventory[tick] + + differences = compare_inventories(fle_actions_inv, replay_inv) + + if differences: + # Check if differences are the same as the previous tick + if current_differences == differences: + # Same differences, add to current range + current_tick_range.append(tick) + else: + # Different differences, print previous group if it exists + if current_differences is not None and current_tick_range: + print_differences_group(current_tick_range, current_differences) + + # Start new group + current_differences = differences + current_tick_range = [tick] + else: + # No differences, print previous group if it exists + if current_differences is not None and current_tick_range: + print_differences_group(current_tick_range, current_differences) + current_differences = None + current_tick_range = [] + + # Print the final group if it exists + if current_differences is not None and current_tick_range: + print_differences_group(current_tick_range, current_differences) + + if differences_found == 0: + print("\n✅ No differences found! All inventories match at common tick times.") + else: + print(f"\n❌ Found differences at {differences_found} group(s) of tick times.") + + +if __name__ == "__main__": + main() diff --git a/fle/data/replays/scripts/example_batch_streaming.py b/fle/data/replays/scripts/example_batch_streaming.py new file mode 100644 index 000000000..78d74d94f --- /dev/null +++ b/fle/data/replays/scripts/example_batch_streaming.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 + +import sys +import os +import time + +# Add the FLE package to Python path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".")) + +from fle.env.instance import FactorioInstance +from fle.env.entities import ( + Position, + Direction, + Dimensions, + TileDimensions, + PlaceholderEntity, +) +from fle.env.game_types import Prototype + + +def convert_dict_to_entity(entity_dict): + """Convert dictionary entity data to proper Entity object.""" + if not isinstance(entity_dict, dict): + return entity_dict + + # Find the matching Prototype + matching_prototype = None + for prototype in Prototype: + if prototype.value[0] == entity_dict["name"].replace("_", "-"): + matching_prototype = prototype + break + + if matching_prototype is None: + print(f"Warning: No matching Prototype found for {entity_dict['name']}") + return entity_dict + + # Get the metaclass from the prototype + metaclass = matching_prototype.value[1] + while isinstance(metaclass, tuple): + metaclass = metaclass[1] + + # Convert the entity data + entity_data = entity_dict.copy() + + # Convert direction from int to Direction enum + if "direction" in entity_data and isinstance(entity_data["direction"], int): + direction_value = entity_data["direction"] + # Convert factorio direction (0,2,4,6) to Direction enum + direction_map = { + 0: Direction.UP, + 2: Direction.RIGHT, + 4: Direction.DOWN, + 6: Direction.LEFT, + } + entity_data["direction"] = direction_map.get(direction_value, Direction.UP) + + # Convert position dict to Position object + if "position" in entity_data and isinstance(entity_data["position"], dict): + pos_dict = entity_data["position"] + entity_data["position"] = Position(x=pos_dict["x"], y=pos_dict["y"]) + + # Convert dimensions dict to Dimensions object + if "dimensions" in entity_data and isinstance(entity_data["dimensions"], dict): + dim_dict = entity_data["dimensions"] + entity_data["dimensions"] = Dimensions( + width=dim_dict["width"], height=dim_dict["height"] + ) + + # Convert tile_dimensions dict to TileDimensions object + if "tile_dimensions" in entity_data and isinstance( + entity_data["tile_dimensions"], dict + ): + tile_dim_dict = entity_data["tile_dimensions"] + entity_data["tile_dimensions"] = TileDimensions( + tile_width=tile_dim_dict["tile_width"], + tile_height=tile_dim_dict["tile_height"], + ) + + # Convert warnings dict to list of strings + if "warnings" in entity_data and isinstance(entity_data["warnings"], dict): + warnings_dict = entity_data["warnings"] + entity_data["warnings"] = list(warnings_dict.values()) + + # Add the prototype to the entity data + entity_data["prototype"] = matching_prototype + + # Remove any empty values that might cause issues + entity_data = { + k: v for k, v in entity_data.items() if v is not None or isinstance(v, int) + } + + try: + # Create the Entity object + entity = metaclass(**entity_data) + return entity + except Exception as e: + print(f"Could not create {entity_data['name']} Entity object: {e}") + print(f"Entity data: {entity_data}") + return entity_dict # Return original dict if conversion fails + + +def example_batch_streaming(): + """Example showing streaming batch execution where results are yielded as they become available.""" + + # Create a Factorio instance with expanded inventory for more complex operations + instance = FactorioInstance( + address="localhost", + tcp_port=27000, + inventory={ + "stone": 50, + "iron-ore": 30, + "coal": 20, # Added coal for furnace fuel demonstration + "burner-mining-drill": 10, + "stone-furnace": 10, + "transport-belt": 20, + "inserter": 10, + "assembling-machine-1": 5, + "iron-plate": 20, + "copper-plate": 10, + "iron-gear-wheel": 5, + "iron-chest": 5, + }, + fast=True, + ) + + namespace = instance.namespace + + try: + print( + "=== Enhanced Streaming Batch Processing Example (Two-Batch Approach) ===\n" + ) + + # Reset the instance to clean state + print("0. Resetting instance...") + instance.reset() + + # current_tick = instance.get_elapsed_ticks() + current_tick = 0 + print(f" Current game tick: {current_tick}") + + # ===================== FIRST BATCH ===================== + print("1. First batch: Creating entities and basic operations...") + instance.batch_manager.activate() + + # Add commands that don't depend on entity references + commands_info_batch1 = [] + + # Command 1: Set research first (quick operation) + # tick1 = current_tick + 5 + # result1 = namespace.set_research(Technology.Automation, tick=tick1) + # commands_info_batch1.append( + # (f"Set research to Automation at tick {tick1}", result1) + # ) + + # Command 2: Move to working area + tick2 = current_tick + 20 + result2 = namespace.move_to(Position(x=10, y=5), tick=tick2) + commands_info_batch1.append((f"Move to (10,5) at tick {tick2}", result2)) + + # Command 3: Harvest some stone nearby + tick3 = current_tick + 80 + result3 = namespace.harvest_resource( + Position(x=12, y=5), quantity=10, tick=tick3 + ) + commands_info_batch1.append( + (f"Harvest iron ore at (12,5) at tick {tick3}", result3) + ) + + # Command 4: Craft some iron gear wheels + tick4 = current_tick + 130 + result4 = namespace.craft_item(Prototype.IronGearWheel, quantity=3, tick=tick4) + commands_info_batch1.append( + (f"Craft 3 iron gear wheels at tick {tick4}", result4) + ) + + # Command 5: Place stone furnace + tick5 = current_tick + 180 + result5 = namespace.place_entity( + Prototype.StoneFurnace, position=Position(x=10, y=6), tick=tick5 + ) + commands_info_batch1.append( + (f"Place stone furnace at (10,6) at tick {tick5}", result5) + ) + + # Command 6: Place iron chest (for insert/extract operations) + tick6 = current_tick + 220 + result6 = namespace.place_entity( + Prototype.IronChest, position=Position(x=10, y=7), tick=tick6 + ) + commands_info_batch1.append( + (f"Place iron chest at (10,7) at tick {tick6}", result6) + ) + + # Command 7: Place inserter (we'll need this entity for batch 2) + tick7 = current_tick + 260 + result7 = namespace.place_entity( + Prototype.Inserter, position=Position(x=11, y=6), tick=tick7 + ) + commands_info_batch1.append( + (f"Place inserter at (11,6) at tick {tick7}", result7) + ) + + # Command 8: Place assembling machine (we'll need this entity for batch 2) + tick8 = current_tick + 300 + result8 = namespace.place_entity( + Prototype.AssemblingMachine1, position=Position(x=11, y=10), tick=tick8 + ) + commands_info_batch1.append( + (f"Place assembling machine at (11,10) at tick {tick8}", result8) + ) + + # Command 9: Insert iron plates into the chest using PlaceholderEntity + # Create a PlaceholderEntity for the chest we placed earlier + chest_placeholder = PlaceholderEntity( + name="iron-chest", position=Position(x=10, y=7) + ) + tick9 = current_tick + 340 + result9 = namespace.insert_item( + Prototype.IronPlate, chest_placeholder, quantity=10, tick=tick9 + ) + commands_info_batch1.append( + ( + f"Insert 10 iron plates into chest using PlaceholderEntity at tick {tick9}", + result9, + ) + ) + + # Command 10: Extract iron plates from the chest using PlaceholderEntity + tick10 = current_tick + 380 + result10 = namespace.extract_item( + Prototype.IronPlate, chest_placeholder, quantity=5, tick=tick10 + ) + commands_info_batch1.append( + ( + f"Extract 5 iron plates from chest using PlaceholderEntity at tick {tick10}", + result10, + ) + ) + + print(" Commands added to first batch:") + for desc, result in commands_info_batch1: + print(f" {desc}: {result}") + + # Submit first batch and stream results as they arrive + print("\n2. Submitting first batch and streaming results...") + start_time = time.time() + + batch1_results = {} # Store results indexed by command_index + batch1_results_count = 0 + + # Stream results from first batch + for result in instance.batch_manager.submit_batch_and_stream( + timeout_seconds=30, poll_interval=0.05 + ): + batch1_results_count += 1 + elapsed = time.time() - start_time + command_index = result["command_index"] + + # Store the result for later entity extraction + batch1_results[command_index] = result + + print( + f" ✓ First batch result {batch1_results_count} received after {elapsed:.2f}s:" + ) + print(f" Command: {result['command']} (index {command_index})") + print(f" Success: {result['success']}") + if result["success"]: + print(f" Result: {result['result']}") + else: + print(f" Error: {result['result']}") + print() + + batch1_time = time.time() - start_time + print( + f" First batch completed in {batch1_time:.2f}s with {batch1_results_count} results" + ) + + # Extract real entity references from first batch streamed results + # result7 is the inserter (index 6), result8 is the assembling machine (index 7) + # result5 is the stone furnace (index 4), result6 is the iron chest (index 5) + # result9 is insert_item (index 8), result10 is extract_item (index 9) + inserter_entity = None + assembling_machine_entity = None + stone_furnace_entity = None + iron_chest_entity = None + + for command_index, result in batch1_results.items(): + if result["success"]: + if command_index == 5: # result7 - inserter + inserter_entity = convert_dict_to_entity(result["result"]) + print(f" ✓ Inserter entity created: {inserter_entity}") + elif command_index == 6: # result8 - assembling machine + assembling_machine_entity = convert_dict_to_entity(result["result"]) + print( + f" ✓ Assembling machine entity created: {assembling_machine_entity}" + ) + elif command_index == 3: # result5 - stone furnace + stone_furnace_entity = convert_dict_to_entity(result["result"]) + print( + f" ✓ Stone furnace entity created: {stone_furnace_entity}" + ) + elif command_index == 4: # result6 - iron chest + iron_chest_entity = convert_dict_to_entity(result["result"]) + print(f" ✓ Iron chest entity created: {iron_chest_entity}") + elif command_index == 7: # result9 - insert_item + print(f" ✓ Insert item command executed: {result['result']}") + elif command_index == 8: # result10 - extract_item + print(f" ✓ Extract item command executed: {result['result']}") + else: + print(f" ✗ Command {command_index + 1} failed: {result['result']}") + + instance.batch_manager.deactivate() + + # ===================== SECOND BATCH ===================== + print("\n3. Second batch: Operations using real entity references...") + instance.batch_manager.activate() + + commands_info_batch2 = [] + + # Update current tick after first batch execution + # current_tick = instance.get_elapsed_ticks() + current_tick = 0 + + # Command 9: Rotate the inserter (using real entity reference) + tick9 = current_tick + 40 + if inserter_entity: + result9 = namespace.rotate_entity(inserter_entity, tick=tick9) + commands_info_batch2.append((f"Rotate inserter at tick {tick9}", result9)) + else: + print(" ⚠ Skipping inserter rotation - entity not available") + + # Command 10: Set recipe for assembling machine (using real entity reference) + tick10 = current_tick + 80 + if assembling_machine_entity: + result10 = namespace.set_entity_recipe( + assembling_machine_entity, Prototype.IronGearWheel, tick=tick10 + ) + commands_info_batch2.append( + ( + f"Set assembling machine recipe to iron gear wheel at tick {tick10}", + result10, + ) + ) + else: + print( + " ⚠ Skipping recipe setting - assembling machine entity not available" + ) + + # Command 11: Extract items from furnace (using real entity reference) + # Skip this for now since the furnace won't have any items yet + # tick11 = current_tick + 120 + # if stone_furnace_entity: + # result11 = namespace.extract_item(Prototype.IronPlate, stone_furnace_entity, quantity=5, tick=tick11) + # commands_info_batch2.append((f"Extract 5 iron plates from furnace at tick {tick11}", result11)) + # else: + # print(" ⚠ Skipping item extraction - stone furnace entity not available") + print( + " ℹ Skipping furnace extraction - furnace is empty (no iron ore was added)" + ) + + # Command 11: Insert coal into the stone furnace using PlaceholderEntity + # This demonstrates referencing an entity created in the first batch + furnace_placeholder = PlaceholderEntity( + name="stone-furnace", position=Position(x=10, y=6) + ) + tick11 = current_tick + 80 + result11 = namespace.insert_item( + Prototype.Coal, furnace_placeholder, quantity=5, tick=tick11 + ) + commands_info_batch2.append( + ( + f"Insert 5 coal into stone furnace using PlaceholderEntity at tick {tick11}", + result11, + ) + ) + + # Command 12: Inspect inventory to see current state + tick12 = current_tick + 120 # Moved up since we skipped command 11 + result12 = namespace.inspect_inventory(tick=tick12) + commands_info_batch2.append((f"Inspect inventory at tick {tick12}", result12)) + + # Command 13: Pick up the transport belt (cleanup) - use actual position + tick13 = current_tick + 160 # Moved up since we skipped command 11 + if iron_chest_entity: + result13 = namespace.pickup_entity(iron_chest_entity, tick=tick13) + commands_info_batch2.append( + ( + f"Pick up iron chest at {iron_chest_entity.position} at tick {tick13}", + result13, + ) + ) + else: + print(" ⚠ Skipping iron chest pickup - entity not available") + + print(" Commands added to second batch:") + for desc, result in commands_info_batch2: + print(f" {desc}: {result}") + + # Stream results from second batch as they become available + print("\n4. Streaming results from second batch as they complete...") + print(" (Results will appear as soon as each command finishes)\n") + + results_received = 0 + start_time = time.time() + + # Use the streaming method for second batch + for result in instance.batch_manager.submit_batch_and_stream( + timeout_seconds=25, poll_interval=0.05 + ): + results_received += 1 + elapsed = time.time() - start_time + + print(f" ✓ Result {results_received} received after {elapsed:.2f}s:") + print( + f" Command: {result['command']} (index {result['command_index']})" + ) + print(f" Success: {result['success']}") + print(f" Executed at tick: {result['tick']}") + + if result["success"]: + print(f" Result: {result['result']}") + else: + print(f" Error: {result['result']}") + print() + + batch2_time = time.time() - start_time + total_time = batch1_time + batch2_time + print( + f" Second batch: {results_received} results received in {batch2_time:.2f}s" + ) + print(f" Total processing time: {total_time:.2f}s") + + # Deactivate batch mode + print("\n5. Deactivating batch mode...") + instance.batch_manager.deactivate() + + print("\n=== Enhanced two-batch streaming processing complete! ===") + print("Summary:") + print( + f" - First batch (entity creation): {batch1_results_count} commands in {batch1_time:.2f}s" + ) + print( + f" - Second batch (entity operations): {results_received} commands in {batch2_time:.2f}s" + ) + print(f" - Total time: {total_time:.2f}s") + + except Exception as e: + print(f"Error during batch processing: {e}") + instance.batch_manager.deactivate() + raise + + finally: + instance.cleanup() + + +def comparison_example(): + """Compare streaming vs traditional batch processing.""" + + print("=== Comparison: Streaming vs Traditional Batch Processing ===\n") + + instance = FactorioInstance( + address="localhost", + tcp_port=27000, + inventory={ + "stone": 50, + "transport-belt": 10, + "inserter": 5, + "iron-plate": 20, + "coal": 10, + }, + ) + + namespace = instance.namespace + + try: + # Reset instance + instance.reset() + + # Test with traditional approach first + print("1. Traditional batch processing (wait for all results):") + instance.batch_manager.activate() + + current_tick = instance.get_elapsed_ticks() + + tick1 = current_tick + 10 + tick2 = current_tick + 70 + tick3 = current_tick + 130 + tick4 = current_tick + 180 + + namespace.move_to(Position(x=5, y=5), tick=tick1) + namespace.place_entity( + Prototype.TransportBelt, position=Position(x=5, y=6), tick=tick2 + ) + namespace.craft_item(Prototype.IronGearWheel, quantity=2, tick=tick3) + namespace.inspect_inventory(tick=tick4) + + start_time = time.time() + results = instance.batch_manager.submit_batch_and_wait(timeout_seconds=15) + traditional_time = time.time() - start_time + + print( + f" Traditional approach: received {len(results)} results in {traditional_time:.2f}s" + ) + instance.batch_manager.deactivate() + + # Reset for next test + instance.reset() + + # Test with streaming approach + print("\n2. Streaming batch processing (results as available):") + instance.batch_manager.activate() + + current_tick = instance.get_elapsed_ticks() + + tick1 = current_tick + 10 + tick2 = current_tick + 70 + tick3 = current_tick + 130 + tick4 = current_tick + 180 + + namespace.move_to(Position(x=10, y=10), tick=tick1) + namespace.place_entity( + Prototype.TransportBelt, position=Position(x=10, y=11), tick=tick2 + ) + namespace.craft_item(Prototype.IronGearWheel, quantity=2, tick=tick3) + namespace.inspect_inventory(tick=tick4) + + start_time = time.time() + results_count = 0 + first_result_time = None + + for result in instance.batch_manager.submit_batch_and_stream( + poll_interval=0.05 + ): + results_count += 1 + if first_result_time is None: + first_result_time = time.time() - start_time + + streaming_time = time.time() - start_time + + print( + f" Streaming approach: received {results_count} results in {streaming_time:.2f}s" + ) + print(f" First result available after: {first_result_time:.2f}s") + + instance.batch_manager.deactivate() + + print("\n3. Performance comparison:") + print(f" Traditional: {traditional_time:.2f}s (wait for all)") + print( + f" Streaming: {streaming_time:.2f}s total, {first_result_time:.2f}s for first result" + ) + print( + f" Benefit: Start processing results {traditional_time - first_result_time:.2f}s earlier!" + ) + + except Exception as e: + print(f"Error during comparison: {e}") + instance.batch_manager.deactivate() + raise + + finally: + instance.cleanup() + + +if __name__ == "__main__": + print("Choose an example to run:") + print("1. Enhanced streaming batch processing example") + print("2. Comparison between streaming and traditional approaches") + print("3. Run both examples") + + # choice = input("Enter choice (1-3): ").strip() + choice = "1" + if choice == "1": + example_batch_streaming() + elif choice == "2": + comparison_example() + elif choice == "3": + example_batch_streaming() + print("\n" + "=" * 60 + "\n") + comparison_example() + else: + print("Invalid choice. Running streaming example by default.") + example_batch_streaming() diff --git a/fle/data/replays/scripts/log_analyzer.py b/fle/data/replays/scripts/log_analyzer.py new file mode 100644 index 000000000..8e862b234 --- /dev/null +++ b/fle/data/replays/scripts/log_analyzer.py @@ -0,0 +1,119 @@ +import json + + +def read_entities_log_as_dataframe(log_file_path: str = "logs/entities_log.jsonl"): + """ + Read the entities log file as a pandas DataFrame. + + Args: + log_file_path: Path to the entities log file + + Returns: + pandas.DataFrame with columns 'tick' and 'data' + + Example: + df = read_entities_log_as_dataframe() + print(df.head()) + # Access entities at specific tick + entities_at_tick_100 = df[df['tick'] == 100]['data'].iloc[0] + """ + try: + import pandas as pd + + data = [] + with open(log_file_path, "r") as f: + for line in f: + if line.strip(): + data.append(json.loads(line)) + + return pd.DataFrame(data) + except ImportError: + print("pandas is required to read log files as DataFrames") + print("Install with: pip install pandas") + return None + except FileNotFoundError: + print(f"Log file not found: {log_file_path}") + return None + + +def read_inventory_log_as_dataframe(log_file_path: str = "logs/inventory_log.jsonl"): + """ + Read the inventory log file as a pandas DataFrame. + + Args: + log_file_path: Path to the inventory log file + + Returns: + pandas.DataFrame with columns 'tick' and 'data' + + Example: + df = read_inventory_log_as_dataframe() + print(df.head()) + # Access inventory at specific tick + inventory_at_tick_100 = df[df['tick'] == 100]['data'].iloc[0] + """ + try: + import pandas as pd + + data = [] + with open(log_file_path, "r") as f: + for line in f: + if line.strip(): + data.append(json.loads(line)) + + return pd.DataFrame(data) + except ImportError: + print("pandas is required to read log files as DataFrames") + print("Install with: pip install pandas") + return None + except FileNotFoundError: + print(f"Log file not found: {log_file_path}") + return None + + +def analyze_logs_example(): + """ + Example function showing how to analyze the logged data. + """ + try: + # Read both log files + entities_df = read_entities_log_as_dataframe() + inventory_df = read_inventory_log_as_dataframe() + + if entities_df is not None and inventory_df is not None: + print("=== Entities Log Analysis ===") + print(f"Total ticks logged: {len(entities_df)}") + print( + f"Tick range: {entities_df['tick'].min()} to {entities_df['tick'].max()}" + ) + print("\nFirst few entries:") + print(entities_df.head()) + + print("\n=== Inventory Log Analysis ===") + print(f"Total ticks logged: {len(inventory_df)}") + print( + f"Tick range: {inventory_df['tick'].min()} to {inventory_df['tick'].max()}" + ) + print("\nFirst few entries:") + print(inventory_df.head()) + + # Example: Find ticks where inventory changed + print("\n=== Analysis Example ===") + if len(inventory_df) > 1: + inventory_changes = [] + for i in range(1, len(inventory_df)): + if inventory_df.iloc[i]["data"] != inventory_df.iloc[i - 1]["data"]: + inventory_changes.append(inventory_df.iloc[i]["tick"]) + + print( + f"Inventory changed at ticks: {inventory_changes[:10]}..." + ) # Show first 10 + + else: + print( + "Could not load log files. Make sure they exist and pandas is installed." + ) + + except ImportError: + print("pandas is required for log analysis") + print("Install with: pip install pandas") diff --git a/fle/data/replays/scripts/stress_test_batch.py b/fle/data/replays/scripts/stress_test_batch.py new file mode 100644 index 000000000..9302c8b9d --- /dev/null +++ b/fle/data/replays/scripts/stress_test_batch.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 + +import sys +import os +import time +import random + +# Add the FLE package to Python path +sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".")) + +from fle.env.instance import FactorioInstance +from fle.env.entities import Position + + +def stress_test_batch_processing(): + """Stress test script that executes 2000 move_to commands over 20000 ticks.""" + + print("=== Factorio Server Stress Test: 2000 Commands over 20000 Ticks ===\n") + + # Create a Factorio instance + instance = FactorioInstance( + address="localhost", + tcp_port=27000, + inventory={ + "stone": 10, # Minimal inventory since we're just moving + }, + fast=True, + ) + + namespace = instance.namespace + + try: + print("0. Resetting instance...") + instance.reset() + + # Configuration + TOTAL_COMMANDS = 2000 + TOTAL_TICKS = 20000 + TICKS_PER_COMMAND = 10 # 1 command every 10 ticks + BATCH_SIZE = 1000 # Process in batches of 200 commands to avoid overwhelming + + print(" Configuration:") + print(f" - Total commands: {TOTAL_COMMANDS}") + print(f" - Total ticks: {TOTAL_TICKS}") + print(f" - Ticks per command: {TICKS_PER_COMMAND}") + print(f" - Batch size: {BATCH_SIZE}") + print(f" - Number of batches: {TOTAL_COMMANDS // BATCH_SIZE}") + + # Generate random positions for movement (within a reasonable area) + def generate_random_position(): + x = random.uniform(-50, 50) # Random x between -50 and 50 + y = random.uniform(-50, 50) # Random y between -50 and 50 + return Position(x=x, y=y) + + total_start_time = time.time() + total_results_received = 0 + total_successful = 0 + total_failed = 0 + + # Process commands in batches + num_batches = TOTAL_COMMANDS // BATCH_SIZE + + for batch_num in range(num_batches): + print(f"\n=== BATCH {batch_num + 1}/{num_batches} ===") + + instance.batch_manager.activate() + + batch_commands = [] + start_command = batch_num * BATCH_SIZE + end_command = min((batch_num + 1) * BATCH_SIZE, TOTAL_COMMANDS) + + # Add commands to this batch + for i in range(start_command, end_command): + tick = i * TICKS_PER_COMMAND + position = generate_random_position() + + result = namespace.move_to(position, tick=tick) + batch_commands.append( + ( + f"Move to ({position.x:.1f}, {position.y:.1f}) at tick {tick}", + result, + tick, + ) + ) + + print(f" Added {len(batch_commands)} commands to batch {batch_num + 1}") + print(f" Command range: {start_command} to {end_command - 1}") + print( + f" Tick range: {start_command * TICKS_PER_COMMAND} to {(end_command - 1) * TICKS_PER_COMMAND}" + ) + + # Submit batch and stream results + batch_start_time = time.time() + batch_results_received = 0 + batch_successful = 0 + batch_failed = 0 + first_result_time = None + + print(" Submitting batch and streaming results...") + + for result in instance.batch_manager.submit_batch_and_stream( + timeout_seconds=600, poll_interval=0.1 + ): + batch_results_received += 1 + total_results_received += 1 + + if first_result_time is None: + first_result_time = time.time() - batch_start_time + + if result["success"]: + batch_successful += 1 + total_successful += 1 + else: + batch_failed += 1 + total_failed += 1 + + # Print progress every 50 results or for failed commands + if batch_results_received % 50 == 0 or not result["success"]: + elapsed = time.time() - batch_start_time + print( + f" ✓ Batch progress: {batch_results_received}/{len(batch_commands)} " + f"results received after {elapsed:.2f}s" + ) + + if not result["success"]: + print( + f" ✗ Command failed: {result['command']} - {result['result']}" + ) + + batch_time = time.time() - batch_start_time + + print(f" Batch {batch_num + 1} completed:") + print(f" - Results received: {batch_results_received}") + print(f" - Successful: {batch_successful}") + print(f" - Failed: {batch_failed}") + print(f" - Batch time: {batch_time:.2f}s") + print(f" - First result time: {first_result_time:.2f}s") + print( + f" - Commands per second: {batch_results_received / batch_time:.1f}" + ) + + instance.batch_manager.deactivate() + + # Brief pause between batches to let the server recover + if batch_num < num_batches - 1: + print(" Pausing 2 seconds before next batch...") + time.sleep(2) + + total_time = time.time() - total_start_time + + print("\n=== STRESS TEST COMPLETE ===") + print("Summary:") + print(f" - Total commands sent: {TOTAL_COMMANDS}") + print(f" - Total results received: {total_results_received}") + print(f" - Successful commands: {total_successful}") + print(f" - Failed commands: {total_failed}") + print( + f" - Success rate: {(total_successful / total_results_received * 100):.1f}%" + ) + print(f" - Total execution time: {total_time:.2f}s") + print( + f" - Average commands per second: {total_results_received / total_time:.1f}" + ) + print(f" - Expected game duration: {TOTAL_TICKS} ticks") + + if total_failed > 0: + print( + f"\n⚠️ Warning: {total_failed} commands failed. Check server logs for details." + ) + else: + print("\n✅ All commands completed successfully!") + + except Exception as e: + print(f"Error during stress test: {e}") + instance.batch_manager.deactivate() + raise + + finally: + instance.cleanup() + + +if __name__ == "__main__": + print("⚠️ Running FULL stress test - this will take several minutes!") + stress_test_batch_processing() diff --git a/fle/env/entities.py b/fle/env/entities.py index cda514032..c8f2551df 100644 --- a/fle/env/entities.py +++ b/fle/env/entities.py @@ -177,6 +177,13 @@ def from_string(cls, direction_string): return status return None + @classmethod + def from_int(cls, direction_int): + for status in cls: + if status.value == direction_int: + return status + return None + class Position(BaseModel): x: float @@ -425,6 +432,40 @@ def __repr__(self): return f"Entity(name='{self.name}', direction={self.direction.name}, position=Position({self.position})" +class PlaceholderEntity(BaseModel): + """A placeholder entity used in batch mode when we don't have access to actual server-side entities yet.""" + + name: str + position: Position + direction: Optional[Direction] = Direction.UP + recipe: Optional[str] = None # For assembling machines + filter: Optional[str] = None # For filter inserters + + def __init__( + self, + name: str, + position: Position, + direction: Optional[Direction] = Direction.UP, + **kwargs, + ): + # Handle case where position might be passed as tuple + if isinstance(position, tuple): + position = Position(x=position[0], y=position[1]) + super().__init__(name=name, position=position, direction=direction, **kwargs) + + def __repr__(self): + base_repr = f"PlaceholderEntity(name='{self.name}', position={self.position}, direction={self.direction}" + + # Add recipe or filter info if present + if self.recipe: + base_repr += f", recipe='{self.recipe}'" + if self.filter: + base_repr += f", filter='{self.filter}'" + + base_repr += ")" + return base_repr + + class Entity(EntityCore): """Base class for all entities in the game.""" diff --git a/fle/env/game_types.py b/fle/env/game_types.py index 69eff9e37..fb05fac6b 100644 --- a/fle/env/game_types.py +++ b/fle/env/game_types.py @@ -1,5 +1,6 @@ from __future__ import annotations import enum +import json from difflib import get_close_matches from fle.env import entities as ent @@ -80,6 +81,79 @@ class RecipeName(enum.Enum): EmptyWaterBarrel = "empty-water-barrel" +class PrototypeJSONEncoder(json.JSONEncoder): + """Custom JSON encoder for Prototype objects and Pydantic BaseModel instances.""" + + def default(self, obj): + # Import here to avoid circular imports + from pydantic import BaseModel + + if isinstance(obj, Prototype): + return obj.to_dict() + elif isinstance(obj, BaseModel): + # Handle Pydantic BaseModel instances (like Position) + return { + "__pydantic__": True, + "model": obj.__class__.__name__, + "data": obj.model_dump(), + } + return super().default(obj) + + +def prototype_json_hook(dct): + """JSON decode hook for Prototype objects and Pydantic BaseModel instances.""" + if isinstance(dct, dict): + if dct.get("__prototype__"): + return Prototype.from_dict(dct) + elif dct.get("__pydantic__"): + # Handle Pydantic BaseModel instances + model_name = dct["model"] + data = dct["data"] + + # Map model names to classes + if model_name == "Position": + return ent.Position(**data) + elif model_name == "BoundingBox": + return ent.BoundingBox(**data) + # Add more model mappings as needed + else: + # Fallback: return the data dict for unknown models + return data + return dct + + +def encode_prototypes(obj): + """Encode objects containing Prototypes and Pydantic models to JSON string.""" + return json.dumps(obj, cls=PrototypeJSONEncoder) + + +def decode_prototypes(json_str): + """Decode JSON string back to objects with Prototypes and Pydantic models.""" + return json.loads(json_str, object_hook=prototype_json_hook) + + +def encode_prototypes_safe(obj): + """Safely encode objects that might contain Prototypes and Pydantic models.""" + try: + return json.dumps(obj, cls=PrototypeJSONEncoder) + except (TypeError, ValueError): + # Fallback: convert complex objects to their string representation + from pydantic import BaseModel + + def convert_objects(item): + if isinstance(item, Prototype): + return str(item) + elif isinstance(item, BaseModel): + return item.model_dump() + elif isinstance(item, dict): + return {k: convert_objects(v) for k, v in item.items()} + elif isinstance(item, (list, tuple)): + return type(item)(convert_objects(i) for i in item) + return item + + return json.dumps(convert_objects(obj)) + + class Prototype(enum.Enum, metaclass=PrototypeMetaclass): AssemblingMachine1 = "assembling-machine-1", ent.AssemblingMachine AssemblingMachine2 = "assembling-machine-2", ent.AdvancedAssemblingMachine @@ -226,6 +300,19 @@ def __init__(self, prototype_name, entity_class_name): self.prototype_name = prototype_name self.entity_class = entity_class_name + def __reduce_ex__(self, protocol): + """Enable pickling/JSON serialization by returning the prototype name.""" + return (self.__class__._prototype_reconstructor, (self.name,)) + + @classmethod + def _prototype_reconstructor(cls, name): + """Reconstruct a Prototype instance from its name.""" + return getattr(cls, name) + + def __str__(self): + """Return the prototype name for string representation.""" + return self.prototype_name + @property def WIDTH(self): return self.entity_class._width.default # Access the class attribute directly @@ -234,6 +321,42 @@ def WIDTH(self): def HEIGHT(self): return self.entity_class._height.default + def __json__(self): + """Custom JSON serialization method.""" + return { + "__prototype__": True, + "name": self.name, + "prototype_name": self.prototype_name, + } + + @classmethod + def from_json(cls, data): + """Create a Prototype instance from JSON data.""" + if isinstance(data, dict) and data.get("__prototype__"): + prototype_name = data["name"] + # Look up the prototype by name + for prototype in cls: + if prototype.name == prototype_name: + return prototype + raise ValueError(f"Unknown prototype name: {prototype_name}") + return data + + def to_dict(self): + """Convert to a simple dictionary for JSON serialization.""" + return { + "__prototype__": True, + "name": self.name, + "prototype_name": self.prototype_name, + } + + @classmethod + def from_dict(cls, data): + """Create a Prototype instance from a dictionary.""" + if isinstance(data, dict) and data.get("__prototype__"): + name = data["name"] + return getattr(cls, name) + return data + prototype_by_name = {prototype.value[0]: prototype for prototype in Prototype} prototype_by_title = {str(prototype): prototype for prototype in Prototype} diff --git a/fle/env/instance.py b/fle/env/instance.py index 6e7e8feb8..e1792b32b 100644 --- a/fle/env/instance.py +++ b/fle/env/instance.py @@ -1,4 +1,5 @@ import atexit +import contextlib import enum import functools import importlib @@ -32,6 +33,16 @@ from fle.env.utils.controller_loader.system_prompt_generator import ( SystemPromptGenerator, ) +from fle.env.game_types import PrototypeJSONEncoder + +# Add Lua syntax validation +try: + from luaparser import ast as lua_ast + + LUAPARSER_AVAILABLE = True +except ImportError: + LUAPARSER_AVAILABLE = False + print("WARNING: luaparser not available. Install with: pip install luaparser") CHUNK_SIZE = 32 MAX_SAMPLES = 5000 @@ -87,6 +98,499 @@ def get_commands(self): return self.commands +class BatchManager: + def __init__(self, instance, grace_period=5, rcon_connection=None, manager_id=0): + self.instance = instance + self.is_active = False + self.scheduled_commands = [] + self.current_batch_id = None + self.grace_period = grace_period # Default 5 tick grace period + self.rcon_connection = rcon_connection # Store the RCON connection + self.manager_id = manager_id # Store the manager ID + + # Add connection validation + self.connection_healthy = False + self.last_error = None + self._validate_connection() + + # Generate unique sequence ID per Python script run + import uuid + + self.sequence_id = f"seq_{uuid.uuid4()}" + + def _validate_connection(self): + """Validate that the RCON connection is working properly.""" + if not self.rcon_connection: + self.connection_healthy = True + return + + try: + # Test basic connectivity + self.rcon_connection.send_command( + "/sc rcon.print('health_check_" + str(self.manager_id) + "')" + ) + + # Test if we can access global actions + self.rcon_connection.send_command("/sc rcon.print(type(global.actions))") + + # Test if we can create a sequence + self.rcon_connection.send_command( + f'/sc rcon.print("sequence_test_manager_{self.manager_id}")' + ) + + self.connection_healthy = True + + except Exception as e: + self.connection_healthy = False + self.last_error = str(e) + print( + f"❌ BatchManager {self.manager_id}: Connection validation failed: {e}" + ) + + def get_health_status(self): + """Get the health status of this batch manager.""" + return { + "manager_id": self.manager_id, + "connection_healthy": self.connection_healthy, + "last_error": self.last_error, + "is_active": self.is_active, + "current_batch_id": self.current_batch_id, + "has_dedicated_connection": self.rcon_connection is not None, + } + + def activate(self): + """Enable batch mode for all tools.""" + self.is_active = True + # Clear any existing commands + self.scheduled_commands.clear() + self.current_batch_id = None + # Set this batch manager on all tools + for namespace in self.instance.namespaces: + for tool_name in self.instance.controllers: + controller = self.instance.controllers[tool_name] + controller.set_batch_manager(self) + + def deactivate(self): + """Disable batch mode.""" + self.is_active = False + self.scheduled_commands.clear() + self.current_batch_id = None + for namespace in self.instance.namespaces: + for tool_name in self.instance.controllers: + controller = self.instance.controllers[tool_name] + controller.set_batch_manager(None) + + def add_tool_command(self, tick: int, tool_name: str, player_index: int, *args): + """Add a processed tool command to the batch.""" + if not self.is_active: + raise RuntimeError("BatchManager not activated - call activate() first") + + # Store command with all necessary info for later submission + # Add player_index to front since Lua functions expect it as first parameter + parameters = [player_index] + list(args) + self.scheduled_commands.append( + {"tick": tick, "command": tool_name, "parameters": parameters, "raw": False} + ) + return {"batched": True, "tick": tick, "tool": tool_name} + + def reset_sequence(self, grace_period=None): + """Reset the batch sequence to start fresh timing with optional grace period.""" + if grace_period is None: + grace_period = self.grace_period + + self.instance.begin_transaction() + self.instance.add_command("/sc global.actions.reset_sequence()", raw=True) + result = self.instance.execute_transaction() + + # Store the grace period for this sequence + self.grace_period = grace_period + return result + + def emergency_cleanup(self): + """Emergency cleanup method to clear all server-side memory and queued actions. + + This method is designed to be called during KeyboardInterrupt or error conditions + to ensure that no scheduled commands continue running on the server. + """ + cleanup_results = {} + rcon_client = self.rcon_connection or self.instance.rcon_client + + try: + # Clear local state first + self.scheduled_commands.clear() + old_batch_id = self.current_batch_id + self.current_batch_id = None + + print( + f" 📝 Manager {self.manager_id}: Clearing local state (batch_id: {old_batch_id})" + ) + + # Reset the sequence to clear all scheduled commands for this manager's sequence + print( + f" 🔄 Manager {self.manager_id}: Resetting sequence {self.sequence_id}" + ) + reset_cmd = ( + f'/sc global.actions.register_sequence_start("{self.sequence_id}", 0)' + ) + rcon_client.send_command(reset_cmd) + + # Use the new clear_sequence_commands function to precisely clear this sequence's commands + clear_sequence_cmd = ( + f'/sc global.actions.clear_sequence_commands("{self.sequence_id}")' + ) + rcon_client.send_command(clear_sequence_cmd) + + # Force reset sequence to clear any remaining scheduled commands + reset_sequence_cmd = "/sc global.actions.reset_sequence()" + rcon_client.send_command(reset_sequence_cmd) + + cleanup_results["sequence_reset"] = "success" + print(f" ✅ Manager {self.manager_id}: Sequence reset completed") + + except Exception as e: + cleanup_results["sequence_reset"] = f"error: {str(e)}" + print(f" ❌ Manager {self.manager_id}: Failed to reset sequence: {e}") + + try: + # Clear batch results for this specific batch if it exists + if old_batch_id: + clear_specific_cmd = ( + f'/sc global.actions.clear_batch_results("{old_batch_id}")' + ) + rcon_client.send_command(clear_specific_cmd) + print( + f" 🗑️ Manager {self.manager_id}: Cleared results for batch {old_batch_id}" + ) + + # Clear all batch results to free server memory + clear_all_cmd = "/sc global.actions.clear_batch_results()" + rcon_client.send_command(clear_all_cmd) + + cleanup_results["batch_results_cleared"] = "success" + print(f" 🧹 Manager {self.manager_id}: All batch results cleared") + + except Exception as e: + cleanup_results["batch_results_cleared"] = f"error: {str(e)}" + print( + f" ❌ Manager {self.manager_id}: Failed to clear batch results: {e}" + ) + + try: + # Deactivate batch mode to ensure tools don't add more commands + self.deactivate() + cleanup_results["deactivated"] = "success" + print(f" 🛑 Manager {self.manager_id}: Batch mode deactivated") + + except Exception as e: + cleanup_results["deactivated"] = f"error: {str(e)}" + print(f" ❌ Manager {self.manager_id}: Failed to deactivate: {e}") + + return cleanup_results + + def submit_batch(self): + """Submit all batched commands to the server and return batch info.""" + if not self.scheduled_commands: + return {} + + # Check connection health before submission + if not self.connection_healthy: + print( + f"❌ Manager {self.manager_id}: Refusing to submit - connection unhealthy: {self.last_error}" + ) + return {"error": "Connection unhealthy", "last_error": self.last_error} + + # Use the dedicated RCON connection for this batch manager + rcon_client = self.rcon_connection or self.instance.rcon_client + + # Store the total number of commands for completion tracking + total_commands = len(self.scheduled_commands) + + # Store metadata before clearing for debugging purposes + self.last_batch_metadata = { + "commands": self.scheduled_commands.copy(), + "total_count": total_commands, + } + + # CRITICAL: Sort commands by tick before submitting to ensure Lua indices + # match the expected sorted order from Python side + self.scheduled_commands.sort(key=lambda x: x["tick"]) + + # Submit the batch directly to the server + import json + + batch_json = json.dumps(self.scheduled_commands, cls=PrototypeJSONEncoder) + + # Validate the JSON as a Lua string + escaped_batch_json = batch_json.replace('"', '\\"') + + try: + # Register sequence start with grace period using our unique sequence ID + sequence_cmd = f'/sc global.actions.register_sequence_start("{self.sequence_id}", {self.grace_period})' + rcon_client.send_command(sequence_cmd) + + # Submit the batch - let Lua generate the batch_id + lua_command = ( + f'/sc global.actions.submit_scheduled_batch("{escaped_batch_json}")' + ) + is_valid, validation_msg = validate_lua_syntax(lua_command) + if not is_valid: + print( + f"❌ Manager {self.manager_id}: Lua validation failed: {validation_msg}" + ) + + submit_result_raw = rcon_client.send_command(lua_command) + submit_result = _lua2python("submit_batch", submit_result_raw) + + except Exception as e: + print(f"❌ Manager {self.manager_id}: Exception during submission: {e}") + return {"error": str(e)} + + # Extract batch_id from submission result + batch_id_from_result = None + if isinstance(submit_result, tuple) and len(submit_result) >= 1: + response_data = submit_result[0] + if isinstance(response_data, dict) and "batch_id" in response_data: + batch_id_from_result = response_data["batch_id"] + else: + print( + f"❌ Manager {self.manager_id}: No batch_id found in response data" + ) + else: + print( + f"❌ Manager {self.manager_id}: Unexpected result format: {type(submit_result)}" + ) + + if not batch_id_from_result: + print(f"❌ Manager {self.manager_id}: Failed to get batch_id from server") + return {"error": "Failed to get batch_id"} + + self.current_batch_id = batch_id_from_result + self.expected_command_count = total_commands + + # Clear the batch + self.scheduled_commands.clear() + + return {"batch_id": batch_id_from_result, "submitted": True} + + def wait_for_batch_completion(self, timeout_seconds=30): + """Wait for the batch to complete and return results.""" + if not self.current_batch_id: + raise RuntimeError("No active batch to wait for") + + import time + + rcon_client = self.rcon_connection or self.instance.rcon_client + + start_time = time.time() + + while time.time() - start_time < timeout_seconds: + # Check batch status using dedicated RCON connection + result_raw = rcon_client.send_command( + f'/sc global.actions.get_batch_results("{self.current_batch_id}")' + ) + result = _lua2python("get_batch_results", result_raw) + + # Extract batch results + batch_results = None + if isinstance(result, tuple) and len(result) >= 1: + response_data = result[0] + if isinstance(response_data, dict) and "completed" in response_data: + batch_results = response_data + + if batch_results and batch_results.get("completed"): + # Clean up server-side results + rcon_client.send_command( + f'/sc global.actions.clear_batch_results("{self.current_batch_id}")' + ) + + self.current_batch_id = None + return batch_results["results"] + + time.sleep(0.1) # Wait before checking again + + raise TimeoutError(f"Batch did not complete within {timeout_seconds} seconds") + + def submit_batch_and_wait(self, timeout_seconds=30): + """Submit batch and wait for completion, returning all results.""" + self.submit_batch() + return self.wait_for_batch_completion(timeout_seconds) + + def submit_batch_and_stream(self, timeout_seconds=30, poll_interval=0.1): + """Submit batch and yield results as they become available. + + Args: + timeout_seconds: Maximum time to wait for batch completion + poll_interval: How often to check for new results (in seconds) + + Yields: + dict: Individual command results as they complete + + Each yielded result contains: + - command_index: The index of the command in the batch + - command: The command name + - success: Whether the command succeeded + - result: The command result or error message + - tick: The game tick when the command was executed + """ + if not self.scheduled_commands: + return + + # Submit the batch + submit_result = self.submit_batch() + + if "error" in submit_result: + print( + f"❌ Manager {self.manager_id}: Batch submission failed: {submit_result}" + ) + return + + if not self.current_batch_id: + print(f"❌ Manager {self.manager_id}: No current batch ID after submission") + return + + import time + + rcon_client = self.rcon_connection or self.instance.rcon_client + + start_time = time.time() + yielded_results = set() # Track which results we've already yielded + poll_count = 0 + + try: + while time.time() - start_time < timeout_seconds: + poll_count += 1 + # Check batch status using dedicated RCON connection + status_cmd = ( + f'/sc global.actions.get_batch_results("{self.current_batch_id}")' + ) + + try: + result_raw = rcon_client.send_command(status_cmd) + result = _lua2python("get_batch_results", result_raw) + + except Exception as e: + print( + f"❌ Manager {self.manager_id}: Poll {poll_count} failed: {e}" + ) + time.sleep(poll_interval) + continue + + # Process responses + if isinstance(result, tuple) and len(result) >= 1: + response_data = result[0] + + # Handle proper Lua table response format + if isinstance(response_data, dict): + batch_id_from_response = response_data.get("batch_id") + if batch_id_from_response != self.current_batch_id: + time.sleep(poll_interval) + continue + + # Process results + batch_results_data = response_data.get("results", {}) + is_complete = response_data.get("completed", False) + sequence_start_tick = response_data.get( + "sequence_start_tick", 0 + ) + + # Yield new results - handle both numeric and string keys + new_results_count = 0 + num_commands = 0 + for cmd_index_key, cmd_result in batch_results_data.items(): + try: + # Convert key to integer (Lua arrays start at 1, Python at 0) + if isinstance(cmd_index_key, str): + command_index = ( + int(cmd_index_key) - 1 + ) # Convert 1-based to 0-based + elif isinstance(cmd_index_key, int): + command_index = ( + cmd_index_key - 1 + ) # Convert 1-based to 0-based + else: + print( + f"❌ Manager {self.manager_id}: Unexpected command index type: {type(cmd_index_key)}" + ) + continue + + num_commands += 1 + if command_index not in yielded_results: + yielded_results.add(command_index) + new_results_count += 1 + + # Calculate sequence-relative tick: game_tick - sequence_start_tick + game_tick = cmd_result.get("game_tick", 0) + sequence_relative_tick = ( + game_tick - sequence_start_tick + if sequence_start_tick > 0 + else game_tick + ) + + result_dict = { + "command_index": command_index, + "command": cmd_result.get("command", "unknown"), + "success": cmd_result.get("success", False), + "result": cmd_result.get("result"), + "tick": sequence_relative_tick, + "planned_tick": cmd_result.get( + "planned_tick", "?" + ), # Pass through planned_tick from Lua + } + + yield result_dict + + except (ValueError, TypeError) as e: + print( + f"❌ Manager {self.manager_id}: Failed to parse command index {cmd_index_key}: {e}" + ) + print( + f"POLL {poll_count} - num_commands: {num_commands}, new_results: {new_results_count}" + ) + # Check if batch is complete + if is_complete: + # Clean up using dedicated RCON connection + try: + rcon_client.send_command( + f'/sc global.actions.clear_batch_results("{self.current_batch_id}")' + ) + print(f"POLL {poll_count} - CLEANED UP") + except Exception as e: + print( + f"❌ Manager {self.manager_id}: Cleanup failed: {e}" + ) + + self.current_batch_id = None + return + else: + print(f"POLL {poll_count} - NO RESULTS, result: {result}") + else: + print( + f"POLL {poll_count} - NO RESULTS, type(result): {type(result)}, len(result): {len(result)}" + ) + + time.sleep(poll_interval) + + print( + f"❌ Manager {self.manager_id}: Timeout after {timeout_seconds}s ({poll_count} polls)" + ) + raise TimeoutError( + f"Batch did not complete within {timeout_seconds} seconds" + ) + + except Exception as e: + print(f"❌ Manager {self.manager_id}: Exception in streaming: {e}") + # Clean up on error + if self.current_batch_id: + try: + rcon_client.send_command( + f'/sc global.actions.clear_batch_results("{self.current_batch_id}")' + ) + except: + pass # Best effort cleanup + self.current_batch_id = None + raise + + class FactorioInstance: namespace_class = FactorioNamespace _cleanup_registered = False # Only register cleanup once per process @@ -101,18 +605,51 @@ def __init__( all_technologies_researched=True, peaceful=True, num_agents=1, + regenerate="map", + batch_grace_period=5, # New parameter + max_concurrent_batches=1, # New parameter for concurrency **kwargs, ): self.id = str(uuid.uuid4())[:8] self.num_agents = num_agents self.persistent_vars = {} self.tcp_port = tcp_port + self.max_concurrent_batches = max_concurrent_batches + + # Create multiple RCON connections for concurrent batch processing + self.rcon_connections = [] + self.batch_managers = [] + + # Primary connection for main operations self.rcon_client, self.address = self.connect_to_server(address, tcp_port) + self.rcon_connections.append(self.rcon_client) + + # Create additional connections for concurrent batch processing + for i in range(max_concurrent_batches - 1): + try: + additional_rcon = self.connect_to_server(address, tcp_port)[0] + self.rcon_connections.append(additional_rcon) + except Exception as e: + print(f"❌ Could not create additional RCON connection {i + 2}: {e}") + break + + # Test all connections + for i, conn in enumerate(self.rcon_connections): + try: + conn.send_command("/sc rcon.print('test_connection_" + str(i) + "')") + except Exception as e: + print(f"❌ RCON connection {i} test failed: {e}") + + print( + f"Created {len(self.rcon_connections)} RCON connection(s) for batch processing" + ) + self.all_technologies_researched = all_technologies_researched self.fast = fast self._speed = 1 self._ticks_elapsed = 0 self._is_initialised = False + self.regenerate = regenerate self.peaceful = peaceful self.namespaces = [self.namespace_class(self, i) for i in range(num_agents)] @@ -130,6 +667,29 @@ def __init__( # Load the python controllers that correspond to the Lua scripts self.setup_tools(self.lua_script_manager) + # Initialize multiple batch managers - one per RCON connection + for i, rcon_conn in enumerate(self.rcon_connections): + batch_manager = BatchManager( + self, batch_grace_period, rcon_connection=rcon_conn, manager_id=i + ) + self.batch_managers.append(batch_manager) + + # Primary batch manager for backward compatibility + self.batch_manager = ( + self.batch_managers[0] + if self.batch_managers + else BatchManager(self, batch_grace_period) + ) + + # Print health summary + print("Batch Manager Health Summary:") + for manager in self.batch_managers: + status = manager.get_health_status() + health_status = ( + "✅ Healthy" if status["connection_healthy"] else "❌ Unhealthy" + ) + print(f" Manager {status['manager_id']}: {health_status}") + if inventory is None: inventory = {} self.initial_inventory = inventory @@ -625,17 +1185,18 @@ def _reset_elapsed_ticks(self): def _reset(self, inventories: List[Dict[str, Any]]): self.begin_transaction() + if self.regenerate == "resources": + regenerate_func = "global.actions.regenerate_resources(1)" + elif self.regenerate == "map": + regenerate_func = "global.actions.regenerate_map(1)" + else: + raise ValueError(f"Invalid regenerate value: {self.regenerate}") + self.add_command( - "/sc global.alerts = {}; game.reset_game_state(); global.actions.reset_production_stats(); global.actions.regenerate_resources(1)", + f"/sc global.alerts = {{}}; game.reset_game_state(); global.actions.reset_production_stats(); {regenerate_func}", raw=True, ) # self.add_command('/sc script.on_nth_tick(nil)', raw=True) # Remove all dangling event handlers - for i in range(self.num_agents): - player_index = i + 1 - self.add_command( - f"/sc global.actions.regenerate_resources({player_index})", raw=True - ) - # self.add_command('clear_inventory', player_index) self.execute_transaction() @@ -725,6 +1286,7 @@ def initialise(self, fast=True): "serialize", "production_score", "initialise_inventory", + "scheduled_batch", # Add the new batch scheduler ] if self.peaceful: init_scripts.append("enemies") @@ -1021,11 +1583,30 @@ def execute_pre_tool_hooks(self, tool_name, tool_instance, *args, **kwargs): except Exception as e: print(f"Error in pre-tool hook for {tool_name}: {e}") + def get_available_batch_manager(self): + """Get an available batch manager for concurrent processing.""" + # Find a batch manager that's not currently active + for manager in self.batch_managers: + if not manager.is_active or not manager.current_batch_id: + return manager + + # If all are busy, return the first one (will queue) + return self.batch_managers[0] if self.batch_managers else self.batch_manager + def cleanup(self): - # Close the RCON connection + # Close all RCON connections if hasattr(self, "rcon_client") and self.rcon_client: self.rcon_client.close() + # Close additional RCON connections + if hasattr(self, "rcon_connections"): + for rcon_conn in self.rcon_connections[1:]: # Skip the primary one + try: + if rcon_conn: + rcon_conn.close() + except Exception as e: + print(f"Error closing RCON connection: {e}") + self.post_tool_hooks = {} self.pre_tool_hooks = {} @@ -1040,3 +1621,17 @@ def cleanup(self): thread.join(timeout=5) # Wait up to 5 seconds for each thread except Exception as e: print(f"Error joining thread {thread.name}: {e}") + + +def validate_lua_syntax(lua_code: str) -> Tuple[bool, str]: + """Validate Lua syntax using luaparser if available""" + if not LUAPARSER_AVAILABLE: + return True, "luaparser not available - skipping validation" + + try: + lua_code = lua_code.replace("/sc", "").replace("/c", "").strip() + with contextlib.redirect_stdout(None): + lua_ast.parse(lua_code) + return True, "Valid Lua syntax" + except Exception as e: + return False, f"Lua syntax error: {str(e)}" diff --git a/fle/env/lua_manager.py b/fle/env/lua_manager.py index 4cc0f4a72..55ec59ced 100644 --- a/fle/env/lua_manager.py +++ b/fle/env/lua_manager.py @@ -28,7 +28,6 @@ def __init__(self, rcon_client: RCONClient, cache_scripts: bool = False): if cache_scripts: self.init_action_checksums() self.game_checksums = self._get_game_checksums(rcon_client) - self.tool_scripts = self.get_tools_to_load() self.lib_scripts = self.get_libs_to_load() @@ -102,8 +101,7 @@ def load_init_into_game(self, name): if name in self.game_checksums and self.game_checksums[name] == checksum: return self.update_game_checksum(self.rcon_client, name, checksum) - - self.rcon_client.send_command("/sc " + script) + self.rcon_client.send_command("/c " + script) def calculate_checksum(self, content: str) -> str: return hashlib.md5(content.encode()).hexdigest() diff --git a/fle/env/mods/initialise.lua b/fle/env/mods/initialise.lua index 9d401830d..30e6074fc 100644 --- a/fle/env/mods/initialise.lua +++ b/fle/env/mods/initialise.lua @@ -17,7 +17,7 @@ end -- Note: The debug_rendering.lua library will be loaded separately by the LuaScriptManager ---local player = game.players[arg1] +local player = global.agent_characters[1] player.surface.always_day=true --game.players[1].character_collision_mask = "not-colliding-with-itself" player.force.character_build_distance_bonus = 100 @@ -452,7 +452,6 @@ function create_arrow_with_direction(player, direction, position) target_position = end_position, duration = 6000, force = 'neutral', - player = player } -- Calculate and create the two side beams for the arrowhead @@ -475,7 +474,6 @@ function create_arrow_with_direction(player, direction, position) target_position = arrow_left, duration = 100000, force = 'neutral', - player = player } player.surface.create_entity{ @@ -485,7 +483,6 @@ function create_arrow_with_direction(player, direction, position) target_position = arrow_right, duration = 100000, force = 'neutral', - player = player } end diff --git a/fle/env/mods/scheduled_batch.lua b/fle/env/mods/scheduled_batch.lua new file mode 100644 index 000000000..bd320071a --- /dev/null +++ b/fle/env/mods/scheduled_batch.lua @@ -0,0 +1,418 @@ +-- Batch processing system for scheduling commands at specific game ticks +-- This script allows submitting multiple commands that will be executed at future ticks + +-- Initialize storage for scheduled commands and results +if not global.scheduled_commands then + global.scheduled_commands = {} +end + +if not global.batch_results then + global.batch_results = {} +end + +if not global.last_error then + global.last_error = nil +end + +-- Initialize sequence tracking +if not global.sequence_start_tick then + global.sequence_start_tick = nil +end + +if not global.sequence_grace_period then + global.sequence_grace_period = 0 +end + +if not global.current_sequence_id then + global.current_sequence_id = nil +end + +if not global.batch_metadata then + global.batch_metadata = {} +end + +-- Helper function to generate unique batch IDs +local function generate_batch_id() + local batch_id = "batch_" .. game.tick .. "_" .. math.random(1000, 9999) + return batch_id +end + +-- Helper function to extract clean error message from Lua stack trace +local function extract_clean_error(error_string) + if type(error_string) ~= "string" then + return tostring(error_string) + end + + -- Find all matches of the pattern "number:" in the string + local last_match_end = 0 + for match_start, match_end in string.gmatch(error_string, "()%d+:()") do + last_match_end = match_end + end + + -- If we found at least one match, return everything after the final match + if last_match_end > 0 then + local clean_message = string.sub(error_string, last_match_end + 1) + -- Trim leading whitespace + clean_message = string.match(clean_message, "^%s*(.-)%s*$") or clean_message + return clean_message + end + + -- If no pattern found, return the original string + return error_string +end + +-- Function to register a sequence start with optional grace period +global.actions.register_sequence_start = function(sequence_id, grace_period) + sequence_id = sequence_id or "default" + grace_period = grace_period or 0 + + local result + + -- Check if this is a new sequence ID (different from current) + if global.current_sequence_id and global.current_sequence_id ~= sequence_id then + -- Reset sequence for new sequence ID + global.sequence_start_tick = nil + global.sequence_grace_period = 0 + global.current_sequence_id = nil + global.batch_metadata = {} + + -- Clear any pending scheduled commands + local cleared_commands = 0 + for tick, commands in pairs(global.scheduled_commands) do + cleared_commands = cleared_commands + #commands + end + global.scheduled_commands = {} + end + + if not global.sequence_start_tick then + -- First batch submission or after reset - establish the sequence start with grace period + global.sequence_start_tick = game.tick + grace_period + global.sequence_grace_period = grace_period + global.current_sequence_id = sequence_id -- Set the current sequence ID + result = { + sequence_id = sequence_id, + sequence_start_tick = global.sequence_start_tick, + grace_period = grace_period, + current_game_tick = game.tick, + message = "New sequence started with grace period", + is_new_sequence = true + } + game.print("[BATCH] Started new sequence '" .. sequence_id .. "' at game_tick " .. global.sequence_start_tick .. " (grace period: " .. grace_period .. ")") + else + -- Subsequent batch - use existing sequence start (same sequence ID) + result = { + sequence_id = sequence_id, + sequence_start_tick = global.sequence_start_tick, + grace_period = global.sequence_grace_period, + current_game_tick = game.tick, + message = "Using existing sequence start", + is_new_sequence = false + } + game.print("[BATCH] Using existing sequence '" .. sequence_id .. "' start at game_tick " .. global.sequence_start_tick) + end + + rcon.print(dump(result)) + return result +end + +-- Function to reset/clear the sequence (for new batch sequences) +global.actions.reset_sequence = function() + global.sequence_start_tick = nil + global.sequence_grace_period = 0 + global.current_sequence_id = nil -- Reset current sequence ID + global.batch_metadata = {} + + -- Clear any pending scheduled commands + local cleared_commands = 0 + for tick, commands in pairs(global.scheduled_commands) do + cleared_commands = cleared_commands + #commands + end + global.scheduled_commands = {} + + local result = { + message = "Sequence reset", + cleared_scheduled_commands = cleared_commands, + current_game_tick = game.tick + } + + rcon.print(dump(result)) + return result +end + +-- Function to clear scheduled commands for a specific sequence ID +global.actions.clear_sequence_commands = function(target_sequence_id) + if not target_sequence_id then + local error_result = {error = "target_sequence_id parameter is required"} + rcon.print(dump(error_result)) + return + end + + local cleared_commands = 0 + local total_commands_before = 0 + + -- Count total commands before cleanup + for tick, commands in pairs(global.scheduled_commands) do + total_commands_before = total_commands_before + #commands + end + + -- Clear scheduled commands that belong to the target sequence + for tick, commands in pairs(global.scheduled_commands) do + local remaining_commands = {} + for _, cmd in ipairs(commands) do + -- Check if this command belongs to a batch from the target sequence + -- We need to check batch metadata to see which sequence each batch belongs to + local batch_metadata = global.batch_metadata[cmd.batch_id] + if batch_metadata and batch_metadata.sequence_id == target_sequence_id then + cleared_commands = cleared_commands + 1 + else + table.insert(remaining_commands, cmd) + end + end + + if #remaining_commands == 0 then + global.scheduled_commands[tick] = nil + else + global.scheduled_commands[tick] = remaining_commands + end + end + + -- Also clear batch metadata for the target sequence + local cleared_batches = 0 + for batch_id, metadata in pairs(global.batch_metadata) do + if metadata.sequence_id == target_sequence_id then + global.batch_metadata[batch_id] = nil + cleared_batches = cleared_batches + 1 + end + end + + -- If this was the current sequence, reset it + if global.current_sequence_id == target_sequence_id then + global.sequence_start_tick = nil + global.sequence_grace_period = 0 + global.current_sequence_id = nil + game.print("[BATCH] Reset current sequence as it matched target sequence " .. target_sequence_id) + end + + local result = { + message = "Sequence-specific commands cleared", + target_sequence_id = target_sequence_id, + cleared_commands = cleared_commands, + cleared_batches = cleared_batches, + total_commands_before = total_commands_before, + total_commands_after = total_commands_before - cleared_commands, + current_game_tick = game.tick + } + + rcon.print(dump(result)) + return result +end + +-- Submit a batch of scheduled commands +global.actions.submit_scheduled_batch = function(batch_json) + local success, result = pcall(function() + if not batch_json then + error("batch_json parameter is nil") + end + + if not global.sequence_start_tick then + error("No sequence start registered. Call register_sequence_start first.") + end + + local batch_data = game.json_to_table(batch_json) + if not batch_data then + error("Failed to parse batch_json") + end + + local submitted_count = 0 + local immediate_count = 0 + local sequence_start = global.sequence_start_tick + local current_tick = game.tick + local batch_id = generate_batch_id() -- Generate batch_id internally + + -- Store batch metadata + global.batch_metadata[batch_id] = { + sequence_start_tick = sequence_start, + submitted_tick = current_tick, + total_commands = #batch_data, + sequence_id = global.current_sequence_id -- Store the sequence ID for cleanup purposes + } + + -- Initialize results storage for this batch + global.batch_results[batch_id] = { + commands = {}, + completed = false, + total_commands = #batch_data + } + + -- Schedule each command for execution at its absolute tick relative to sequence start + for i, cmd_data in ipairs(batch_data) do + local absolute_tick_in_sequence = cmd_data.tick -- This is your [0, 10, 20] etc. + local actual_execution_tick = sequence_start + absolute_tick_in_sequence + + -- Check if this tick has already passed + if actual_execution_tick <= current_tick then + -- Execute immediately - schedule for next tick + actual_execution_tick = current_tick + 1 + immediate_count = immediate_count + 1 + game.print("[BATCH] Command " .. i .. " (" .. cmd_data.command .. ") scheduled for immediate execution (tick " .. actual_execution_tick .. ") - original tick " .. (sequence_start + absolute_tick_in_sequence) .. " has passed") + end + + -- Initialize the execution tick if it doesn't exist + if not global.scheduled_commands[actual_execution_tick] then + global.scheduled_commands[actual_execution_tick] = {} + end + + -- Add the command to be executed at this tick + table.insert(global.scheduled_commands[actual_execution_tick], { + batch_id = batch_id, + command_index = i, + command = cmd_data.command, + parameters = cmd_data.parameters, + raw = cmd_data.raw, + sequence_tick = absolute_tick_in_sequence, -- For debugging + execution_tick = actual_execution_tick, -- For debugging + was_immediate = actual_execution_tick <= current_tick + 1, -- For debugging + planned_tick = cmd_data.tick -- Store the original absolute tick from Python + }) + + submitted_count = submitted_count + 1 + end + + return { + submitted = submitted_count, + immediate_commands = immediate_count, + message = "Batch scheduled successfully", + batch_id = batch_id, + sequence_start_tick = sequence_start, + current_game_tick = current_tick + } + end) + + if success then + rcon.print(dump(result)) + else + local error_msg = type(result) == "string" and result or tostring(result) + global.last_error = error_msg + local error_result = { + error = error_msg, + submitted = 0, + message = "Batch submission failed: " .. error_msg + } + rcon.print(dump(error_result)) + end +end + +-- Process scheduled commands on each tick +script.on_event(defines.events.on_tick, function(event) + local current_game_tick = event.tick + + if global.scheduled_commands[current_game_tick] then + for _, scheduled_cmd in ipairs(global.scheduled_commands[current_game_tick]) do + -- Print the sequence-level tick when executing command + + -- Execute the command using the instance's command system + local success, result = pcall(function() + if scheduled_cmd.raw then + return global.actions.raw_command(table.unpack(scheduled_cmd.parameters)) + else + return global.actions[scheduled_cmd.command](table.unpack(scheduled_cmd.parameters)) + end + end) + + -- Store the result with explicit error handling + local final_result + if success then + final_result = result + else + -- Error case: preserve the original error message and add debugging + if type(result) == "string" then + local clean_error = extract_clean_error(result) + final_result = clean_error + game.print("[BATCH ERROR] Command " .. scheduled_cmd.command .. " at tick " .. scheduled_cmd.sequence_tick .. " failed with string error: " .. clean_error) + else + final_result = tostring(result) + game.print("[BATCH ERROR] Command " .. scheduled_cmd.command .. " at tick " .. scheduled_cmd.sequence_tick .. " failed with non-string error (type: " .. type(result) .. "): " .. tostring(result)) + end + end + + -- Store the result + if not global.batch_results[scheduled_cmd.batch_id] then + global.batch_results[scheduled_cmd.batch_id] = {commands = {}, completed = false} + end + + global.batch_results[scheduled_cmd.batch_id].commands[scheduled_cmd.command_index] = { + command = scheduled_cmd.command, + success = success, + result = final_result, + game_tick = event.tick, + was_immediate = scheduled_cmd.was_immediate or false, + planned_tick = scheduled_cmd.planned_tick -- Include planned_tick in results + } + -- game.print("[BATCH] Executed command at " .. scheduled_cmd.sequence_tick .. " - " .. scheduled_cmd.command) + end + + -- Clean up this tick's commands + global.scheduled_commands[current_game_tick] = nil + end +end) + +-- Get results for a specific batch +global.actions.get_batch_results = function(batch_id) + if not batch_id then + local error_result = {error = "batch_id is required"} + rcon.print(dump(error_result)) + return + end + + local batch_results = global.batch_results[batch_id] + if not batch_results then + local error_result = {error = "Batch not found", batch_id = batch_id} + rcon.print(dump(error_result)) + return + end + + local current_tick = game.tick + local tick_cutoff = current_tick - 600 + + -- Filter results to only include those from the last 1800 ticks + local filtered_commands = {} + local completed_count = 0 + for index, command_result in pairs(batch_results.commands) do + if command_result.game_tick and command_result.game_tick >= tick_cutoff then + filtered_commands[index] = command_result + completed_count = completed_count + 1 + end + end + + local is_complete = completed_count >= (batch_results.total_commands or 0) + batch_results.completed = is_complete + + local result = { + batch_id = batch_id or "unknown", + results = filtered_commands, + completed = is_complete or false, + total_commands = batch_results.total_commands or 0, + completed_commands = completed_count or 0, + current_game_tick = game.tick or 0, + sequence_start_tick = global.sequence_start_tick, + sequence_grace_period = global.sequence_grace_period, + tick_cutoff = tick_cutoff - global.sequence_start_tick + } + + rcon.print(dump(result)) +end + +-- Clear results for a specific batch or all batches +global.actions.clear_batch_results = function(batch_id) + local result + if batch_id then + global.batch_results[batch_id] = nil + global.batch_metadata[batch_id] = nil + result = {message = "Cleared results for batch: " .. batch_id} + else + global.batch_results = {} + global.batch_metadata = {} + result = {message = "Cleared all batch results and metadata"} + end + rcon.print(dump(result)) +end diff --git a/fle/env/tools/admin/regenerate_map/client.py b/fle/env/tools/admin/regenerate_map/client.py new file mode 100644 index 000000000..0e9b1d1b5 --- /dev/null +++ b/fle/env/tools/admin/regenerate_map/client.py @@ -0,0 +1,14 @@ +from fle.env.tools import Tool + + +class RegenerateMap(Tool): + def __init__(self, *args): + super().__init__(*args) + + def __call__(self) -> bool: + """ + Regenerates the map with autoplace and crash-site. + """ + self.execute(self.player_index) + + return True diff --git a/fle/env/tools/admin/regenerate_map/server.lua b/fle/env/tools/admin/regenerate_map/server.lua new file mode 100644 index 000000000..5812c6ebd --- /dev/null +++ b/fle/env/tools/admin/regenerate_map/server.lua @@ -0,0 +1,42 @@ +function global.actions.regenerate_map(player_index) + ------------------------------------------- 0. Quick handles + local player = global.agent_characters[player_index] or error("bad player") + local surface = player.surface -- normally "nauvis" + local force = player.force -- normally "player" + + ------------------------------------------- 1. Reset the random seed for deterministic drops + local map_gen_seed = surface.map_gen_settings.seed + + ------------------------------------------- 2. Create a new seeded random generator for the map + -- This ensures that any RNG-dependent operations (like rock mining) are deterministic + if not global.map_random_generator then + global.map_random_generator = game.create_random_generator(map_gen_seed) + end + global.map_random_generator.re_seed(map_gen_seed) + + local radius = 10 -- in *chunks* + local center_cx = math.floor(player.position.x / 32) + local center_cy = math.floor(player.position.y / 32) + + local chunks = {} + for dx = -radius, radius do + for dy = -radius, radius do + table.insert(chunks, {x = center_cx + dx, y = center_cy + dy}) + end + end + + ------------------------------------------- 3. Rerun the map generator + local tree_names = {} + for name, proto in pairs(game.entity_prototypes) do + if proto.type == "tree" then + table.insert(tree_names, name) + end + end + + -- Add rocks to the list + table.insert(tree_names, 'rock-huge') + table.insert(tree_names, 'rock-big') + + -- Regenerate them + surface.regenerate_entity(tree_names, chunks) +end diff --git a/fle/env/tools/agent/craft_item/client.py b/fle/env/tools/agent/craft_item/client.py index e9932cc40..9c6d8c99d 100644 --- a/fle/env/tools/agent/craft_item/client.py +++ b/fle/env/tools/agent/craft_item/client.py @@ -10,11 +10,12 @@ def __init__(self, connection, game_state): super().__init__(connection, game_state) self.inspect_inventory = InspectInventory(connection, game_state) - def __call__(self, entity: Prototype, quantity: int = 1) -> int: + def __call__(self, entity: Prototype, quantity: int = 1, tick: int = None) -> int: """ Craft an item from a Prototype if the ingredients exist in your inventory. :param entity: Entity to craft :param quantity: Quantity to craft + :param tick: Game tick to execute this command at (for batch mode) :return: Number of items crafted """ @@ -27,7 +28,15 @@ def __call__(self, entity: Prototype, quantity: int = 1) -> int: if not self.game_state.instance.fast: count_in_inventory = self.inspect_inventory()[entity] - success, elapsed = self.execute(self.player_index, name, quantity) + success, elapsed = self.execute_or_batch( + tick, self.player_index, name, quantity + ) + + # Check if we're in batch mode - if so, return early without processing response + if isinstance(success, dict) and success.get("batched"): + # In batch mode, return expected quantity as placeholder + return quantity + if success != {} and isinstance(success, str): if success is None: raise Exception( diff --git a/fle/env/tools/agent/extract_item/client.py b/fle/env/tools/agent/extract_item/client.py index 311719f61..0ac851e36 100644 --- a/fle/env/tools/agent/extract_item/client.py +++ b/fle/env/tools/agent/extract_item/client.py @@ -1,6 +1,6 @@ from typing import Union -from fle.env.entities import Position, Entity +from fle.env.entities import Position, Entity, PlaceholderEntity from fle.env.game_types import Prototype from fle.env.tools import Tool @@ -10,13 +10,18 @@ def __init__(self, connection, game_state): super().__init__(connection, game_state) def __call__( - self, entity: Prototype, source: Union[Position, Entity], quantity=5 + self, + entity: Prototype, + source: Union[Position, Entity, PlaceholderEntity], + quantity=5, + tick: int = None, ) -> int: """ Extract an item from an entity's inventory at position (x, y) if it exists on the world. :param entity: Entity prototype to extract, e.g Prototype.IronPlate - :param source: Entity or position to extract from + :param source: Entity, PlaceholderEntity, or position to extract from :param quantity: Quantity to extract + :param tick: Game tick to execute this command at (for batch mode) :example extract_item(Prototype.IronPlate, stone_furnace.position, 5) :example extract_item(Prototype.CopperWire, stone_furnace, 5) :return The number of items extracted. @@ -25,18 +30,24 @@ def __call__( if isinstance(source, Position): x, y = self.get_position(source) - elif isinstance(source, Entity): + elif isinstance(source, (Entity, PlaceholderEntity)): x, y = self.get_position(source.position) source_name = source.name name, _ = entity.value - response, elapsed = self.execute( - self.player_index, name, quantity, x, y, source_name + response, elapsed = self.execute_or_batch( + tick, self.player_index, name, quantity, x, y, source_name ) + + # Check if we're in batch mode - if so, return early without processing response + if isinstance(response, dict) and response.get("batched"): + # In batch mode, return expected quantity as placeholder + return quantity + if isinstance(response, str): msg = self.get_error_message(response) - if source_name: + if source_name is not None: raise Exception( f"Could not extract {name} from {source_name} at ({x}, {y}): {msg}" ) diff --git a/fle/env/tools/agent/harvest_resource/client.py b/fle/env/tools/agent/harvest_resource/client.py index 959d50124..e60de2573 100644 --- a/fle/env/tools/agent/harvest_resource/client.py +++ b/fle/env/tools/agent/harvest_resource/client.py @@ -15,11 +15,15 @@ def __init__(self, connection, game_state): self.nearest = Nearest(connection, game_state) self.get_entity = GetEntity(connection, game_state) - def __call__(self, position: Position, quantity=1, radius=10) -> int: + def __call__( + self, position: Position, quantity=1, radius=10, tick: int = None + ) -> int: """ Harvest a resource at position (x, y) if it exists on the world. :param position: Position to harvest resource :param quantity: Quantity to harvest + :param radius: Radius to search for resources + :param tick: Game tick to execute this command at (for batch mode) :example harvest_resource(nearest(Resource.Coal), 5) :example harvest_resource(nearest(Resource.Stone), 5) :return: The quantity of the resource harvested @@ -38,7 +42,14 @@ def __call__(self, position: Position, quantity=1, radius=10) -> int: # Now we attempt to harvest. # In fast mode, this will always be successful (because we don't check if the resource is reachable) - response, elapsed = self.execute(self.player_index, x, y, quantity, radius) + response, elapsed = self.execute_or_batch( + tick, self.player_index, x, y, quantity, radius + ) + + # Check if we're in batch mode - if so, return early without processing response + if isinstance(response, dict) and response.get("batched"): + # In batch mode, return expected quantity as placeholder + return quantity if response != {} and response == 0 or isinstance(response, str): msg = response.split(":")[-1].strip() diff --git a/fle/env/tools/agent/harvest_resource/server.lua b/fle/env/tools/agent/harvest_resource/server.lua index 9cea09536..ca7ea1acf 100644 --- a/fle/env/tools/agent/harvest_resource/server.lua +++ b/fle/env/tools/agent/harvest_resource/server.lua @@ -337,53 +337,75 @@ local function harvest_resource_slow(player, player_index, surface, position, co return expected_yield end -function harvest(entities, count, from_position, player) - if count == 0 then return 0 end - local yield = 0 - - -- Check inventory space first using the first valid entity as reference - local reference_entity = nil - for _, entity in ipairs(entities) do - if entity.valid and entity.minable then - reference_entity = entity - break - end - end - - if reference_entity then - check_inventory_space(player, reference_entity, count) - end - - - entities = sort_entities_by_distance(entities, from_position) +-- helper: true when this is one of the 3 vanilla rocks on Nauvis +local function is_rock(e) + return e.type == "simple-entity" and e.name:find("rock") + -- catches rock-big, sand‑rock‑big, rock‑huge +end - ::start:: - local has_mined = false - for _, entity in ipairs(entities) do - if entity.valid and entity.minable then +function harvest(entities, count, from_position, player) + if count == 0 then return 0 end + local yield = 0 - -- Calculate mining ticks before mining the entity - if global.fast then - global.elapsed_ticks = global.elapsed_ticks + calculate_mining_ticks(entity) - end + -- unchanged: inventory‑space check and distance sort … + ---------------------------------------------------------------- + -- … your original pre‑amble here … + ---------------------------------------------------------------- - local products = entity.prototype.mineable_properties.products - for _, product in pairs(products) do - local amount = product.amount or 1 + ::start:: + local has_mined = false + for _, entity in ipairs(entities) do + if entity.valid and entity.minable then + if global.fast then + global.elapsed_ticks = global.elapsed_ticks + calculate_mining_ticks(entity) + end + + if is_rock(entity) then + if entity.name == "rock-huge" then + -- remove the rock so it no longer blocks the terrain / is re‑harvested + entity.mine{ignore_minable = false, raise_destroyed = true} + + -- give the player the hard‑coded drops + local s = player.insert{name = "stone", count = 50} + update_production_stats(player.force, "stone", s) + yield = yield + s + local c = player.insert{name = "coal", count = 50} + update_production_stats(player.force, "coal", c) + yield = yield + c + else + local buffer = game.create_inventory(2) + entity.mine{inventory = buffer, ignore_minable = false, raise_destroyed = true} + + for name, amount in pairs(buffer.get_contents()) do + player.insert{name = name, count = amount} + update_production_stats(player.force, name, amount) yield = yield + amount - entity.mine({ignore_minable=false, raise_destroyed=true}) - player.insert({name=product.name, count=amount}) - update_production_stats(player.force, product.name, amount) - has_mined = true - if yield >= count then break end end + buffer.destroy() + + has_mined = true if yield >= count then break end end + else + ------------------------------------------------------------ + -- non‑rock path – identical to your original implementation + ------------------------------------------------------------ + local products = entity.prototype.mineable_properties.products + for _, product in pairs(products) do + local amount = product.amount or 1 + yield = yield + amount + entity.mine{ignore_minable = false, raise_destroyed = true} + player.insert{name = product.name, count = amount} + update_production_stats(player.force, product.name, amount) + has_mined = true + if yield >= count then break end + end + end + if yield >= count then break end end - if has_mined == true and yield < count then - goto start - end - return yield + end + if has_mined and yield < count then goto start end + return yield end function harvest_trees(entities, count, from_position, player) @@ -429,29 +451,73 @@ local function harvest_simple_entities(entities, count, from_position, player) if count == 0 then return 0 end local yield = 0 entities = sort_entities_by_distance(entities, from_position) - + for _, entity in ipairs(entities) do - if entity.valid and entity.minable then - -- Calculate mining ticks before mining the entity - if global.fast then - global.elapsed_ticks = global.elapsed_ticks + calculate_mining_ticks(entity) - end + if entity.valid and entity.minable then + -- mining‑time bookkeeping (unchanged) + if global.fast then + global.elapsed_ticks = global.elapsed_ticks + calculate_mining_ticks(entity) + end + + ---------------------------------------------------------------- + -- ROCK‑AWARE BRANCH + ---------------------------------------------------------------- + if is_rock(entity) then + -- 1. make a throw‑away inventory that will catch the drops + if entity.name == "rock-huge" then + -- remove the rock so it no longer blocks the terrain / is re‑harvested + entity.mine{ignore_minable = false, raise_destroyed = true} + + -- give the player the hard‑coded drops + local s = player.insert{name = "stone", count = 50} + update_production_stats(player.force, "stone", s) + yield = yield + s + + local c = player.insert{name = "coal", count = 50} + update_production_stats(player.force, "coal", c) + yield = yield + c + else + local buffer = game.create_inventory(4) -- 4 slots = safe for mods + local ok = entity.mine{ + inventory = buffer, + force = true, + ignore_minable = false, + raise_destroyed = true + } + if ok then + for name, amount in pairs(buffer.get_contents()) do + player.insert{name = name, count = amount} + yield = yield + amount + update_production_stats(player.force, name, amount) + end + else + game.print("Mining failed for " .. entity.name) + end + buffer.destroy() + end + + ---------------------------------------------------------------- + -- ORIGINAL PATH (tree stumps etc.) + ---------------------------------------------------------------- + else + game.print("Harvesting " .. entity.name .. " type " .. entity.type) local products = entity.prototype.mineable_properties.products for _, product in pairs(products) do local amount = product.amount or 1 yield = yield + amount - entity.mine({ignore_minable=false, raise_destroyed=true}) - player.insert({name=product.name, count=amount}) + entity.mine{ignore_minable = false, raise_destroyed = true} + player.insert{name = product.name, count = amount} update_production_stats(player.force, product.name, amount) - if yield >= count then break end end - if yield >= count then break end end + + -- if yield >= count then break end + end end return yield -end + end global.actions.harvest_resource = function(player_index, x, y, count, radius) @@ -464,15 +530,15 @@ global.actions.harvest_resource = function(player_index, x, y, count, radius) local position = {x=x, y=y} local distance = math.sqrt((position.x - player_position.x)^2 + (position.y - player_position.y)^2) - if distance > player.resource_reach_distance then - error("Nothing within reach to harvest") + if distance > player.resource_reach_distance * 2 then + error("Nothing within reach to harvest (wrong distance)") end local surface = player.surface local target_type, target_name = find_entity_type_at_position(surface, position) if not target_type then - error("Nothing within reach to harvest") + error("Nothing within reach to harvest (wrong type)") end --if not global.fast then @@ -492,7 +558,7 @@ global.actions.harvest_resource = function(player_index, x, y, count, radius) if total_yield < count then -- Try trees first local tree_entities = surface.find_entities_filtered{ - position = position, + position = {position.x, position.y}, radius = radius, type = "tree" } @@ -502,8 +568,8 @@ global.actions.harvest_resource = function(player_index, x, y, count, radius) if total_yield < count then -- Then try simple entities (rocks, stumps, etc.) local simple_entities = surface.find_entities_filtered{ - position = position, - radius = radius, + position = {position.x, position.y}, + radius = 5, type = "simple-entity" } total_yield = total_yield + harvest_simple_entities(simple_entities, count - total_yield, position, player) @@ -513,14 +579,14 @@ global.actions.harvest_resource = function(player_index, x, y, count, radius) -- Finally try standard resources local mineable_entities = surface.find_entities_filtered{ position = position, - radius = radius, + radius = 2, type = "resource" } total_yield = total_yield + harvest(mineable_entities, count - total_yield, position, player) end if total_yield == 0 then - error("Nothing within reach to harvest") + error("Nothing within reach to harvest (no resources found)") else -- game.print("Harvested resources yielding " .. total_yield .. " items") return total_yield diff --git a/fle/env/tools/agent/insert_item/client.py b/fle/env/tools/agent/insert_item/client.py index bd69b4b15..d3cfedb0f 100644 --- a/fle/env/tools/agent/insert_item/client.py +++ b/fle/env/tools/agent/insert_item/client.py @@ -1,10 +1,18 @@ from time import sleep -from typing import Union +from typing import Optional, Union -from fle.env.entities import Entity, EntityGroup, Position, BeltGroup, PipeGroup +from fle.env.entities import ( + Entity, + EntityGroup, + Position, + BeltGroup, + PipeGroup, + PlaceholderEntity, +) from fle.env.game_types import Prototype from fle.env.tools.agent.get_entities.client import GetEntities from fle.env.tools import Tool +from fle.env.instance import NONE class InsertItem(Tool): @@ -13,22 +21,35 @@ def __init__(self, connection, game_state): super().__init__(connection, game_state) def __call__( - self, entity: Prototype, target: Union[Entity, EntityGroup], quantity=5 + self, + entity: Prototype, + target: Union[Entity, EntityGroup, PlaceholderEntity], + quantity=5, + tick: Optional[int] = None, ) -> Entity: """ Insert an item into a target entity's inventory :param entity: Type to insert from inventory - :param target: Entity to insert into + :param target: Entity to insert into (can be Entity, EntityGroup, or PlaceholderEntity) :param quantity: Quantity to insert + :param tick: Game tick to execute this command at (for batch mode) :return: The target entity inserted into """ assert quantity is not None, "Quantity cannot be None" assert isinstance(entity, Prototype), "The first argument must be a Prototype" - assert isinstance(target, Entity) or isinstance(target, EntityGroup), ( - "The second argument must be an Entity or EntityGroup, you passed in a {0}".format( - type(target) - ) - ) + + # Check if PlaceholderEntity is being used with unsupported entity types + if isinstance(target, PlaceholderEntity): + # Check for belt group names that aren't supported with PlaceholderEntity + belt_names = [ + "transport-belt", + "fast-transport-belt", + "express-transport-belt", + ] + if target.name in belt_names: + raise Exception( + f"PlaceholderEntity cannot be used with belt entities ('{target.name}'). BeltGroup functionality requires actual entities." + ) if isinstance(target, Position): x, y = target.x, target.y @@ -53,8 +74,25 @@ def __call__( if not x or not y: x, y = target.belts[0].position.x, target.belts[0].position.y + # Handle tick parameter - either batch mode or error + if tick is not None: + response, elapsed = self.execute_or_batch( + tick, self.player_index, name, quantity, x, y, NONE + ) + + # Check if we're in batch mode + if isinstance(response, dict) and response.get("batched"): + # Return the original target as placeholder since we can't process the actual result yet + return target + else: + # tick was provided but we're not in batch mode - this is invalid + raise Exception( + "tick parameter provided but batch mode is not active" + ) + + # Original iterative approach for non-batch mode (tick is None) while items_inserted < quantity: - response, elapsed = self.execute(self.player_index, name, 1, x, y, None) + response, elapsed = self.execute(self.player_index, name, 1, x, y, NONE) if isinstance(response, str): if ( @@ -86,10 +124,26 @@ def __call__( return target - response, elapsed = self.execute( - self.player_index, name, quantity, x, y, target_name - ) + # Handle tick parameter for regular entities + if tick is not None: + response, elapsed = self.execute_or_batch( + tick, self.player_index, name, quantity, x, y, target_name + ) + + # Check if we're in batch mode + if isinstance(response, dict) and response.get("batched"): + # Return the original target as placeholder since we can't process the actual result yet + return target + else: + # tick was provided but we're not in batch mode - this is invalid + raise Exception("tick parameter provided but batch mode is not active") + else: + # Regular execution without tick (non-batch mode) + response, elapsed = self.execute( + self.player_index, name, quantity, x, y, target_name + ) + # Process response for non-batch execution if isinstance(response, str): raise Exception(f"Could not insert: {response.split(':')[-1].strip()}") @@ -97,8 +151,30 @@ def __call__( if isinstance(cleaned_response, dict): if not isinstance(target, (BeltGroup, PipeGroup)): _type = type(target) - prototype = Prototype._value2member_map_[(target.name, type(target))] - target = _type(prototype=prototype, **cleaned_response) + + # Handle PlaceholderEntity specially + if isinstance(target, PlaceholderEntity): + # Find the prototype by name + matching_prototype = None + for prototype in Prototype: + if prototype.value[0] == target.name: + matching_prototype = prototype + break + + if matching_prototype is None: + raise Exception( + f"No matching Prototype found for PlaceholderEntity with name '{target.name}'" + ) + entity_class = matching_prototype.value[1] + target = entity_class( + prototype=matching_prototype, **cleaned_response + ) + else: + # Original logic for non-PlaceholderEntity + prototype = Prototype._value2member_map_[ + (target.name, type(target)) + ] + target = _type(prototype=prototype, **cleaned_response) elif isinstance(target, BeltGroup): group = self.get_entities( { diff --git a/fle/env/tools/agent/inspect_inventory/client.py b/fle/env/tools/agent/inspect_inventory/client.py index 059f38a86..466a3971f 100644 --- a/fle/env/tools/agent/inspect_inventory/client.py +++ b/fle/env/tools/agent/inspect_inventory/client.py @@ -2,6 +2,7 @@ from fle.env import Inventory, Entity, Position from fle.env.tools import Tool +from fle.env.instance import NONE class InspectInventory(Tool): @@ -9,39 +10,64 @@ def __init__(self, *args): super().__init__(*args) def __call__( - self, entity=None, all_players: bool = False + self, entity=None, all_players: bool = False, tick: int = None ) -> Union[Inventory, List[Inventory]]: """ Inspects the inventory of the given entity. If no entity is given, inspect your own inventory. If all_players is True, returns a list of inventories for all players. :param entity: Entity to inspect :param all_players: If True, returns inventories for all players + :param tick: Game tick to execute this command at (for batch mode) :return: Inventory of the given entity or list of inventories for all players """ if all_players: - response, execution_time = self.execute( - self.player_index, True, 0, 0, "", True + response, execution_time = self.execute_or_batch( + tick, self.player_index, True, NONE, NONE, NONE, True ) + + # Check if we're in batch mode - if so, return early without processing response + if isinstance(response, dict) and response.get("batched"): + # In batch mode, return empty inventory list as placeholder + return [] + + # Non-batch mode - process the response normally if not isinstance(response, list): raise Exception("Could not get inventories for all players", response) return [Inventory(**inv) for inv in response] - if entity: - if isinstance(entity, Entity): + if entity is None: + response, execution_time = self.execute_or_batch( + tick, self.player_index, True, NONE, NONE, NONE, False + ) + + # Check if we're in batch mode - if so, return early without processing response + if isinstance(response, dict) and response.get("batched"): + # In batch mode, return empty inventory as placeholder + return Inventory(items=[]) + + else: + if isinstance(entity, int) or isinstance(entity, str): + response, execution_time = self.execute_or_batch( + tick, self.player_index, False, NONE, entity, NONE, False + ) + elif isinstance(entity, Entity): x, y = self.get_position(entity.position) + response, execution_time = self.execute_or_batch( + tick, self.player_index, False, x, y, entity.name, False + ) elif isinstance(entity, Position): x, y = entity.x, entity.y - else: - raise ValueError( - f"The first argument must be an Entity or Position object, you passed in a {type(entity)} object." + response, execution_time = self.execute_or_batch( + tick, self.player_index, False, x, y, "", False ) - else: - x, y = 0, 0 + else: + raise ValueError(f"Invalid entity type: {type(entity)}") - response, execution_time = self.execute( - self.player_index, entity is None, x, y, entity.name if entity else "" - ) + # Check if we're in batch mode - if so, return early without processing response + if isinstance(response, dict) and response.get("batched"): + # In batch mode, return empty inventory as placeholder + return Inventory(items=[]) if not isinstance(response, dict): if entity: diff --git a/fle/env/tools/agent/move_to/client.py b/fle/env/tools/agent/move_to/client.py index 257726793..f604f033e 100644 --- a/fle/env/tools/agent/move_to/client.py +++ b/fle/env/tools/agent/move_to/client.py @@ -18,11 +18,18 @@ def __init__(self, connection: LuaScriptManager, game_state): self.get_path = GetPath(connection, game_state) def __call__( - self, position: Position, laying: Prototype = None, leading: Prototype = None + self, + position: Position, + laying: Prototype = None, + leading: Prototype = None, + tick: int = None, ) -> Position: """ Move to a position. :param position: Position to move to. + :param laying: Entity to lay while moving + :param leading: Entity to lead while moving + :param tick: Game tick to execute this command at (for batch mode) :return: Your final position """ @@ -46,19 +53,24 @@ def __call__( try: if laying is not None: entity_name = laying.value[0] - response, execution_time = self.execute( - self.player_index, path_handle, entity_name, 1 + response, execution_time = self.execute_or_batch( + tick, self.player_index, path_handle, entity_name, 1 ) elif leading: entity_name = leading.value[0] - response, execution_time = self.execute( - self.player_index, path_handle, entity_name, 0 + response, execution_time = self.execute_or_batch( + tick, self.player_index, path_handle, entity_name, 0 ) else: - response, execution_time = self.execute( - self.player_index, path_handle, NONE, NONE + response, execution_time = self.execute_or_batch( + tick, self.player_index, path_handle, NONE, NONE ) + # Check if we're in batch mode - if so, return early without processing response + if isinstance(response, dict) and response.get("batched"): + # In batch mode, return the target position since we can't get actual position yet + return position + if isinstance(response, int) and response == 0: raise Exception("Could not move.") @@ -80,8 +92,7 @@ def __call__( remaining_steps = self.connection.rcon_client.send_command( f"/silent-command rcon.print(global.actions.get_walking_queue_length({self.player_index}))" ) - self.game_state.player_location = Position(x=position.x, y=position.y) - return Position(x=response["x"], y=response["y"]) # , execution_time + return self.game_state.player_location except Exception as e: raise Exception(f"Cannot move. {e}") diff --git a/fle/env/tools/agent/pickup_entity/client.py b/fle/env/tools/agent/pickup_entity/client.py index dffe076df..7c4babcce 100644 --- a/fle/env/tools/agent/pickup_entity/client.py +++ b/fle/env/tools/agent/pickup_entity/client.py @@ -13,11 +13,13 @@ def __call__( self, entity: Union[ent.Entity, Prototype, ent.EntityGroup], position: Optional[ent.Position] = None, + tick: int = None, ) -> bool: """ Pick up an entity if it exists on the world at a given position. :param entity: Entity prototype to pickup, e.g Prototype.IronPlate :param position: Position to pickup entity + :param tick: Game tick to execute this command at (for batch mode) :return: True if the entity was picked up successfully, False otherwise. """ if not isinstance(entity, (Prototype, ent.Entity, ent.EntityGroup)): @@ -36,14 +38,14 @@ def __call__( if isinstance(entity, ent.BeltGroup): belts = entity.belts for belt in belts: - resp = self.__call__(belt) + resp = self.__call__(belt, tick=tick) if not resp: return False return True elif isinstance(entity, ent.PipeGroup): pipes = entity.pipes for pipe in pipes: - resp = self.__call__(pipe) + resp = self.__call__(pipe, tick=tick) if not resp: return False return True @@ -51,31 +53,48 @@ def __call__( elif isinstance(entity, ent.ElectricityGroup): poles = entity.poles for pole in poles: - resp = self.__call__(pole) + resp = self.__call__(pole, tick=tick) if not resp: return False return True if position: x, y = position.x, position.y - response, elapsed = self.execute(self.player_index, x, y, name) + response, elapsed = self.execute_or_batch( + tick, self.player_index, x, y, name + ) elif isinstance(entity, ent.UndergroundBelt): x, y = entity.position.x, entity.position.y - response, elapsed = self.execute(self.player_index, x, y, name) + response, elapsed = self.execute_or_batch( + tick, self.player_index, x, y, name + ) + + # Check if we're in batch mode - if so, return early + if isinstance(response, dict) and response.get("batched"): + return True + if response != 1 and response != {}: raise Exception(f"Could not pickup: {self.get_error_message(response)}") x, y = entity.output_position.x, entity.output_position.y - response, elapsed = self.execute(self.player_index, x, y, name) + response, elapsed = self.execute_or_batch( + tick, self.player_index, x, y, name + ) if response != 1 and response != {}: raise Exception(f"Could not pickup: {self.get_error_message(response)}") elif isinstance(entity, ent.Entity): x, y = entity.position.x, entity.position.y - response, elapsed = self.execute(self.player_index, x, y, name) + response, elapsed = self.execute_or_batch( + tick, self.player_index, x, y, name + ) else: raise ValueError("The second argument must be a Position object") + # Check if we're in batch mode - if so, return early + if isinstance(response, dict) and response.get("batched"): + return True + if response != 1 and response != {}: raise Exception(f"Could not pickup: {self.get_error_message(response)}") return True diff --git a/fle/env/tools/agent/pickup_entity/server.lua b/fle/env/tools/agent/pickup_entity/server.lua index f5225e88d..4f79d46ca 100644 --- a/fle/env/tools/agent/pickup_entity/server.lua +++ b/fle/env/tools/agent/pickup_entity/server.lua @@ -144,7 +144,7 @@ global.actions.pickup_entity = function(player_index, x, y, entity) if not success then if #player_entities == 0 and #ground_items == 0 then - error("Couldn't find "..entity.." at position ("..x..", "..y..") to pick up.") + error("Could not find "..entity.." at position ("..x..", "..y..") to pick up.") else error("Could not pick up "..entity) end diff --git a/fle/env/tools/agent/place_entity/client.py b/fle/env/tools/agent/place_entity/client.py index cc539dc9e..af37685d3 100644 --- a/fle/env/tools/agent/place_entity/client.py +++ b/fle/env/tools/agent/place_entity/client.py @@ -1,6 +1,6 @@ from time import sleep -from fle.env.entities import Position, Entity +from fle.env.entities import Position, Entity, PlaceholderEntity from fle.env import DirectionInternal, Direction from fle.env.game_types import Prototype from fle.env.tools.agent.get_entity.client import GetEntity @@ -22,6 +22,7 @@ def __call__( direction: Direction = Direction.UP, position: Position = Position(x=0, y=0), exact: bool = True, + tick: int = None, # relative=False ) -> Entity: """ @@ -30,6 +31,7 @@ def __call__( :param direction: Cardinal direction to place :param position: Position to place entity :param exact: If True, place entity at exact position, else place entity at nearest possible position + :param tick: Game tick to execute this command at (for batch mode) :return: Entity object """ @@ -57,10 +59,28 @@ def __call__( factorio_direction = DirectionInternal.to_factorio_direction(direction) try: - # If we are in `fast` mode, this is synchronous - response, elapsed = self.execute( - self.player_index, name, factorio_direction, x, y, exact + response, elapsed = self.execute_or_batch( + tick, + self.player_index, + name, # entity name + factorio_direction, # direction + x, # x position + y, # y position + exact, # exact placement flag ) + + # Check if we're in batch mode - if so, return early without processing response + if isinstance(response, dict) and response.get("batched"): + # In batch mode, return a PlaceholderEntity since we can't get actual result yet + # Use entity.value[0] to get the string name from the prototype tuple + entity_name = ( + entity.value[0] + if isinstance(entity.value, tuple) + else str(entity.value) + ) + + return PlaceholderEntity(entity_name, position, direction) + except Exception as e: try: msg = self.get_error_message(str(e)) diff --git a/fle/env/tools/agent/place_entity/server.lua b/fle/env/tools/agent/place_entity/server.lua index a954efca5..a6bbffa77 100644 --- a/fle/env/tools/agent/place_entity/server.lua +++ b/fle/env/tools/agent/place_entity/server.lua @@ -19,7 +19,7 @@ local function find_offshore_pump_position(player, center_pos) {dx = -1, dy = 0, dir = defines.direction.east} } - for radius = 1, max_radius do + for radius = 0, max_radius do for y = -radius, radius do for x = -radius, radius do if math.abs(x) == radius or math.abs(y) == radius then diff --git a/fle/env/tools/agent/rotate_entity/client.py b/fle/env/tools/agent/rotate_entity/client.py index 326c98dc2..2d767f54e 100644 --- a/fle/env/tools/agent/rotate_entity/client.py +++ b/fle/env/tools/agent/rotate_entity/client.py @@ -13,12 +13,16 @@ def __init__(self, connection, game_state): super().__init__(connection, game_state) def __call__( - self, entity: Entity, direction: DirectionInternal = DirectionInternal.UP + self, + entity: Entity, + direction: DirectionInternal = DirectionInternal.UP, + tick: int = None, ) -> Entity: """ Rotate an entity to a specified direction :param entity: Entity to rotate :param direction: Direction to rotate + :param tick: Game tick to execute this command at (for batch mode) :example rotate_entity(iron_chest, Direction.UP) :return: Returns the rotated entity """ @@ -39,10 +43,17 @@ def __call__( factorio_direction = DirectionInternal.to_factorio_direction(direction) - response, elapsed = self.execute( - self.player_index, x, y, factorio_direction, entity.name + response, elapsed = self.execute_or_batch( + tick, self.player_index, x, y, factorio_direction, entity.name ) + # Check if we're in batch mode - if so, return early without processing response + if isinstance(response, dict) and response.get("batched"): + # In batch mode, return a modified copy of the input entity with new direction + rotated_entity = entity.model_copy() + rotated_entity.direction = direction + return rotated_entity + if not response: raise Exception(f"Could not rotate: {response}") diff --git a/fle/env/tools/agent/set_entity_recipe/client.py b/fle/env/tools/agent/set_entity_recipe/client.py index d1a1799b0..ad6f0fe52 100644 --- a/fle/env/tools/agent/set_entity_recipe/client.py +++ b/fle/env/tools/agent/set_entity_recipe/client.py @@ -10,12 +10,13 @@ def __init__(self, connection, game_state): super().__init__(connection, game_state) def __call__( - self, entity: Entity, prototype: Union[Prototype, RecipeName] + self, entity: Entity, prototype: Union[Prototype, RecipeName], tick: int = None ) -> Entity: """ Sets the recipe of an given entity. :param entity: Entity to set recipe :param prototype: The prototype to create, or a recipe name for more complex processes + :param tick: Game tick to execute this command at (for batch mode) :return: Entity that had its recipe set """ @@ -28,7 +29,22 @@ def __call__( else: raise ValueError(f"Invalid entity type: {prototype}") - response, elapsed = self.execute(self.player_index, name, x, y) + response, elapsed = self.execute_or_batch(tick, self.player_index, name, x, y) + + # Check if we're in batch mode - if so, return early without processing response + if isinstance(response, dict) and response.get("batched"): + # In batch mode, return a modified copy of the input entity with recipe set + modified_entity = ( + entity.model_copy() if hasattr(entity, "model_copy") else entity + ) + # Handle filter inserters differently + if "filter" in entity.name: + modified_entity.filter = name # Store the filter item + else: + modified_entity.recipe = ( + name # Store the recipe for assembling machines + ) + return modified_entity if not isinstance(response, dict): raise Exception( diff --git a/fle/env/tools/agent/set_research/client.py b/fle/env/tools/agent/set_research/client.py index f6d9dc3bf..50ba6fc98 100644 --- a/fle/env/tools/agent/set_research/client.py +++ b/fle/env/tools/agent/set_research/client.py @@ -14,10 +14,11 @@ class SetResearch(Tool): def __init__(self, connection, game_state): super().__init__(connection, game_state) - def __call__(self, technology: Technology) -> List[Ingredient]: + def __call__(self, technology: Technology, tick: int = None) -> List[Ingredient]: """ Set the current research technology for the player's force. :param technology: Technology to research + :param tick: Game tick to execute this command at (for batch mode) :return: Required ingredients to research the technology. """ if hasattr(technology, "value"): @@ -25,7 +26,12 @@ def __call__(self, technology: Technology) -> List[Ingredient]: else: name = technology - success, elapsed = self.execute(self.player_index, name) + success, elapsed = self.execute_or_batch(tick, self.player_index, name) + + # Check if we're in batch mode - if so, return early without processing response + if isinstance(success, dict) and success.get("batched"): + # In batch mode, return empty ingredients list as placeholder + return [] if success != {} and isinstance(success, str): if success is None: diff --git a/fle/env/tools/controller.py b/fle/env/tools/controller.py index 5f5f6cd75..f5d7e017f 100644 --- a/fle/env/tools/controller.py +++ b/fle/env/tools/controller.py @@ -29,6 +29,29 @@ def __init__( game_state.agent_index + 1 ) # +1 because Factorio is 1-indexed + # Batch mode support + self.batch_manager = None # Will be set when batch mode is enabled + + def set_batch_manager(self, batch_manager): + """Set the batch manager for this tool.""" + self.batch_manager = batch_manager + + def execute_or_batch(self, tick: int = None, *args): + """Execute immediately or add to batch based on batch manager.""" + if self.batch_manager: + # If no tick provided, get current tick from game state + if tick is None: + tick = self.game_state.instance.get_elapsed_ticks() + # Add to batch instead of executing + # Skip the first arg (self.player_index) since add_tool_command will add it + batch_info = self.batch_manager.add_tool_command( + tick, self.name, self.player_index, *args[1:] + ) + return batch_info, 0 # Return tuple for consistency + else: + # Execute immediately (current behavior) + return self.execute(*args) + def clean_response(self, response): def is_lua_list(d): """Check if dictionary represents a Lua-style list (keys are consecutive numbers from 1)"""