diff --git a/fle/agents/data/blueprints_to_policies/blueprint_analyzer_with_connect.py b/fle/agents/data/blueprints_to_policies/blueprint_analyzer_with_connect.py index 2ddc94bea..4045b8db1 100644 --- a/fle/agents/data/blueprints_to_policies/blueprint_analyzer_with_connect.py +++ b/fle/agents/data/blueprints_to_policies/blueprint_analyzer_with_connect.py @@ -4,10 +4,12 @@ from typing import List, Dict from typing import Union +from fle.agents.data.blueprints_to_policies.models.blueprint_entity import BlueprintEntity from fle.env import EntityGroup from fle.env import FactorioInstance from fle.env.game_types import prototype_by_name -from data.blueprints_to_policies.models.blueprint_entity import BlueprintEntity +from fle.logger import info, error +#from data.blueprints_to_policies.models.blueprint_entity import BlueprintEntity class BlueprintAnalyzerWithConnect: @@ -351,11 +353,11 @@ def _name_to_prototype_string(self, name: str) -> str: #if os.path.exists(execution_dir + filename.replace(".json", ".py")): # continue with open(execution_dir + filename, "r") as f: - print(filename) + info(filename) blueprint_json = f.read() blueprint = json.loads(blueprint_json) if len(blueprint['entities']) > 200: - print("Skipping large blueprint") + info("Skipping large blueprint") continue analyzer = BlueprintAnalyzerWithConnect(blueprint) code = analyzer.generate_program() @@ -374,17 +376,17 @@ def _name_to_prototype_string(self, name: str) -> str: if "error" in result: raise Exception(result["error"]) except Exception as e: - print(e) - print("Error in blueprint") + error(str(e)) + error("Error in blueprint") continue - print(code) + info(code) game_entities = instance.get_entities() try: analyzer.verify_placement(game_entities) except AssertionError as e: - print(e) - print("Error in blueprint") + error(str(e)) + error("Error in blueprint") continue # Write the code to a python file of the same name with open(execution_dir + filename.split('.json')[0] + " with connect"+ ".py", "w") as f1: diff --git a/fle/agents/data/blueprints_to_policies/parse_blueprints.py b/fle/agents/data/blueprints_to_policies/parse_blueprints.py index 1f03e170b..8b82d4349 100644 --- a/fle/agents/data/blueprints_to_policies/parse_blueprints.py +++ b/fle/agents/data/blueprints_to_policies/parse_blueprints.py @@ -6,7 +6,7 @@ import matplotlib.pyplot as plt from dataclasses import dataclass, field -from game_types import Prototype +from fle.env import Prototype @dataclass(frozen=True) # Make the class immutable and hashable diff --git a/fle/agents/data/run_trace.py b/fle/agents/data/run_trace.py index b2852aec7..d539a9444 100644 --- a/fle/agents/data/run_trace.py +++ b/fle/agents/data/run_trace.py @@ -4,6 +4,7 @@ from fle.commons.db_client import DBClient from fle.env import FactorioInstance +from fle.logger import info, error, debug, warning # Load environment variables load_dotenv() @@ -99,20 +100,20 @@ def game_instance(): traces = cur.fetchall() # Evaluate each code snippet in the trace - print(f"Found {len(traces)} entries in the program trace") + info(f"Found {len(traces)} entries in the program trace") for i, trace in enumerate(traces): code = trace[1] # code is the second column - print(f"\nEvaluating trace {i + 1}/{len(traces)}") - print(f"Program ID: {trace[3]}") # id is the fourth column + info(f"Evaluating trace {i + 1}/{len(traces)}") + info(f"Program ID: {trace[3]}") # id is the fourth column try: # Evaluate the code reward, _, result = instance.eval(code, timeout=30) - print(f"Evaluation result:") - print(f"Reward: {reward}") - print(f"Result: {result}") + info(f"Evaluation result:") + info(f"Reward: {reward}") + info(f"Result: {result}") except Exception as e: - print(f"Error evaluating code: {e}") + error(f"Error evaluating code: {e}") continue diff --git a/fle/cluster/docker/build.sh b/fle/cluster/docker/build.sh deleted file mode 100644 index 42b420723..000000000 --- a/fle/cluster/docker/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -docker build -t factorio . - -cd mods -tar -czf headless-player_0.1.0.tgz headless-player_0.1.0 -tar -czf stdlib_1.4.6.tgz stdlib_1.4.6 - -mv stdlib_1.4.6.tgz ~/Library/Application\ Support/Factorio/mods/stdlib_1.4.6.tgz -mv headless-player_0.1.0.tgz ~/Library/Application\ Support/Factorio/mods/headless-player_0.1.0.tgz - -rm -rf ~/Library/Application\ Support/Factorio/mods/stdlib_1.4.6 -rm -rf ~/Library/Application\ Support/Factorio/mods/headless-player_0.1.0 - -cp -rf stdlib_1.4.6 ~/Library/Application\ Support/Factorio/mods/stdlib_1.4.6 -cp -rf headless-player_0.1.0 ~/Library/Application\ Support/Factorio/mods/headless-player_0.1.0 - -cd .. - -cp config/mod-list.json ~/Library/Application\ Support/Factorio/mods/mod-list.json \ No newline at end of file diff --git a/fle/cluster/docker/build_alternative_installation_location.sh b/fle/cluster/docker/build_alternative_installation_location.sh deleted file mode 100644 index 8e1f4c557..000000000 --- a/fle/cluster/docker/build_alternative_installation_location.sh +++ /dev/null @@ -1,32 +0,0 @@ -# This has has different mod location -docker build -t factorio . --platform linux/amd64 - -cd mods -tar -czf headless-player_0.1.0.tgz headless-player_0.1.0 -tar -czf stdlib_1.4.6.tgz stdlib_1.4.6 - -mv stdlib_1.4.6.tgz ~/Applications/Factorio.app/Contents/Resources/mods/stdlib_1.4.6.tgz -mv headless-player_0.1.0.tgz ~/Applications/Factorio.app/Contents/Resources/mods/headless-player_0.1.0.tgz - -# unzip -# cd ~/Applications/Factorio.app/Contents/Resources/mods - - -# remove old files -rm -rf ~/Applications/Factorio.app/Contents/Resources/mods/stdlib_1.4.6 -rm -rf ~/Applications/Factorio.app/Contents/Resources/mods/headless-player_0.1.0 - -# untar files -#tar -xzf ~/Applications/Factorio.app/Contents/Resources/mods/stdlib_1.4.6.tgz -#tar -xzf ~/Applications/Factorio.app/Contents/Resources/mods/headless-player_0.1.0.tgz - -# move files to mods directory -cp -rf stdlib_1.4.6 ~/Applications/Factorio.app/Contents/Resources/mods/stdlib_1.4.6 -cp -rf headless-player_0.1.0 ~/Applications/Factorio.app/Contents/Resources/mods/headless-player_0.1.0 - -#rm -rf ~/Applications/Factorio.app/Contents/Resources/mods/stdlib_1.4.6.tgz -#rm -rf ~/Applications/Factorio.app/Contents/Resources/mods/headless-player_0.1.0.tgz - -cd .. - -cp config/mod-list.json ~/Applications/Factorio.app/Contents/Resources/mods/mod-list.json \ No newline at end of file diff --git a/fle/cluster/docker/config/docker-entrypoint.sh b/fle/cluster/docker/config/docker-entrypoint.sh deleted file mode 100644 index 70e95f934..000000000 --- a/fle/cluster/docker/config/docker-entrypoint.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash -set -eoux pipefail - -FACTORIO_VOL=/factorio -LOAD_LATEST_SAVE="${LOAD_LATEST_SAVE:-true}" -GENERATE_NEW_SAVE="${GENERATE_NEW_SAVE:-false}" -SAVE_NAME="${SAVE_NAME:-""}" - -mkdir -p "$FACTORIO_VOL" -mkdir -p "$SAVES" -mkdir -p "$CONFIG" -mkdir -p "$MODS" -mkdir -p "$SCENARIOS" -mkdir -p "$SCRIPTOUTPUT" - -if [[ ! -f $CONFIG/rconpw ]]; then - # Generate a new RCON password if none exists - echo "default" > "$CONFIG/rconpw" -fi - -if [[ ! -f $CONFIG/server-settings.json ]]; then - # Copy default settings if server-settings.json doesn't exist - cp /opt/factorio/data/server-settings.example.json "$CONFIG/server-settings.json" -fi - -if [[ ! -f $CONFIG/map-gen-settings.json ]]; then - cp /opt/factorio/data/map-gen-settings.example.json "$CONFIG/map-gen-settings.json" -fi - -if [[ ! -f $CONFIG/map-settings.json ]]; then - cp /opt/factorio/data/map-settings.example.json "$CONFIG/map-settings.json" -fi - -NRTMPSAVES=$( find -L "$SAVES" -iname \*.tmp.zip -mindepth 1 | wc -l ) -if [[ $NRTMPSAVES -gt 0 ]]; then - # Delete incomplete saves (such as after a forced exit) - rm -f "$SAVES"/*.tmp.zip -fi - -if [[ ${UPDATE_MODS_ON_START:-} == "true" ]]; then - ./docker-update-mods.sh -fi - -if [[ $(id -u) = 0 ]]; then - # Update the User and Group ID based on the PUID/PGID variables - usermod -o -u "$PUID" factorio - groupmod -o -g "$PGID" factorio - # Take ownership of factorio data if running as root - chown -R factorio:factorio "$FACTORIO_VOL" - # Drop to the factorio user - SU_EXEC="su-exec factorio" -else - SU_EXEC="" -fi - -sed -i '/write-data=/c\write-data=\/factorio/' /opt/factorio/config/config.ini - -NRSAVES=$(find -L "$SAVES" -iname \*.zip -mindepth 1 | wc -l) -if [[ $GENERATE_NEW_SAVE != true && $NRSAVES == 0 ]]; then - GENERATE_NEW_SAVE=true - SAVE_NAME=_autosave1 -fi - -if [[ $GENERATE_NEW_SAVE == true ]]; then - if [[ -z "$SAVE_NAME" ]]; then - echo "If \$GENERATE_NEW_SAVE is true, you must specify \$SAVE_NAME" - exit 1 - fi - if [[ -f "$SAVES/$SAVE_NAME.zip" ]]; then - echo "Map $SAVES/$SAVE_NAME.zip already exists, skipping map generation" - else - $SU_EXEC /opt/factorio/bin/x64/factorio \ - --create "$SAVES/$SAVE_NAME.zip" \ - --map-gen-settings "$CONFIG/map-gen-settings.json" \ - --map-settings "$CONFIG/map-settings.json" - fi -fi - -FLAGS=(\ - --port "$PORT" \ - --server-settings "$CONFIG/server-settings.json" \ - --server-banlist "$CONFIG/server-banlist.json" \ - --rcon-port "$RCON_PORT" \ - --server-whitelist "$CONFIG/server-whitelist.json" \ - --use-server-whitelist \ - --server-adminlist "$CONFIG/server-adminlist.json" \ - --rcon-password "$(cat "$CONFIG/rconpw")" \ - --server-id /factorio/config/server-id.json \ -) - -if [[ $LOAD_LATEST_SAVE == true ]]; then - FLAGS+=( --start-server-load-latest ) -else - FLAGS+=( --start-server "$SAVE_NAME" ) -fi - -# shellcheck disable=SC2086 -exec $SU_EXEC /opt/factorio/bin/x64/factorio "${FLAGS[@]}" "$@" \ No newline at end of file diff --git a/fle/cluster/docker/config/docker-update-mods.sh b/fle/cluster/docker/config/docker-update-mods.sh deleted file mode 100644 index bd3a223b1..000000000 --- a/fle/cluster/docker/config/docker-update-mods.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -eou pipefail - -if [[ -f /run/secrets/username ]]; then - USERNAME=$(cat /run/secrets/username) -fi - -if [[ -f /run/secrets/token ]]; then - TOKEN=$(cat /run/secrets/token) -fi - -if [[ -z ${USERNAME:-} ]]; then - USERNAME="$(jq -j ".username" "$CONFIG/server-settings.json")" -fi - -if [[ -z ${TOKEN:-} ]]; then - TOKEN="$(jq -j ".token" "$CONFIG/server-settings.json")" -fi - -if [[ -z ${USERNAME:-} ]]; then - echo "You need to provide your Factorio username to update mods." -fi - -if [[ -z ${TOKEN:-} ]]; then - echo "You need to provide your Factorio token to update mods." -fi - -./update-mods.sh "$VERSION" "$MODS" "$USERNAME" "$TOKEN" diff --git a/fle/cluster/docker/config/mod-list.json b/fle/cluster/docker/config/mod-list.json index 096a15ada..2bf421eea 100644 --- a/fle/cluster/docker/config/mod-list.json +++ b/fle/cluster/docker/config/mod-list.json @@ -4,14 +4,6 @@ { "name": "base", "enabled": true - }, - { - "name": "stdlib", - "enabled": true - }, - { - "name": "headless-player", - "enabled": true } ] } \ No newline at end of file diff --git a/fle/cluster/docker/config/update-mods.sh b/fle/cluster/docker/config/update-mods.sh deleted file mode 100644 index 20bfffeb5..000000000 --- a/fle/cluster/docker/config/update-mods.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash -set -eou pipefail - -FACTORIO_VERSION=$1 -MOD_DIR=$2 -USERNAME=$3 -TOKEN=$4 - -MOD_BASE_URL="https://mods.factorio.com" - -print_step() -{ - echo "$1" -} - -print_success() -{ - echo "$1" -} - -print_failure() -{ - echo "$1" -} - -update_mod() -{ - MOD_NAME="$1" - MOD_NAME_ENCODED="${1// /%20}" - - print_step "Checking for update of mod $MOD_NAME..." - - MOD_INFO_URL="$MOD_BASE_URL/api/mods/$MOD_NAME_ENCODED" - MOD_INFO_JSON=$(curl --silent "$MOD_INFO_URL") - - if ! echo "$MOD_INFO_JSON" | jq -e .name >/dev/null; then - print_success " Custom mod not on $MOD_BASE_URL, skipped." - return 0 - fi - - MOD_INFO=$(echo "$MOD_INFO_JSON" | jq -j --arg version "$FACTORIO_VERSION" ".releases|reverse|map(select(.info_json.factorio_version as \$mod_version | \$version | startswith(\$mod_version)))[0]|.file_name, \";\", .download_url, \";\", .sha1") - - MOD_FILENAME=$(echo "$MOD_INFO" | cut -f1 -d";") - MOD_URL=$(echo "$MOD_INFO" | cut -f2 -d";") - MOD_SHA1=$(echo "$MOD_INFO" | cut -f3 -d";") - - if [[ $MOD_FILENAME == null ]]; then - print_failure " Not compatible with version" - return 0 - fi - - if [[ -f $MOD_DIR/$MOD_FILENAME ]]; then - print_success " Already up-to-date." - return 0 - fi - - print_step "Downloading..." - FULL_URL="$MOD_BASE_URL$MOD_URL?username=$USERNAME&token=$TOKEN" - HTTP_STATUS=$(curl --silent -L -w "%{http_code}" -o "$MOD_DIR/$MOD_FILENAME" "$FULL_URL") - - if [[ $HTTP_STATUS != 200 ]]; then - print_failure " Download failed: Code $HTTP_STATUS." - rm -f "$MOD_DIR/$MOD_FILENAME" - return 1 - fi - - if [[ ! -f $MOD_DIR/$MOD_FILENAME ]]; then - print_failure " Downloaded file missing!" - return 1 - fi - - if ! [[ $(sha1sum "$MOD_DIR/$MOD_FILENAME") =~ $MOD_SHA1 ]]; then - print_failure " SHA1 mismatch!" - rm -f "$MOD_DIR/$MOD_FILENAME" - return 1 - fi - - print_success " Download complete." - - for file in "$MOD_DIR/${MOD_NAME}_"*".zip"; do # wildcard does usually not work in quotes: https://unix.stackexchange.com/a/67761 - if [[ $file != $MOD_DIR/$MOD_FILENAME ]]; then - print_success " Deleting old version: $file" - rm -f "$file" - fi - done - - return 0 -} - -if [[ -f $MOD_DIR/mod-list.json ]]; then - jq -r ".mods|map(select(.enabled))|.[].name" "$MOD_DIR/mod-list.json" | while read -r mod; do - if [[ $mod != base ]]; then - update_mod "$mod" - fi - done -fi diff --git a/fle/cluster/docker/install-docker.sh b/fle/cluster/docker/install-docker.sh deleted file mode 100644 index 7e0e7b3eb..000000000 --- a/fle/cluster/docker/install-docker.sh +++ /dev/null @@ -1,24 +0,0 @@ -#ssh -i "factorio.pem" ec2-user@ec2-18-133-239-115.eu-west-2.compute.amazonaws.com - -# Setup -sudo yum update -y -sudo amazon-linux-extras install docker -y -sudo yum install docker -y -sudo service docker start -sudo usermod -a -G docker ec2-user - -# Docker -sudo mkdir -p /opt/factorio -sudo chown 845:845 /opt/factorio -sudo docker run -d \ - -p 34197:34197/udp \ - -p 27015:27015/tcp \ - -v /opt/factorio:/factorio \ - --name factorio \ - --restart=always \ - factoriotools/factorio - -sudo docker logs factorio -sudo docker stop factorio - -nano /opt/factorio/config/server-settings.json \ No newline at end of file diff --git a/fle/cluster/docker/main.py b/fle/cluster/docker/main.py deleted file mode 100644 index eff008e7d..000000000 --- a/fle/cluster/docker/main.py +++ /dev/null @@ -1,8 +0,0 @@ -import docker -client = docker.from_env() - -container = client.containers.run('factorio', {'34197/udp': 34197, '27015/tcp': 27000}, platform='linux/amd64' - ,detach=True) - -print(container.logs()) - diff --git a/fle/cluster/docker/probe.sh b/fle/cluster/docker/probe.sh deleted file mode 100644 index 540a61e21..000000000 --- a/fle/cluster/docker/probe.sh +++ /dev/null @@ -1,9 +0,0 @@ -lines=$(echo | lsof -i -P -n | grep $1 | wc -c) - -if [[ $lines -gt 0 ]]; then - echo 'Running on' $1; - exit; -fi; - -echo 'Dead'; -exit 1; \ No newline at end of file diff --git a/fle/cluster/docker/requirements.txt b/fle/cluster/docker/requirements.txt deleted file mode 100644 index 6d0eac4b3..000000000 --- a/fle/cluster/docker/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -docker \ No newline at end of file diff --git a/fle/cluster/docker/run.sh b/fle/cluster/docker/run.sh deleted file mode 100755 index 23fb1d5f5..000000000 --- a/fle/cluster/docker/run.sh +++ /dev/null @@ -1,23 +0,0 @@ -docker build -t factorio . - -cd mods -tar -czf headless-player_0.1.0.tgz headless-player_0.1.0 -tar -czf stdlib_1.4.6.tgz stdlib_1.4.6 - -mv stdlib_1.4.6.tgz ~/Library/Application\ Support/Factorio/mods/stdlib_1.4.6.tgz -mv headless-player_0.1.0.tgz ~/Library/Application\ Support/Factorio/mods/headless-player_0.1.0.tgz - -rm -rf ~/Library/Application\ Support/Factorio/mods/stdlib_1.4.6 -rm -rf ~/Library/Application\ Support/Factorio/mods/headless-player_0.1.0 - -cp -rf stdlib_1.4.6 ~/Library/Application\ Support/Factorio/mods/stdlib_1.4.6 -cp -rf headless-player_0.1.0 ~/Library/Application\ Support/Factorio/mods/headless-player_0.1.0 - -cd .. - -cp config/mod-list.json ~/Library/Application\ Support/Factorio/mods/mod-list.json - -docker run -d \ - -p 34197:34197/udp \ - -p 27015:27015/tcp \ - factorio \ No newline at end of file diff --git a/fle/cluster/docker/run_local.sh b/fle/cluster/docker/run_local.sh deleted file mode 100644 index d20e07052..000000000 --- a/fle/cluster/docker/run_local.sh +++ /dev/null @@ -1,6 +0,0 @@ -docker run -d \ - -p 34197:34197/udp \ - -p 27015:27015/tcp \ - --name=factorio \ - --restart=always \ - -it factorio \ No newline at end of file diff --git a/fle/cluster/docker/setup_docker_repo.sh b/fle/cluster/docker/setup_docker_repo.sh deleted file mode 100644 index d1de48a7b..000000000 --- a/fle/cluster/docker/setup_docker_repo.sh +++ /dev/null @@ -1,13 +0,0 @@ -aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin 216370203482.dkr.ecr.eu-west-2.amazonaws.com -aws ecr create-repository --repository-name factorio - -#public.ecr.aws/y5j3i3w6/raveler - -#docker login --username AWS --password-stdin public.ecr.aws/y5j3i3w6/paperclip_client:latest -#docker tag hello-world public.ecr.aws/y5j3i3w6/paperclip_client:latest -#docker push public.ecr.aws/y5j3i3w6/paperclip_client - -#docker tag hello-world 216370203482.dkr.ecr.eu-west-2.amazonaws.com/hello-world -#docker push 216370203482.dkr.ecr.eu-west-2.amazonaws.com/hello-world #public.ecr.aws/y5j3i3w6/paperclip_client - -#216370203482.dkr.ecr.eu-west-2.amazonaws.com/hello-world \ No newline at end of file diff --git a/fle/cluster/remote/cluster_ips.py b/fle/cluster/remote/cluster_ips.py index 5abbf44c5..831077280 100644 --- a/fle/cluster/remote/cluster_ips.py +++ b/fle/cluster/remote/cluster_ips.py @@ -4,6 +4,7 @@ import sys from dotenv import load_dotenv +from fle.logger import info, error, debug, warning def get_public_ips(cluster_name): @@ -17,7 +18,7 @@ def get_public_ips(cluster_name): tasks.extend(page['taskArns']) if not tasks: - print(f"No running tasks found in cluster {cluster_name}") + warning(f"No running tasks found in cluster {cluster_name}") return [] public_ips = [] @@ -40,7 +41,7 @@ def get_public_ips(cluster_name): break if not eni_id: - print(f"Warning: No ENI found for task {task['taskArn']}") + warning(f"No ENI found for task {task['taskArn']}") continue # Describe the network interface to get its public IP @@ -50,7 +51,7 @@ def get_public_ips(cluster_name): public_ip = eni_details['NetworkInterfaces'][0]['Association']['PublicIp'] public_ips.append(public_ip) except Exception as e: - print(f"Error getting public IP for ENI {eni_id}: {str(e)}") + error(f"Error getting public IP for ENI {eni_id}: {str(e)}") return public_ips @@ -67,14 +68,14 @@ def get_public_ips(cluster_name): elif default_cluster_name: cluster_name = default_cluster_name else: - print("Error: CLUSTER_NAME not set in .env file and not provided as argument") - print("Usage: python cluster_ip.py [cluster_name]") + error("CLUSTER_NAME not set in .env file and not provided as argument") + error("Usage: python cluster_ip.py [cluster_name]") sys.exit(1) public_ips = get_public_ips(cluster_name) if public_ips: - print("Public IP addresses of running containers:") + info("Public IP addresses of running containers:") for ip in public_ips: - print(ip) + info(ip) else: - print("No public IP addresses found for running containers.") \ No newline at end of file + warning("No public IP addresses found for running containers.") \ No newline at end of file diff --git a/fle/cluster/remote/factorio_server_login.py b/fle/cluster/remote/factorio_server_login.py index d18d752c1..e8912b2be 100644 --- a/fle/cluster/remote/factorio_server_login.py +++ b/fle/cluster/remote/factorio_server_login.py @@ -10,6 +10,7 @@ from cluster.remote.cluster_ips import get_public_ips from fle.env import FactorioInstance +from fle.logger import info, error, debug, warning """ This script is used to connect the client to each Factorio server running on an ECS cluster, @@ -75,7 +76,7 @@ def is_initialised(ip_address, port): fast=True) return True except Exception as e: - print(f"Error connecting to {ip_address}: {str(e)}") + error(f"Error connecting to {ip_address}: {str(e)}") return False @@ -93,7 +94,7 @@ def get_uninitialised_ips(ip_addresses: List[str], tcp_ports: List[str], max_wor invalid_ips = [] total_ips = len(ip_addresses) - print(f"Starting initialization check for {total_ips} IP addresses...") + info(f"Starting initialization check for {total_ips} IP addresses...") start_time = time.time() with ThreadPoolExecutor(max_workers=max_workers) as executor: @@ -106,16 +107,16 @@ def get_uninitialised_ips(ip_addresses: List[str], tcp_ports: List[str], max_wor ip = future_to_ip[future] try: if future.result(): - print(f"Progress: {i}/{total_ips} - {ip} is valid") + info(f"Progress: {i}/{total_ips} - {ip} is valid") else: invalid_ips.append(ip) - print(f"Progress: {i}/{total_ips} - {ip} is invalid") + warning(f"Progress: {i}/{total_ips} - {ip} is invalid") except Exception as e: - print(f"Progress: {i}/{total_ips} - Unexpected error with {ip}: {str(e)}") + error(f"Progress: {i}/{total_ips} - Unexpected error with {ip}: {str(e)}") elapsed_time = time.time() - start_time - print(f"\nCompleted in {elapsed_time:.2f} seconds") - print(f"Found {len(invalid_ips)} uninitialised IPs out of {total_ips}") + info(f"Completed in {elapsed_time:.2f} seconds") + info(f"Found {len(invalid_ips)} uninitialised IPs out of {total_ips}") return invalid_ips @@ -131,7 +132,7 @@ def main(cluster_name): factorio_process = launch_factorio() for ip in ip_addresses: connect_to_server(ip) - print(f"Connected to and quit from {ip}") + info(f"Connected to and quit from {ip}") factorio_process.terminate() # Ensure Factorio is closed at the end if __name__ == "__main__": @@ -144,7 +145,7 @@ def main(cluster_name): elif default_cluster_name: cluster_name = default_cluster_name else: - print("Error: CLUSTER_NAME not set in .env file and not provided as argument") - print("Usage: python factorio_server_login.py [cluster_name]") + error("CLUSTER_NAME not set in .env file and not provided as argument") + error("Usage: python factorio_server_login.py [cluster_name]") sys.exit(1) main(cluster_name) diff --git a/fle/commons/cluster_ips.py b/fle/commons/cluster_ips.py index e05ce5067..e554bfebb 100644 --- a/fle/commons/cluster_ips.py +++ b/fle/commons/cluster_ips.py @@ -1,6 +1,7 @@ import subprocess import json from typing import List +from fle.logger import info, error, debug, warning def get_local_container_ips() -> tuple[List[str], List[int], List[int]]: @@ -12,7 +13,7 @@ def get_local_container_ips() -> tuple[List[str], List[int], List[int]]: container_ids = [id.strip('"') for id in container_ids] if not container_ids or container_ids[0] == '': - print("No running Factorio containers found") + warning("No running Factorio containers found") return [] ips = [] @@ -51,8 +52,8 @@ def get_local_container_ips() -> tuple[List[str], List[int], List[int]]: if __name__ == "__main__": ips, udp_ports, tcp_ports = get_local_container_ips() if ips: - print("Local Factorio container addresses:") + info("Local Factorio container addresses:") for ip in ips: - print(ip) + info(ip) else: - print("No local Factorio containers found.") \ No newline at end of file + warning("No local Factorio containers found.") \ No newline at end of file diff --git a/fle/commons/db_client.py b/fle/commons/db_client.py index 633afabfd..63b135ca7 100644 --- a/fle/commons/db_client.py +++ b/fle/commons/db_client.py @@ -21,6 +21,7 @@ from fle.commons.models.conversation import Conversation from fle.commons.models.game_state import GameState from fle.commons.models.program import Program +from fle.logger import info, error, debug, warning # Configure logging logging.basicConfig(level=logging.INFO) @@ -107,7 +108,7 @@ async def version_exists(self, version: int) -> bool: """, (version,)) return cur.fetchone()[0] except Exception as e: - print(f"Error checking version existence: {e}") + error(f"Error checking version existence: {e}") return False async def get_version_metadata(self, version: int) -> dict: @@ -124,7 +125,7 @@ async def get_version_metadata(self, version: int) -> dict: result = cur.fetchone() return dict(result) if result else {} except Exception as e: - print(f"Error fetching version metadata: {e}") + error(f"Error fetching version metadata: {e}") return {} @tenacity.retry( @@ -170,7 +171,7 @@ async def create_program(self, program: Program) -> Program: return program except Exception as e: conn.rollback() - print(f"Error creating program: {e}") + error(f"Error creating program: {e}") raise e async def cleanup(self): @@ -206,7 +207,7 @@ async def get_all_program_rewards(self, version: int = None) -> List[float]: results = cur.fetchall() return [row[0] for row in results] except Exception as e: - print(f"Error fetching program rewards: {e}") + error(f"Error fetching program rewards: {e}") return [] @@ -226,7 +227,7 @@ async def get_largest_version(self) -> int: result = cur.fetchone() return result[0] if result and result[0] is not None else 0 except Exception as e: - print(f"Error fetching largest version: {e}") + error(f"Error fetching largest version: {e}") return 0 async def get_largest_depth_in_version(self, version): @@ -243,7 +244,7 @@ async def get_largest_depth_in_version(self, version): result = cur.fetchone() return result[0] if result and result[0] is not None else 0 except Exception as e: - print(f"Error fetching largest depth: {e}") + error(f"Error fetching largest depth: {e}") return 0 @@ -309,7 +310,7 @@ def transform_reward(value): return (compressed + 1.0) / 2.0 + 1e-6 # Log current compression state - print(f"Using compression strength: {compression_strength:.3f} " + info(f"Using compression strength: {compression_strength:.3f} " f"({'adaptive' if compression_strength is None else 'fixed'})") # Calculate transformed weights @@ -338,7 +339,7 @@ def transform_reward(value): row = cur.fetchone() return Program.from_row(dict(row)) if row else None except Exception as e: - print(f"Error sampling parent: {e}") + error(f"Error sampling parent: {e}") raise e @@ -366,7 +367,7 @@ async def update_program(self, program_id: int, updates: Dict[str, Any]) -> Prog row = cur.fetchone() return Program.from_row(dict(zip([desc[0] for desc in cur.description], row))) except Exception as e: - print(f"Error updating program: {e}") + error(f"Error updating program: {e}") raise e async def get_resume_state(self, resume_version, process_id, agent_idx=-1) -> tuple[Optional[GameState], Optional[Conversation], Optional[int], Optional[int]]: @@ -389,7 +390,7 @@ async def get_resume_state(self, resume_version, process_id, agent_idx=-1) -> tu cur.execute(query, (resume_version, process_id, agent_idx)) results = cur.fetchall() if not results: - print(f"No valid programs found for version {resume_version} ") + warning(f"No valid programs found for version {resume_version} ") return None, None, None, None # Choose a program to resume from @@ -397,7 +398,7 @@ async def get_resume_state(self, resume_version, process_id, agent_idx=-1) -> tu return program.state, program.conversation, program.id, program.depth except Exception as e: - print(f"Error getting resume state: {e}") + error(f"Error getting resume state: {e}") return None, None, None, None @@ -440,7 +441,7 @@ def get_connection(self): try: self._pool.putconn(conn) except Exception as e: - print(f"Error returning connection to pool: {e}") + error(f"Error returning connection to pool: {e}") try: self._pool.putconn(conn, close=True) except: @@ -489,7 +490,7 @@ async def get_largest_version(self) -> int: result = cur.fetchone() return result[0] if result and result[0] is not None else 0 except Exception as e: - print(f"Error fetching largest version: {e}") + error(f"Error fetching largest version: {e}") return 0 @@ -514,7 +515,7 @@ async def get_resume_state(self, resume_version, process_id, agent_idx=-1) -> tu results = cur.fetchall() if not results: - print(f"No valid programs found for version {resume_version}") + warning(f"No valid programs found for version {resume_version}") return None, None, None, None resulting_program_dict = dict(zip([desc[0] for desc in cur.description], results[0])) # make meta, state_json achievements_json and conversation_json table a dict @@ -527,7 +528,7 @@ async def get_resume_state(self, resume_version, process_id, agent_idx=-1) -> tu return program.state, program.conversation, program.id, program.depth except Exception as e: - print(f"Error getting resume state: {e}") + error(f"Error getting resume state: {e}") return None, None, None, None @@ -584,7 +585,7 @@ async def create_program(self, program: Program) -> Program: return program except Exception as e: conn.rollback() - print(f"Error creating program: {e}") + error(f"Error creating program: {e}") raise e def create_default_sqlite_db(db_file: str) -> None: @@ -606,7 +607,7 @@ def create_default_sqlite_db(db_file: str) -> None: """) if not cursor.fetchone(): - print(f"Creating SQLite database schema in {db_file}") + info(f"Creating SQLite database schema in {db_file}") cursor.execute(""" CREATE TABLE programs ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -636,7 +637,7 @@ def create_default_sqlite_db(db_file: str) -> None: ) """) conn.commit() - print("SQLite database schema created successfully!") + info("SQLite database schema created successfully!") finally: conn.close() @@ -661,7 +662,7 @@ def create_default_postgres_db(**db_config) -> None: table_exists = cursor.fetchone()[0] if not table_exists: - print(f"Creating PostgreSQL database schema") + info(f"Creating PostgreSQL database schema") cursor.execute(""" CREATE TABLE programs ( id SERIAL PRIMARY KEY, @@ -691,10 +692,10 @@ def create_default_postgres_db(**db_config) -> None: ) """) conn.commit() - print("PostgreSQL database schema created successfully!") + info("PostgreSQL database schema created successfully!") except Exception as e: - print(f"Error creating PostgreSQL schema: {e}") + error(f"Error creating PostgreSQL schema: {e}") if conn: conn.rollback() raise @@ -719,8 +720,8 @@ async def create_db_client(max_conversation_length: int = 40, missing_vars = [var for var in required_vars if not os.getenv(var)] if missing_vars: - print(f"Warning: PostgreSQL requested but missing environment variables: {missing_vars}") - print("Falling back to SQLite...") + warning(f"PostgreSQL requested but missing environment variables: {missing_vars}") + warning("Falling back to SQLite...") raise Exception(f"Missing environment variables: {missing_vars}") db_config = { "host": os.getenv("SKILLS_DB_HOST"), @@ -740,7 +741,7 @@ async def create_db_client(max_conversation_length: int = 40, elif db_type == "sqlite": # Default to SQLite sqlite_file = os.getenv("SQLITE_DB_FILE", ".fle/data.db") - print(f"Using SQLite database file: {sqlite_file}") + info(f"Using SQLite database file: {sqlite_file}") # Auto-create SQLite database if it doesn't exist create_default_sqlite_db(sqlite_file) diff --git a/fle/env/gym_env/run_eval.py b/fle/env/gym_env/run_eval.py index a73ab1ddb..908c98aa5 100644 --- a/fle/env/gym_env/run_eval.py +++ b/fle/env/gym_env/run_eval.py @@ -67,15 +67,9 @@ async def run_trajectory(run_idx: int, config: GymEvalConfig): await runner.run() await db_client.cleanup() -async def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--run_config', type=str, - help='Path of the run config file', - default=Path("eval", "open", "independent_runs", "gym_run_config.json")) - args = parser.parse_args() - +async def main(run_configs): # Read and validate run configurations - run_configs = get_validated_run_configs(args.run_config) + run_configs = get_validated_run_configs(run_configs) # Get starting version number for new runs base_version = await get_next_version() diff --git a/fle/env/utils/rcon.py b/fle/env/utils/rcon.py index 51599c2a0..a757b7be6 100644 --- a/fle/env/utils/rcon.py +++ b/fle/env/utils/rcon.py @@ -14,6 +14,7 @@ import contextlib from timeit import default_timer as timer import re +from fle.logger import info, error, debug @@ -167,7 +168,7 @@ def _lua2python(command, response, *parameters, trace=False, start=0): except Exception as e: if trace: - print(f"Parsing error: {str(e)}") + debug(f"Parsing error: {str(e)}") return None, (timer() - start) @deprecated("Doesn't handle nested structures that well") @@ -177,11 +178,11 @@ def _lua2python_old(command, response, *parameters, trace=False, start=0): with contextlib.redirect_stdout(stdout): if trace: - print(command, parameters, response) + debug(f"Command: {command}, Parameters: {parameters}, Response: {response}") if response: if trace: - print(f"success: {command}") + debug(f"success: {command}") end = timer() if response[0] != '{': @@ -205,7 +206,7 @@ def _lua2python_old(command, response, *parameters, trace=False, start=0): output = None if trace: - print("{hbar}\nCOMMAND: {command}\nPARAMETERS: {parameters}\n\n{response}\n\nOUTPUT:{output}" + debug("{hbar}\nCOMMAND: {command}\nPARAMETERS: {parameters}\n\n{response}\n\nOUTPUT:{output}" .format(hbar="-" * 100, command=command, parameters=parameters, response=response, output=output)) @@ -221,7 +222,7 @@ def _lua2python_old(command, response, *parameters, trace=False, start=0): return output, (end - start) else: if trace: - print(f"failure: {command} \t") + debug(f"failure: {command} \t") end = timer() try: diff --git a/fle/logger.py b/fle/logger.py new file mode 100644 index 000000000..11fc900ae --- /dev/null +++ b/fle/logger.py @@ -0,0 +1,123 @@ +import logging +import os +import sys +from logging.handlers import RotatingFileHandler +from typing import Optional + + +class FLELogger: + """ + Centralized logger for the FLE project. + Provides different log levels and handles both file and console output. + """ + + _instance: Optional['FLELogger'] = None + _initialized = False + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + def __init__(self): + if not self._initialized: + self._setup_logger() + FLELogger._initialized = True + + def _setup_logger(self): + """Setup the logger with appropriate handlers and formatters.""" + # Create logger + self.logger = logging.getLogger('fle') + self.logger.setLevel(logging.DEBUG) + + # Create formatter + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + # Console handler + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.INFO) + console_handler.setFormatter(formatter) + self.logger.addHandler(console_handler) + + # File handler (rotating) + log_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'logs') + os.makedirs(log_dir, exist_ok=True) + + file_handler = RotatingFileHandler( + os.path.join(log_dir, 'fle.log'), + maxBytes=10*1024*1024, # 10MB + backupCount=5 + ) + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(formatter) + self.logger.addHandler(file_handler) + + def debug(self, message: str, *args, **kwargs): + """Log debug message.""" + self.logger.debug(message, *args, **kwargs) + + def info(self, message: str, *args, **kwargs): + """Log info message.""" + self.logger.info(message, *args, **kwargs) + + def warning(self, message: str, *args, **kwargs): + """Log warning message.""" + self.logger.warning(message, *args, **kwargs) + + def error(self, message: str, *args, **kwargs): + """Log error message.""" + self.logger.error(message, *args, **kwargs) + + def critical(self, message: str, *args, **kwargs): + """Log critical message.""" + self.logger.critical(message, *args, **kwargs) + + def exception(self, message: str, *args, **kwargs): + """Log exception with traceback.""" + self.logger.exception(message, *args, **kwargs) + + +# Global logger instance +_logger_instance = None + + +def get_logger() -> FLELogger: + """Get the global logger instance.""" + global _logger_instance + if _logger_instance is None: + _logger_instance = FLELogger() + return _logger_instance + + +# Convenience functions for direct usage +def debug(message: str, *args, **kwargs): + """Log debug message.""" + get_logger().debug(message, *args, **kwargs) + + +def info(message: str, *args, **kwargs): + """Log info message.""" + get_logger().info(message, *args, **kwargs) + + +def warning(message: str, *args, **kwargs): + """Log warning message.""" + get_logger().warning(message, *args, **kwargs) + + +def error(message: str, *args, **kwargs): + """Log error message.""" + get_logger().error(message, *args, **kwargs) + + +def critical(message: str, *args, **kwargs): + """Log critical message.""" + get_logger().critical(message, *args, **kwargs) + + +def exception(message: str, *args, **kwargs): + """Log exception with traceback.""" + get_logger().exception(message, *args, **kwargs) \ No newline at end of file diff --git a/fle/logs/fle.log b/fle/logs/fle.log new file mode 100644 index 000000000..b6f52a711 --- /dev/null +++ b/fle/logs/fle.log @@ -0,0 +1,5 @@ +2025-07-09 23:00:36 - fle - INFO - Using SQLite database file: .fle/data.db +2025-07-09 23:00:36 - fle - INFO - Creating SQLite database schema in .fle/data.db +2025-07-09 23:00:36 - fle - INFO - SQLite database schema created successfully! +2025-07-09 23:00:38 - fle - INFO - Using SQLite database file: .fle/data.db +2025-07-09 23:00:39 - fle - WARNING - No valid programs found for version 1 diff --git a/fle/run.py b/fle/run.py index 7b8f8f2ed..511eb5248 100644 --- a/fle/run.py +++ b/fle/run.py @@ -4,6 +4,7 @@ import sys from pathlib import Path from fle.env.gym_env.run_eval import main as run_eval +from fle.logger import error def main(): @@ -32,19 +33,18 @@ def main(): # Validate that the config file exists config_path = Path(args.run_config) if not config_path.exists(): - print(f"Error: Configuration file '{args.run_config}' not found.", file=sys.stderr) + error(f"Configuration file '{args.run_config}' not found.") sys.exit(1) # Set up arguments for run_eval and call it original_argv = sys.argv.copy() try: - sys.argv = ['run_eval', '--run_config', str(config_path)] - asyncio.run(run_eval()) + asyncio.run(run_eval(config_path)) except KeyboardInterrupt: - print("\nInterrupted by user.", file=sys.stderr) + error("Interrupted by user.") sys.exit(1) except Exception as e: - print(f"Error: {e}", file=sys.stderr) + error(f"Error: {e}") sys.exit(1) finally: sys.argv = original_argv diff --git a/fle/server.py b/fle/server.py index 384a8c409..abf84d579 100644 --- a/fle/server.py +++ b/fle/server.py @@ -5,6 +5,7 @@ import env.protocols.mcp.prompts import env.protocols.mcp.unix_tools import env.protocols.mcp.version_control +from fle.logger import info, error # Command-line interface for the MCP server if __name__ == "__main__": @@ -20,11 +21,11 @@ args = parser.parse_args() if args.transport == "stdio": - print("Starting Factorio MCP server with stdio transport") + info("Starting Factorio MCP server with stdio transport") mcp.run(transport='stdio') elif args.transport == "sse": - print(f"Starting Factorio MCP server with SSE transport on {args.host}:{args.port}") + info(f"Starting Factorio MCP server with SSE transport on {args.host}:{args.port}") mcp.run(transport='sse', host=args.host, port=args.port) else: - print(f"Unknown transport: {args.transport}") + error(f"Unknown transport: {args.transport}") sys.exit(1) \ No newline at end of file