flashbots · ilyaluk · Oct 21, 2025 · Oct 21, 2025 · Oct 1, 2025 · Oct 23, 2025
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
@@ -27,6 +27,7 @@ flashboxes/
 │   └── debloat*.sh         # System cleanup scripts
 ├── bob-common/             # TEE Searcher common image
 ├── bob-l1/                 # L1 TEE Searcher sandbox image
+├── bob-l2/                 # L2 TEE Searcher sandbox image
 ├── buildernet/             # BuilderNet
 ├── tdx-dummy/              # TDX test environment
 ├── kernel/                 # Kernel configuration

diff --git a/Makefile b/Makefile
@@ -39,11 +39,11 @@ setup: ## Install dependencies (Linux only)
 
 # Build module
 build: check-perms setup ## Build the specified module
-	$(WRAPPER) mkosi --force -I $(IMAGE).conf
+	$(WRAPPER) mkosi --force --include=$(IMAGE).conf
 
 # Build module with devtools profile
 build-dev: check-perms setup ## Build module with development tools
-	$(WRAPPER) mkosi --force --profile=devtools -I $(IMAGE).conf
+	$(WRAPPER) mkosi --force --profile=devtools --include=$(IMAGE).conf
 
 ##@ Utilities
 

diff --git a/bob-common/mkosi.build b/bob-common/mkosi.build
@@ -35,7 +35,7 @@ make_git_package \
 # Build tdx-init
 make_git_package \
     "tdx-init" \
-    "v0.1.1" \
+    "ilya/disk-resize" \
     "https://github.com/flashbots/tdx-init" \
     'go build -trimpath -ldflags "-s -w -buildid=" -o ./build/tdx-init' \
     "build/tdx-init:/usr/bin/tdx-init"

diff --git a/bob-common/mkosi.extra/usr/bin/init-container.sh b/bob-common/mkosi.extra/usr/bin/init-container.sh
@@ -5,7 +5,6 @@ NAME=searcher-container
 
 # PORT FORWARDS
 SEARCHER_SSH_PORT=10022
-EL_P2P_PORT=30303
 SEARCHER_INPUT_CHANNEL=27017
 
 # Run extra commands which are customized per image,
@@ -21,8 +20,6 @@ su -s /bin/sh searcher -c "cd ~ && podman run -d \
     --name $NAME --replace \
     --init \
     -p ${SEARCHER_SSH_PORT}:22 \
-    -p ${EL_P2P_PORT}:${EL_P2P_PORT} \
-    -p ${EL_P2P_PORT}:${EL_P2P_PORT}/udp \
     -p ${SEARCHER_INPUT_CHANNEL}:${SEARCHER_INPUT_CHANNEL}/udp \
     -v /persistent/searcher:/persistent:rw \
     -v /etc/searcher/ssh_hostkey:/etc/searcher/ssh_hostkey:rw \

diff --git a/bob-common/mkosi.extra/usr/bin/init-firewall.sh b/bob-common/mkosi.extra/usr/bin/init-firewall.sh
@@ -124,6 +124,18 @@ accept_dst_ip_port() {
                          -m comment --comment "$comment"
 }
 
+accept_src_ip_dst_port() {
+    chain="$1"
+    protocol="$2"
+    ip="$3"
+    port="$4"
+    comment="$5"
+
+    iptables -A "$chain" -p "$protocol" -s "$ip" --dport "$port" \
+                         -m conntrack --ctstate NEW -j ACCEPT \
+                         -m comment --comment "$comment"
+}
+
 drop_dst_ip() {
     chain="$1"
     ip="$2"

diff --git a/bob-l1/mkosi.extra/etc/bob/searcher-container-before-init b/bob-l1/mkosi.extra/etc/bob/searcher-container-before-init
@@ -2,9 +2,12 @@
 # See also: bob-common/mkosi.extra/usr/bin/init-container.sh
 
 ENGINE_API_PORT=8551
+EL_P2P_PORT=30303
 
 BOB_SEARCHER_EXTRA_PODMAN_FLAGS="\
     -p ${ENGINE_API_PORT}:${ENGINE_API_PORT} \
+    -p ${EL_P2P_PORT}:${EL_P2P_PORT} \
+    -p ${EL_P2P_PORT}:${EL_P2P_PORT}/udp \
     -v /persistent/lighthouse_logs:/var/log/lighthouse:ro \
     -v /tmp/jwt.hex:/secrets/jwt.hex:ro \
 "
diff --git a/bob-l2.conf b/bob-l2.conf
@@ -0,0 +1,13 @@
+[Include]
+Include=base/mkosi.conf
+Include=bob-common/mkosi.conf
+Include=bob-l2/mkosi.conf
+
+[Config]
+Profiles=gcp
+
+[Distribution]
+Mirror=https://snapshot.debian.org/archive/debian/20250526T142542Z/
+
+[Build]
+ToolsTreeMirror=https://snapshot.debian.org/archive/debian/20250526T142542Z/
diff --git a/bob-l2/kernel.config b/bob-l2/kernel.config
@@ -0,0 +1,45 @@
+CONFIG_NET_VENDOR_GOOGLE=y
+CONFIG_GVE=y
+
+# Enable iptables interface for CONFIG_NF_TABLES
+# Same config as in bob-l1
+CONFIG_IPV6=n
+CONFIG_NETFILTER_NETLINK=y
+CONFIG_NETFILTER_NETLINK_LOG=y
+CONFIG_NF_CONNTRACK_MARK=y
+CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NF_CT_PROTO_SCTP=y
+CONFIG_NF_CT_PROTO_UDPLITE=y
+CONFIG_NF_CT_NETLINK=y
+CONFIG_NF_NAT_NEEDED=y
+CONFIG_NF_TABLES=y
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_IPV4=y
+CONFIG_NF_TABLES_BRIDGE=y
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NETFILTER_XTABLES_COMPAT=y
+CONFIG_NFT_CT=y
+CONFIG_NFT_COUNTER=y
+CONFIG_NFT_LOG=y
+CONFIG_NFT_LIMIT=y
+CONFIG_NFT_MASQ=y
+CONFIG_NFT_REJECT=y
+CONFIG_NFT_REJECT_INET=y
+CONFIG_NFT_COMPAT=y
+CONFIG_NFT_NAT=y
+CONFIG_NFT_REDIR=y
+CONFIG_NFT_OBJREF=y
+CONFIG_NETFILTER_XT_TARGET_LOG=y
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y
+CONFIG_NETFILTER_XT_MATCH_STATE=y
+CONFIG_IP_NF_TARGET_REJECT=y
+CONFIG_IP_NF_TARGET_NETMAP=y
+CONFIG_IP_NF_TARGET_REDIRECT=y
+CONFIG_IP_NF_MANGLE=y
+CONFIG_IP_NF_RAW=y
+CONFIG_NET_SCHED=y
+CONFIG_CRYPTO_USER_API_HASH=y
+CONFIG_CRYPTO_USER_API_SKCIPHER=y
+CONFIG_CRYPTO_USER_API_RNG=y
+CONFIG_CRYPTO_USER_API_AEAD=y
diff --git a/bob-l2/mkosi.conf b/bob-l2/mkosi.conf
@@ -0,0 +1,10 @@
+[Build]
+Environment=KERNEL_CONFIG_SNIPPETS=kernel/snippets/ubuntu.config,bob-l2/kernel.config
+WithNetwork=true
+
+[Content]
+ExtraTrees=bob-l2/mkosi.extra
+PostInstallationScripts=bob-l2/mkosi.postinst
+
+Packages=chrony
+         dmidecode
diff --git a/bob-l2/mkosi.extra/etc/bob/firewall-config b/bob-l2/mkosi.extra/etc/bob/firewall-config
@@ -0,0 +1,85 @@
+# This script is sourced from firewall script and contains image-specific rules
+# See also: bob-common/mkosi.extra/usr/bin/init-firewall.sh
+
+# Image-specific ports
+SSH_CONTROL_PORT=22
+SSH_DATA_PORT=10022
+SSH_REGISTER_PORT=8080
+CVM_REVERSE_PROXY_PORT=8745
+SEARCHER_INPUT_PORT=27017
+
+# Well-known ports
+DNS_PORT=53
+HTTP_PORT=80
+HTTPS_PORT=443
+NTP_PORT=123
+OP_NODE_P2P_PORT=9222
+OP_GETH_P2P_PORT=40404
+ENGINE_API_PORT=8651
+
+###########################################################################
+# (1) ALWAYS_IN: Inbound rules that are always applied
+###########################################################################
+
+accept_dst_port $CHAIN_ALWAYS_IN tcp $SSH_CONTROL_PORT "SSH control port"
+accept_dst_port $CHAIN_ALWAYS_IN udp $SEARCHER_INPUT_PORT "Searcher input channel"
+
+# We drive op-geth in the searcher container from external op-node
+# We assume here that static peers in config are only syn nodes
+accept_src_ip_dst_port $CHAIN_ALWAYS_IN tcp "$CONFIG_EL_PEERS_IPS" $ENGINE_API_PORT "Engine API"
+
+# CVM reverse-proxy serves server attestation
+# Also forwards request to ssh pubkey server on localhost:5001,
+#    which serves searcher-container openssh server pubkey
+accept_dst_port $CHAIN_ALWAYS_IN tcp $CVM_REVERSE_PROXY_PORT "CVM reverse-proxy"
+
+###########################################################################
+# (2) ALWAYS_OUT: Outbound rules that are always applied
+###########################################################################
+
+# Note: this is accessible only from host, searcher netns has DROP on those
+# See also init-container.sh
+accept_dst_port $CHAIN_ALWAYS_OUT udp $NTP_PORT "NTP"
+
+accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp "$CONFIG_SIMULATOR_IP" $HTTP_PORT "bundle"
+
+# TODO: this is temporary for tests
+iptables -A $CHAIN_ALWAYS_OUT -j ACCEPT
+
+###########################################################################
+# (3) MAINTENANCE_IN: Inbound rules for Maintenance Mode
+###########################################################################
+
+accept_dst_port $CHAIN_MAINTENANCE_IN tcp $SSH_DATA_PORT "SSH data plane"
+accept_dst_port $CHAIN_MAINTENANCE_IN tcp $SSH_REGISTER_PORT "SSH register service"
+
+accept_dst_port $CHAIN_MAINTENANCE_IN tcp $OP_GETH_P2P_PORT "op-geth P2P (TCP)"
+accept_dst_port $CHAIN_MAINTENANCE_IN udp $OP_GETH_P2P_PORT "op-geth P2P (UDP)"
+
+###########################################################################
+# (4) MAINTENANCE_OUT: Outbound rules for Maintenance Mode
+###########################################################################
+
+# Block tx endpoint during maintenance
+drop_dst_ip $CHAIN_MAINTENANCE_OUT "$CONFIG_SIMULATOR_IP" "tx stream (DROP before accept-all rules)"
+
+accept_dst_port $CHAIN_MAINTENANCE_OUT udp $DNS_PORT "DNS (UDP)"
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $DNS_PORT "DNS (TCP)"
+
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $HTTP_PORT "HTTP"
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $HTTPS_PORT "HTTPS"
+
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $OP_GETH_P2P_PORT "op-geth P2P (TCP)"
+accept_dst_port $CHAIN_MAINTENANCE_OUT udp $OP_GETH_P2P_PORT "op-geth P2P (UDP)"
+
+###########################################################################
+# (5) PRODUCTION_IN: Inbound rules for Production Mode
+###########################################################################
+
+# None at the moment
+
+###########################################################################
+# (6) PRODUCTION_OUT: Outbound rules for Production Mode
+###########################################################################
+
+accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp "$CONFIG_SIMULATOR_IP" $HTTP_PORT "tx stream"
diff --git a/bob-l2/mkosi.extra/etc/bob/searcher-container-after-init b/bob-l2/mkosi.extra/etc/bob/searcher-container-after-init
@@ -0,0 +1,7 @@
+# This script is sourced from init-container.sh and contains image-specific stuff
+# See also: bob-common/mkosi.extra/usr/bin/init-container.sh
+
+exec_in_container "
+    cat <<EOF >> /etc/hosts
+$CONFIG_SIMULATOR_IP simulator.internal
+EOF"
diff --git a/bob-l2/mkosi.extra/etc/bob/searcher-container-before-init b/bob-l2/mkosi.extra/etc/bob/searcher-container-before-init
@@ -0,0 +1,12 @@
+# This script is sourced from init-container.sh and contains image-specific stuff
+# See also: bob-common/mkosi.extra/usr/bin/init-container.sh
+
+ENGINE_API_PORT=8651
+EL_P2P_PORT=40404
+
+BOB_SEARCHER_EXTRA_PODMAN_FLAGS="\
+    -p ${ENGINE_API_PORT}:${ENGINE_API_PORT} \
+    -p ${EL_P2P_PORT}:${EL_P2P_PORT} \
+    -p ${EL_P2P_PORT}:${EL_P2P_PORT}/udp \
+    -v /etc/bob/config.env:/etc/config.env:ro \
+"
diff --git a/bob-l2/mkosi.extra/etc/searcher-container-init-extra b/bob-l2/mkosi.extra/etc/searcher-container-init-extra
@@ -0,0 +1,7 @@
+# This script is sourced from init-container.sh and contains image-specific stuff
+# See also: bob-common/mkosi.extra/usr/bin/init-container.sh
+
+echo "Injecting static hosts into searcher container..."
+exec_in_container '
+    cat <<EOF >> /etc/hosts
+EOF'
diff --git a/bob-l2/mkosi.extra/etc/systemd/system/fetch-config.service b/bob-l2/mkosi.extra/etc/systemd/system/fetch-config.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Fetch some configuration variables from Vault
+After=network.target network-setup.service
+Requires=network-setup.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/fetch-config.sh
+RemainAfterExit=yes
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=minimal.target
diff --git a/bob-l2/mkosi.extra/etc/systemd/system/searcher-container.service.d/needs-config.conf b/bob-l2/mkosi.extra/etc/systemd/system/searcher-container.service.d/needs-config.conf
@@ -0,0 +1,6 @@
+[Unit]
+After=fetch-config.service
+Requires=fetch-config.service
+
+[Service]
+EnvironmentFile=/etc/bob/config.env
diff --git a/bob-l2/mkosi.extra/etc/systemd/system/searcher-firewall.service.d/needs-config.conf b/bob-l2/mkosi.extra/etc/systemd/system/searcher-firewall.service.d/needs-config.conf
@@ -0,0 +1,6 @@
+[Unit]
+After=fetch-config.service
+Requires=fetch-config.service
+
+[Service]
+EnvironmentFile=/etc/bob/config.env
diff --git a/bob-l2/mkosi.extra/etc/tdx-init/disk-glob b/bob-l2/mkosi.extra/etc/tdx-init/disk-glob
@@ -0,0 +1,2 @@
+/dev/disk/by-path/*nvme-2
+/dev/disk/by-path/*:10
diff --git a/bob-l2/mkosi.extra/usr/bin/fetch-config.sh b/bob-l2/mkosi.extra/usr/bin/fetch-config.sh
@@ -0,0 +1,95 @@
+#!/bin/sh
+set -eu -o pipefail
+
+# This script fetches couple of pre-defined keys from Vault
+# and writes them to /etc/bob/config.env as:
+#   CONFIG_{KEY}='{VALUE}'
+
+CONFIG_PATH=/etc/bob/config.env
+
+if dmidecode -s system-manufacturer 2>/dev/null | grep -q "QEMU"; then
+    echo "Running in local QEMU, using hardcoded metadata values"
+
+    cat <<EOF >> "$CONFIG_PATH"
+CONFIG_NETWORK_ID='12345'
+CONFIG_NETWORK_NAME='local-testnet'
+CONFIG_JWT_SECRET='1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'
+CONFIG_EL_STATIC_PEERS='enode://[email protected]:30303'
+CONFIG_EL_PEERS_IPS='192.168.1.10'
+CONFIG_SIMULATOR_RPC_URL='http://192.168.1.100:8545'
+CONFIG_SIMULATOR_WS_URL='ws://192.168.1.100:8546'
+CONFIG_SIMULATOR_IP='192.168.1.100'
+EOF
+
+    # Ideally, this logic should be somewhere else, but it's fine for now
+    chattr -i /etc/resolv.conf || true
+    echo "nameserver 1.1.1.1" > /etc/resolv.conf
+
+    exit 0
+fi
+
+fetch_metadata_value() {
+    curl -s \
+        --header "Metadata-Flavor: Google" \
+        "http://metadata/computeMetadata/v1/instance/attributes/$1"
+}
+
+instance_name=$(fetch_metadata_value "name")
+vault_addr=$(fetch_metadata_value "vault_addr")
+vault_auth_mount=$(fetch_metadata_value "vault_auth_mount_gcp")
+vault_kv_path=$(fetch_metadata_value "vault_kv_path")
+vault_kv_common_suffix=$(fetch_metadata_value "vault_kv_common_suffix")
+
+gcp_token=$(curl \
+  --header "Metadata-Flavor: Google" \
+  --data-urlencode "audience=http://vault/$instance_name" \
+  --data-urlencode "format=full" \
+  "http://metadata/computeMetadata/v1/instance/service-accounts/default/identity")
+
+vault_token=$(curl \
+    --data "$(printf '{"role":"%s","jwt":"%s"}' "$instance_name" "$gcp_token")" \
+    "${vault_addr}/v1/${vault_auth_mount}/login" | \
+    jq -r .auth.client_token)
+
+common_data=$(curl \
+    --header "X-Vault-Token: ${vault_token}" \
+    "${vault_addr}/v1/${vault_kv_path}/node/${vault_kv_common_suffix}" |
+    jq -c .data.data)
+secret_data=$(curl \
+    --header "X-Vault-Token: ${vault_token}" \
+    "${vault_addr}/v1/${vault_kv_path}/node/${instance_name}" |
+    jq -c .data.data)
+
+# merge objects
+data=$(echo "$common_data $secret_data" | jq -s 'add')
+
+get_data_value() {
+    echo "$data" | jq -rc --arg key "$1" '.[$key]'
+}
+
+get_ips_from_uris() {
+    # eh, good enough for our usecase
+    echo "$1" | grep -oE '[0-9]{1,3}(\.[0-9]{1,3}){3}'
+}
+
+network_id=$(get_data_value network_id)
+network_name=$(get_data_value network_name)
+jwt_secret=$(get_data_value jwt_secret)
+
+el_static_peers=$(get_data_value el_static_peers | jq -r 'join(",")')
+el_peers_ips=$(get_ips_from_uris "$el_static_peers" | tr '\n' ',' | sed 's/,$//')
+
+simulator_rpc_url=$(get_data_value simulator_rpc_url)
+simulator_ws_url=$(get_data_value simulator_ws_url)
+simulator_ip=$(get_ips_from_uris "$simulator_rpc_url")
+
+cat <<EOF >> "$CONFIG_PATH"
+CONFIG_NETWORK_ID='${network_id}'
+CONFIG_NETWORK_NAME='${network_name}'
+CONFIG_JWT_SECRET='${jwt_secret}'
+CONFIG_EL_STATIC_PEERS='${el_static_peers}'
+CONFIG_EL_PEERS_IPS='${el_peers_ips}'
+CONFIG_SIMULATOR_RPC_URL='${simulator_rpc_url}'
+CONFIG_SIMULATOR_WS_URL='${simulator_ws_url}'
+CONFIG_SIMULATOR_IP='${simulator_ip}'
+EOF
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		/dev/disk/by-path/*nvme-2
		/dev/disk/by-path/*:10