Implement L2 BoB

DEVELOPMENT.md

@@ -27,6 +27,7 @@ flashboxes/
 │   └── debloat*.sh         # System cleanup scripts
 ├── bob-common/             # TEE Searcher common image
 ├── bob-l1/                 # L1 TEE Searcher sandbox image
+├── bob-l2/                 # L2 TEE Searcher sandbox image
 ├── buildernet/             # BuilderNet
 ├── tdx-dummy/              # TDX test environment
 ├── kernel/                 # Kernel configuration

Makefile

@@ -39,11 +39,11 @@ setup: ## Install dependencies (Linux only)
 
 # Build module
 build: check-perms setup ## Build the specified module
-	$(WRAPPER) mkosi --force -I $(IMAGE).conf
+	$(WRAPPER) mkosi --force --include=$(IMAGE).conf
 
 # Build module with devtools profile
 build-dev: check-perms setup ## Build module with development tools
-	$(WRAPPER) mkosi --force --profile=devtools -I $(IMAGE).conf
+	$(WRAPPER) mkosi --force --profile=devtools --include=$(IMAGE).conf
 
 ##@ Utilities
 

bob-common/mkosi.build

@@ -35,7 +35,7 @@ make_git_package \
 # Build tdx-init
 make_git_package \
     "tdx-init" \
-    "v0.1.1" \
+    "ilya/luks-resize-on-unlock" \
     "https://github.com/flashbots/tdx-init" \
     'go build -trimpath -ldflags "-s -w -buildid=" -o ./build/tdx-init' \
     "build/tdx-init:/usr/bin/tdx-init"

bob-common/mkosi.extra/usr/bin/init-firewall.sh

@@ -124,6 +124,18 @@ accept_dst_ip_port() {
                          -m comment --comment "$comment"
 }
 
+accept_src_ip_dst_port() {
+    chain="$1"
+    protocol="$2"
+    ip="$3"
+    port="$4"
+    comment="$5"
+
+    iptables -A "$chain" -p "$protocol" -s "$ip" --dport "$port" \
+                         -m conntrack --ctstate NEW -j ACCEPT \
+                         -m comment --comment "$comment"
+}
+
 drop_dst_ip() {
     chain="$1"
     ip="$2"

bob-l2.conf

@@ -0,0 +1,13 @@
+[Include]
+Include=base/mkosi.conf
+Include=bob-common/mkosi.conf
+Include=bob-l2/mkosi.conf
+
+[Config]
+Profiles=gcp
+
+[Distribution]
+Mirror=https://snapshot.debian.org/archive/debian/20250526T142542Z/
+
+[Build]
+ToolsTreeMirror=https://snapshot.debian.org/archive/debian/20250526T142542Z/

bob-l2/kernel.config

@@ -0,0 +1,45 @@
+CONFIG_NET_VENDOR_GOOGLE=y
+CONFIG_GVE=y
+
+# Enable iptables interface for CONFIG_NF_TABLES
+# Same config as in bob-l1
+CONFIG_IPV6=n
+CONFIG_NETFILTER_NETLINK=y
+CONFIG_NETFILTER_NETLINK_LOG=y
+CONFIG_NF_CONNTRACK_MARK=y
+CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NF_CT_PROTO_SCTP=y
+CONFIG_NF_CT_PROTO_UDPLITE=y
+CONFIG_NF_CT_NETLINK=y
+CONFIG_NF_NAT_NEEDED=y
+CONFIG_NF_TABLES=y
+CONFIG_NF_TABLES_INET=y
+CONFIG_NF_TABLES_IPV4=y
+CONFIG_NF_TABLES_BRIDGE=y
+CONFIG_NF_TABLES_ARP=y
+CONFIG_NF_TABLES_NETDEV=y
+CONFIG_NETFILTER_XTABLES_COMPAT=y
+CONFIG_NFT_CT=y
+CONFIG_NFT_COUNTER=y
+CONFIG_NFT_LOG=y
+CONFIG_NFT_LIMIT=y
+CONFIG_NFT_MASQ=y
+CONFIG_NFT_REJECT=y
+CONFIG_NFT_REJECT_INET=y
+CONFIG_NFT_COMPAT=y
+CONFIG_NFT_NAT=y
+CONFIG_NFT_REDIR=y
+CONFIG_NFT_OBJREF=y
+CONFIG_NETFILTER_XT_TARGET_LOG=y
+CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y
+CONFIG_NETFILTER_XT_MATCH_STATE=y
+CONFIG_IP_NF_TARGET_REJECT=y
+CONFIG_IP_NF_TARGET_NETMAP=y
+CONFIG_IP_NF_TARGET_REDIRECT=y
+CONFIG_IP_NF_MANGLE=y
+CONFIG_IP_NF_RAW=y
+CONFIG_NET_SCHED=y
+CONFIG_CRYPTO_USER_API_HASH=y
+CONFIG_CRYPTO_USER_API_SKCIPHER=y
+CONFIG_CRYPTO_USER_API_RNG=y
+CONFIG_CRYPTO_USER_API_AEAD=y

bob-l2/mkosi.conf

@@ -0,0 +1,10 @@
+[Build]
+Environment=KERNEL_CONFIG_SNIPPETS=kernel/snippets/ubuntu.config,bob-l2/kernel.config
+WithNetwork=true
+
+[Content]
+ExtraTrees=bob-l2/mkosi.extra
+PostInstallationScripts=bob-l2/mkosi.postinst
+
+Packages=chrony
+         dmidecode

bob-l2/mkosi.extra/etc/bob/firewall-config

@@ -0,0 +1,81 @@
+# This script is sourced from firewall script and contains image-specific rules
+# See also: bob-common/mkosi.extra/usr/bin/init-firewall.sh
+
+# Image-specific ports
+SSH_CONTROL_PORT=22
+SSH_DATA_PORT=10022
+SSH_REGISTER_PORT=8080
+CVM_REVERSE_PROXY_PORT=8745
+SEARCHER_INPUT_PORT=27017
+
+# Well-known ports
+DNS_PORT=53
+HTTP_PORT=80
+HTTPS_PORT=443
+NTP_PORT=123
+OP_NODE_P2P_PORT=9222
+OP_GETH_P2P_PORT=30303
+ENGINE_API_PORT=8651
+
+###########################################################################
+# (1) ALWAYS_IN: Inbound rules that are always applied
+###########################################################################
+
+accept_dst_port $CHAIN_ALWAYS_IN tcp $SSH_CONTROL_PORT "SSH control port"
+accept_dst_port $CHAIN_ALWAYS_IN udp $SEARCHER_INPUT_PORT "Searcher input channel"
+
+# We drive op-geth in the searcher container from external op-node
+accept_src_ip_dst_port $CHAIN_ALWAYS_IN tcp "$METADATA_BOB_L2_OP_NODE_CIDR" $ENGINE_API_PORT "Engine API"
+
+# CVM reverse-proxy serves server attestation
+# Also forwards request to ssh pubkey server on localhost:5001,
+#    which serves searcher-container openssh server pubkey
+accept_dst_port $CHAIN_ALWAYS_IN tcp $CVM_REVERSE_PROXY_PORT "CVM reverse-proxy"
+
+###########################################################################
+# (2) ALWAYS_OUT: Outbound rules that are always applied
+###########################################################################
+
+# Note: this is accessible only from host, searcher netns has DROP on those
+# See also init-container.sh
+accept_dst_port $CHAIN_ALWAYS_OUT udp $NTP_PORT "NTP"
+
+accept_dst_ip_port $CHAIN_ALWAYS_OUT tcp "$METADATA_BOB_L2_BACKRUNS_IP" $HTTP_PORT "bundle"
+
+###########################################################################
+# (3) MAINTENANCE_IN: Inbound rules for Maintenance Mode
+###########################################################################
+
+accept_dst_port $CHAIN_MAINTENANCE_IN tcp $SSH_DATA_PORT "SSH data plane"
+accept_dst_port $CHAIN_MAINTENANCE_IN tcp $SSH_REGISTER_PORT "SSH register service"
+
+accept_dst_port $CHAIN_MAINTENANCE_IN tcp $OP_GETH_P2P_PORT "op-geth P2P (TCP)"
+accept_dst_port $CHAIN_MAINTENANCE_IN udp $OP_GETH_P2P_PORT "op-geth P2P (UDP)"
+
+###########################################################################
+# (4) MAINTENANCE_OUT: Outbound rules for Maintenance Mode
+###########################################################################
+
+# Block tx endpoint during maintenance
+drop_dst_ip $CHAIN_MAINTENANCE_OUT "$METADATA_BOB_L2_TX_STREAM_IP" "tx stream (DROP before accept-all rules)"
+
+accept_dst_port $CHAIN_MAINTENANCE_OUT udp $DNS_PORT "DNS (UDP)"
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $DNS_PORT "DNS (TCP)"
+
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $HTTP_PORT "HTTP"
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $HTTPS_PORT "HTTPS"
+
+accept_dst_port $CHAIN_MAINTENANCE_OUT tcp $OP_GETH_P2P_PORT "op-geth P2P (TCP)"
+accept_dst_port $CHAIN_MAINTENANCE_OUT udp $OP_GETH_P2P_PORT "op-geth P2P (UDP)"
+
+###########################################################################
+# (5) PRODUCTION_IN: Inbound rules for Production Mode
+###########################################################################
+
+# None at the moment
+
+###########################################################################
+# (6) PRODUCTION_OUT: Outbound rules for Production Mode
+###########################################################################
+
+accept_dst_ip_port $CHAIN_PRODUCTION_OUT tcp "$METADATA_BOB_L2_TX_STREAM_IP" $HTTP_PORT "tx stream"

bob-l2/mkosi.extra/etc/bob/searcher-container-after-init

@@ -0,0 +1,8 @@
+# This script is sourced from init-container.sh and contains image-specific stuff
+# See also: bob-common/mkosi.extra/usr/bin/init-container.sh
+
+exec_in_container "
+    cat <<EOF >> /etc/hosts
+$METADATA_BOB_L2_TX_STREAM_IP tx-stream.internal
+$METADATA_BOB_L2_BACKRUNS_IP  backruns.internal
+EOF"

bob-l2/mkosi.extra/etc/bob/searcher-container-before-init

@@ -0,0 +1,9 @@
+# This script is sourced from init-container.sh and contains image-specific stuff
+# See also: bob-common/mkosi.extra/usr/bin/init-container.sh
+
+ENGINE_API_PORT=8651
+
+BOB_SEARCHER_EXTRA_PODMAN_FLAGS="\
+    -p ${ENGINE_API_PORT}:${ENGINE_API_PORT} \
+    -v /etc/metadata.env:/etc/metadata.env:ro \
+"

bob-l2/mkosi.extra/etc/searcher-container-init-extra

@@ -0,0 +1,7 @@
+# This script is sourced from init-container.sh and contains image-specific stuff
+# See also: bob-common/mkosi.extra/usr/bin/init-container.sh
+
+echo "Injecting static hosts into searcher container..."
+exec_in_container '
+    cat <<EOF >> /etc/hosts
+EOF'

bob-l2/mkosi.extra/etc/systemd/system/fetch-metadata.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Fetch some environment variables from metadata service
+After=network.target network-setup.service
+Requires=network-setup.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/fetch-metadata.sh
+RemainAfterExit=yes
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=minimal.target

bob-l2/mkosi.extra/etc/systemd/system/searcher-container.service.d/needs-metadata.conf

@@ -0,0 +1,6 @@
+[Unit]
+After=fetch-metadata.service
+Requires=fetch-metadata.service
+
+[Service]
+EnvironmentFile=/etc/metadata.env

bob-l2/mkosi.extra/etc/systemd/system/searcher-firewall.service.d/needs-metadata.conf

@@ -0,0 +1,6 @@
+[Unit]
+After=fetch-metadata.service
+Requires=fetch-metadata.service
+
+[Service]
+EnvironmentFile=/etc/metadata.env

bob-l2/mkosi.extra/etc/tdx-init/disk-glob

@@ -0,0 +1,2 @@
+/dev/disk/by-path/*nvme-2
+/dev/disk/by-path/*:10

bob-l2/mkosi.extra/usr/bin/fetch-metadata.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+set -eu -o pipefail
+
+# This script fetches couple of pre-defined keys from instance metadata server
+# and writes them to /etc/metadata.env in the format:
+#   METADATA_{KEY}='{VALUE}'
+#
+# It also checks that received values do not contain newlines and conform to
+# ^[a-zA-Z0-9.,@:/_ -]*$
+
+if dmidecode -s system-manufacturer 2>/dev/null | grep -q "QEMU"; then
+    echo "Running in local QEMU, using hardcoded metadata values"
+
+    cat <<EOF >> /etc/metadata.env
+METADATA_BOB_L2_BACKRUNS_IP='1.1.1.1'
+METADATA_BOB_L2_TX_STREAM_IP='1.0.0.1'
+METADATA_BOB_L2_OP_NODE_CIDR='10.0.0.0/8'
+EOF
+
+    # Ideally, this logic should be somewhere else, but it's fine for now
+    chattr -i /etc/resolv.conf || true
+    echo "nameserver 1.1.1.1" > /etc/resolv.conf
+
+    exit 0
+fi
+
+rm -f /etc/metadata.env # just in case
+touch /etc/metadata.env
+
+METADATA_URL="http://169.254.169.254/computeMetadata/v1/instance/attributes/"
+fetch_metadata_value() {
+    local key="$1"
+    curl -s -H "Metadata-Flavor: Google" "${METADATA_URL}${key}"
+}
+
+for key in \
+    BOB_L2_BACKRUNS_IP \
+    BOB_L2_TX_STREAM_IP \
+    BOB_L2_OP_NODE_CIDR
+do
+    value=$(fetch_metadata_value "$key")
+
+    if [ "$(echo "$value" | wc -l)" -ne 1 ]; then
+        echo "Error: Value for $key contains newlines"
+        exit 1
+    fi
+
+    if echo "$value" | grep -qvE '^[a-zA-Z0-9.,@:/_ -]*$'; then
+        echo "Error: Value for $key contains bad characters"
+        exit 1
+    fi
+
+    echo "METADATA_$key='$value'" >> /etc/metadata.env
+done

bob-l2/mkosi.postinst

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -euxo pipefail
+
+# Install chrony config
+mkdir -p "$BUILDROOT/etc/chrony/"
+install -m 644 services/chrony.conf "$BUILDROOT/etc/chrony/"
+
+# Enable services
+mkdir -p "$BUILDROOT/etc/systemd/system/minimal.target.wants"
+
+for service in \
+    chrony.service
+do
+    mkosi-chroot systemctl enable "$service"
+    ln -sf "/etc/systemd/system/$service" "$BUILDROOT/etc/systemd/system/minimal.target.wants/"
+done

mkosi.profiles/gcp/mkosi.postoutput

@@ -51,3 +51,11 @@ touch -d "2024-01-01 00:00:00 UTC" "$TMP/disk.raw" 2>/dev/null || true
 tar --format=oldgnu -Sczf "$TAR" -C "$TMP" disk.raw
 
 rm -rf "$TMP"
+
+IMAGE_NAME="bob-l2-poc-ilya-tmp-$(date +%s)"
+GS_PATH="gs://flashbots-artifacts-us-east5/${IMAGE_NAME}.tar.gz"
+
+echo export CLOUDSDK_CORE_PROJECT=flashbots-artifacts
+echo gcloud storage cp ./build/tdx-debian.tar.gz "${GS_PATH}"
+echo gcloud beta compute images create --guest-os-features TDX_CAPABLE,UEFI_COMPATIBLE --source-uri "${GS_PATH}" "${IMAGE_NAME}"
+echo "This will create GCP image: projects/flashbots-artifacts/global/images/${IMAGE_NAME}"