Install and configure Envoy

README.md

@@ -165,6 +165,17 @@ curl http://localhost:10001/recording/download --output recording.mp4
 
 Note: the recording file is encoded into a H.264/MPEG-4 AVC video file. [QuickTime has known issues with playback](https://discussions.apple.com/thread/254851789?sortBy=rank) so please make sure to use a compatible media player!
 
+## Proxy configuration
+
+[Envoy](https://www.envoyproxy.io/) is installed with the browser images, which allows for configuration of a forward proxy chain for egress browser traffic. This proxy is part of the Kernel platform, configured by the following environment variables:
+
+- **INST_NAME**: Instance name in the platform of this browser, used to identify this browser with the platform
+- **METRO_NAME**: Dataplane server name in the platform of this browser, used to identify this browser with the platform
+- **XDS_SERVER**: The xDS server hostname, where envoy can discover configuration
+- **XDS_JWT**: A token used to authenticate this browser to the xDS server
+
+Envoy is running alongside of the browser. The browser may be configured to proxy through envoy using the [--proxy-server flag](https://www.chromium.org/developers/design-documents/network-settings/). The default configuration directly egresses traffic to the internet. When configured to receive dynamic configuration, the xDS server can control the egress traffic flows of the browser, for example through a forward proxy chain.
+
 ## Documentation
 
 This repo powers our managed [browser infrastructure](https://onkernel.com/docs).

images/chromium-headful/.gitignore

@@ -3,3 +3,4 @@ recording/
 .tmp/
 .rootfs/
 initrd
+temp.sh

images/chromium-headful/Dockerfile

@@ -126,7 +126,8 @@ RUN set -eux; \
     libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx7 \
     gstreamer1.0-plugins-base gstreamer1.0-plugins-good \
     gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly \
-    gstreamer1.0-pulseaudio gstreamer1.0-omx; \
+    gstreamer1.0-pulseaudio gstreamer1.0-omx \
+    libnss3-tools; \
     #
     # install libxcvt0 (not available in debian:bullseye)
     ARCH=$(dpkg --print-architecture); \
@@ -168,12 +169,24 @@ COPY --from=client /src/dist/ /var/www
 COPY --from=xorg-deps /usr/local/lib/xorg/modules/drivers/dummy_drv.so /usr/lib/xorg/modules/drivers/dummy_drv.so
 COPY --from=xorg-deps /usr/local/lib/xorg/modules/input/neko_drv.so /usr/lib/xorg/modules/input/neko_drv.so
 
+# Install Envoy proxy and BrightData certificates
+COPY shared/envoy/install-proxy.sh /usr/local/bin/install-proxy.sh
+RUN chmod +x /usr/local/bin/install-proxy.sh && /usr/local/bin/install-proxy.sh && rm /usr/local/bin/install-proxy.sh
+
+# Copy Envoy configuration files
+COPY shared/envoy/bootstrap.yaml /etc/envoy/templates/bootstrap.yaml
+# Copy default config to bootstrap.yaml so supervisor can start envoy immediately
+COPY shared/envoy/default.yaml /etc/envoy/bootstrap.yaml
+COPY shared/envoy/init-envoy.sh /usr/local/bin/init-envoy.sh
+RUN chmod +x /usr/local/bin/init-envoy.sh
+
 COPY images/chromium-headful/image-chromium/ /
 COPY images/chromium-headful/start-chromium.sh /images/chromium-headful/start-chromium.sh
 RUN chmod +x /images/chromium-headful/start-chromium.sh
 COPY images/chromium-headful/wrapper.sh /wrapper.sh
 COPY images/chromium-headful/supervisord.conf /etc/supervisor/supervisord.conf
 COPY images/chromium-headful/supervisor/services/ /etc/supervisor/conf.d/services/
+COPY shared/envoy/supervisor-envoy.conf /etc/supervisor/conf.d/services/envoy.conf
 
 # copy the kernel-images API binary built in the builder stage
 COPY --from=server-builder /out/kernel-images-api /usr/local/bin/kernel-images-api

images/chromium-headful/run-docker.sh

@@ -18,6 +18,9 @@ CHROMIUM_FLAGS_DEFAULT="--user-data-dir=/home/kernel/user-data --disable-dev-shm
 if [[ "$RUN_AS_ROOT" == "true" ]]; then
   CHROMIUM_FLAGS_DEFAULT="$CHROMIUM_FLAGS_DEFAULT --no-sandbox --no-zygote"
 fi
+if [[ -n "${XDS_JWT:-}" ]]; then
+  CHROMIUM_FLAGS_DEFAULT="$CHROMIUM_FLAGS_DEFAULT --proxy-server=https://127.0.0.1:3128"
+fi
 CHROMIUM_FLAGS="${CHROMIUM_FLAGS:-$CHROMIUM_FLAGS_DEFAULT}"
 rm -rf .tmp/chromium
 mkdir -p .tmp/chromium
@@ -43,6 +46,22 @@ RUN_ARGS=(
   --mount type=bind,src="$FLAGS_FILE",dst=/chromium/flags,ro
 )
 
+# Add XDS environment variables if provided
+if [[ -n "${INST_NAME:-}" ]]; then
+  RUN_ARGS+=( -e "INST_NAME=$INST_NAME" )
+fi
+if [[ -n "${METRO_NAME:-}" ]]; then
+  RUN_ARGS+=( -e "METRO_NAME=$METRO_NAME" )
+fi
+if [[ -n "${XDS_SERVER:-}" ]]; then
+  RUN_ARGS+=( -e "XDS_SERVER=$XDS_SERVER" )
+fi
+if [[ -n "${XDS_JWT:-}" ]]; then
+  RUN_ARGS+=( -e "XDS_JWT=$XDS_JWT" )
+  RUN_ARGS+=( -p 9901:9901 )
+  RUN_ARGS+=( -p 3128:3128 )
+fi
+
 # WebRTC port mapping
 if [[ "${ENABLE_WEBRTC:-}" == "true" ]]; then
   echo "Running container with WebRTC"

images/chromium-headful/run-unikernel.sh

@@ -54,6 +54,20 @@ deploy_args=(
   -n "$NAME"
 )
 
+# Add XDS environment variables if provided
+if [[ -n "${INST_NAME:-}" ]]; then
+  deploy_args+=(-e "INST_NAME=$INST_NAME")
+fi
+if [[ -n "${METRO_NAME:-}" ]]; then
+  deploy_args+=(-e "METRO_NAME=$METRO_NAME")
+fi
+if [[ -n "${XDS_SERVER:-}" ]]; then
+  deploy_args+=(-e "XDS_SERVER=$XDS_SERVER")
+fi
+if [[ -n "${XDS_JWT:-}" ]]; then
+  deploy_args+=(-e "XDS_JWT=$XDS_JWT")
+fi
+
 if [[ "${ENABLE_WEBRTC:-}" == "true" ]]; then
   echo "Deploying with WebRTC enabled"
   kraft cloud inst create --start \

images/chromium-headful/wrapper.sh

@@ -148,6 +148,8 @@ fi
 sleep 0.2
 done
 
+init-envoy.sh
+
 echo "[wrapper] Starting Xorg via supervisord"
 supervisorctl -c /etc/supervisor/supervisord.conf start xorg
 echo "[wrapper] Waiting for Xorg to open display $DISPLAY..."
@@ -193,6 +195,7 @@ supervisorctl -c /etc/supervisor/supervisord.conf start chromium
 echo "[wrapper] Waiting for Chromium remote debugging on 127.0.0.1:$INTERNAL_PORT..."
 for i in {1..100}; do
   if nc -z 127.0.0.1 "$INTERNAL_PORT" 2>/dev/null; then
+    echo "connected to chrome debugging port."
     break
   fi
   sleep 0.2

images/chromium-headless/image/Dockerfile

@@ -48,7 +48,20 @@ RUN set -xe; \
     xvfb \
     x11-utils \
     software-properties-common \
-    supervisor;
+    supervisor \
+    libnss3-tools \
+    unzip;
+
+# Install Envoy proxy and BrightData certificates
+COPY shared/envoy/install-proxy.sh /usr/local/bin/install-proxy.sh
+RUN chmod +x /usr/local/bin/install-proxy.sh && /usr/local/bin/install-proxy.sh && rm /usr/local/bin/install-proxy.sh
+
+# Copy Envoy configuration files
+COPY shared/envoy/bootstrap.yaml /etc/envoy/templates/bootstrap.yaml
+# Copy default config to bootstrap.yaml so supervisor can start envoy immediately
+COPY shared/envoy/default.yaml /etc/envoy/bootstrap.yaml
+COPY shared/envoy/init-envoy.sh /usr/local/bin/init-envoy.sh
+RUN chmod +x /usr/local/bin/init-envoy.sh
 
 # install chromium and sqlite3 for debugging the cookies file
 RUN add-apt-repository -y ppa:xtradeb/apps
@@ -83,6 +96,7 @@ COPY images/chromium-headless/image/wrapper.sh /usr/bin/wrapper.sh
 # Supervisord configuration
 COPY images/chromium-headless/image/supervisord.conf /etc/supervisor/supervisord.conf
 COPY images/chromium-headless/image/supervisor/services/ /etc/supervisor/conf.d/services/
+COPY shared/envoy/supervisor-envoy.conf /etc/supervisor/conf.d/services/envoy.conf
 
 # Copy the kernel-images API binary built in the builder stage
 COPY --from=server-builder /out/kernel-images-api /usr/local/bin/kernel-images-api

images/chromium-headless/image/wrapper.sh

@@ -194,6 +194,8 @@ for i in {1..30}; do
   sleep 0.2
 done
 
+init-envoy.sh
+
 echo "[wrapper] Starting system D-Bus daemon via supervisord"
 supervisorctl -c /etc/supervisor/supervisord.conf start dbus
 for i in {1..50}; do

shared/envoy/bootstrap.yaml

@@ -0,0 +1,66 @@
+# Envoy bootstrap configuration for xDS-managed proxy
+# This config connects to a control plane for dynamic configuration management
+# Requires: INST_NAME, METRO_NAME, XDS_SERVER, XDS_JWT environment variables
+
+# Node identity sent to xDS server for configuration targeting, authenticated by JWT
+node:
+  id: "{INST_NAME}-{METRO_NAME}"
+  cluster: "kernel"
+
+# Dynamic configuration via xDS protocol
+dynamic_resources:
+  # Aggregated Discovery Service - single gRPC stream for all config types
+  ads_config:
+    api_type: GRPC
+    transport_api_version: V3
+    grpc_services:
+    - envoy_grpc:
+        # Reference to xDS server cluster below
+        cluster_name: xds_server
+        authority: "{XDS_SERVER}"
+      # Send JWT authentication for all xDS requests
+      initial_metadata:
+      - key: "authorization"
+        value: "Bearer {XDS_JWT}"
+
+  # Listener Discovery Service and Cluster Discovery Service use ADS
+  lds_config:
+    ads: {}
+    resource_api_version: V3
+  cds_config:
+    ads: {}
+    resource_api_version: V3
+
+# Static configuration (always present)
+static_resources:
+  clusters:
+  # xDS server: control plane for configuration
+  - name: xds_server
+    # Resolve hostname via DNS, for DNS lookup
+    type: STRICT_DNS
+    connect_timeout: 2s
+    http2_protocol_options: {}
+    dns_lookup_family: V4_ONLY
+    # TLS configuration for secure xDS connection
+    transport_socket:
+      name: envoy.transport_sockets.tls
+      typed_config:
+        # Uses TLS to verify xDS server, and SNI hostname for TLS handshake
+        "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+        sni: {XDS_SERVER}
+    load_assignment:
+      cluster_name: xds_server
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: {XDS_SERVER}
+                port_value: 443
+
+# Enable Envoy admin interface for debugging locally
+# admin:
+#   address:
+#     socket_address:
+#       address: 0.0.0.0
+#       port_value: 9901

shared/envoy/default.yaml

@@ -0,0 +1,98 @@
+# HTTP forward proxy, direct egress to internet
+static_resources:
+  listeners:
+    - name: http_explicit_forward_proxy
+      address:
+        # e.g. on chromium, set --proxy-server=http://127.0.0.1:3128
+        socket_address:
+          address: 0.0.0.0
+          port_value: 3128
+      filter_chains:
+        # One filter chain for HTTP/1.1 proxy traffic
+        - filters:
+            # HTTP Connection Manager filter:
+            # this is handling the connection between the client and the proxy,
+            # which is an HTTP connection.
+            - name: envoy.filters.network.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                stat_prefix: hcm
+                normalize_path: true
+                # Enable forward proxy behavior
+                http_filters:
+                  # Dynamic Forward Proxy filter: resolves upstreams on-the-fly and caches DNS
+                  # Browsers send CONNECT to this proxy to tunnel HTTPS upstreams
+                  # e.g. CONNECT target.example.com
+                  # Envoy establishes a TCP tunnel to the target.
+                  - name: envoy.filters.http.dynamic_forward_proxy
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig
+                      dns_cache_config:
+                        name: local_dns_cache
+                        dns_lookup_family: V4_ONLY
+                  # Router filter: actually routes/tunnels the request once target is known
+                  # See what happens here down in route_config, slightly different
+                  # for HTTPS vs HTTP upstream requests.
+                  - name: envoy.filters.http.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+                route_config:
+                  name: local_route
+                  virtual_hosts:
+                    - name: forward_proxy
+                      domains: ["*"]
+                      routes:
+                        # === HTTPS upstream traffic (CONNECT tunnel) ===
+                        # Handle CONNECT method for HTTPS tunneling (creates TCP tunnel)
+                        # 'connect_matcher' is a special matcher that matches CONNECT requests
+                        # "Note that CONNECT support is currently considered alpha in Envoy."
+                        # https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-field-config-route-v3-routematch-connect-matcher
+                        - match: { connect_matcher: {} }
+                          route:
+                            # Use single DFP cluster; CONNECT handled via connect_config
+                            cluster: dynamic_forward_proxy_cluster
+                            # This tells Envoy to upgrade the connection to a TCP tunnel
+                            # which we only do after getting the CONNECT request.
+                            upgrade_configs:
+                              - upgrade_type: CONNECT
+                                connect_config: {}
+                        # === HTTP upstream traffic (absolute-form proxy) ===
+                        # The client didn't send CONNECT, because it's an HTTP request.
+                        - match: { prefix: "/" }
+                          route:
+                            # Same cluster for HTTP proxying
+                            cluster: dynamic_forward_proxy_cluster
+                access_log:
+                  # Access log sink: print one line per request to stdout
+                  - name: envoy.access_loggers.stdout
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
+                      log_format:
+                        text_format: "[%START_TIME%] %DOWNSTREAM_REMOTE_ADDRESS% %REQ(:method)% %REQ(:authority)% %REQ(:path)% -> %RESPONSE_CODE% (%BYTES_SENT%b) %DURATION%ms %RESPONSE_FLAGS% %UPSTREAM_TRANSPORT_FAILURE_REASON%\n"
+
+  # Connection pooling / load balancing to target(s)
+  clusters:
+    - name: dynamic_forward_proxy_cluster
+      connect_timeout: 5s
+      lb_policy: CLUSTER_PROVIDED
+      typed_extension_protocol_options:
+        envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+          "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+          explicit_http_config:
+            http_protocol_options: {}
+          upstream_http_protocol_options:
+            auto_sni: true
+            auto_san_validation: true
+      cluster_type:
+        name: envoy.clusters.dynamic_forward_proxy
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig
+          dns_cache_config:
+            name: local_dns_cache
+            dns_lookup_family: V4_ONLY
+
+# Admin interface for debugging and monitoring
+admin:
+  address:
+    # Admin interface (metrics, config dump, clusters, listeners). Not exposed publicly.
+    socket_address: { address: 127.0.0.1, port_value: 9901 }