feat(rebalancer): support RPC overrides

typescript/infra/helm/rebalancer/templates/env-var-external-secret.yaml

@@ -20,15 +20,31 @@ spec:
         annotations:
           update-on-redeploy: "{{ now }}"
       data:
-        GCP_SECRET_OVERRIDES_ENABLED: "true"
         COINGECKO_API_KEY: {{ printf "'{{ .%s_coingecko_api_key | toString }}'" .Values.hyperlane.runEnv }}
-        # Extract only the privateKey field from the JSON 
+        # Extract only the privateKey field from the JSON
         REBALANCER_KEY: {{ print "'{{ $json := .rebalancer_key | fromJson }}{{ $json.privateKey }}'" }}
+{{/*
+   * For each network, create an RPC_URL_<CHAIN> environment variable.
+   * The rebalancer service reads these to override registry RPC URLs.
+   * Uses the first URL from the JSON array stored in GCP Secret Manager.
+   */}}
+        {{- range .Values.hyperlane.chains }}
+        RPC_URL_{{ . | upper | replace "-" "_" }}: {{ printf "'{{ index (.%s_rpcs | fromJson) 0 }}'" . }}
+        {{- end }}
   data:
   - secretKey: {{ printf "%s_coingecko_api_key" .Values.hyperlane.runEnv }}
     remoteRef:
       key: {{ printf "%s-coingecko-api-key" .Values.hyperlane.runEnv }}
   - secretKey: rebalancer_key
     remoteRef:
       key: {{ printf "hyperlane-%s-key-rebalancer" .Values.hyperlane.runEnv }}
+{{/*
+   * For each network, load the secret in GCP secret manager with the form: environment-rpc-endpoint-network,
+   * and associate it with the secret key networkname_rpcs.
+   */}}
+  {{- range .Values.hyperlane.chains }}
+  - secretKey: {{ printf "%s_rpcs" . }}
+    remoteRef:
+      key: {{ printf "%s-rpc-endpoints-%s" $.Values.hyperlane.runEnv . }}
+  {{- end }}
 

typescript/infra/helm/rebalancer/values.yaml

@@ -7,6 +7,8 @@ hyperlane:
   registryUri: ''
   rebalancerConfig: ''
   withMetrics: true
+  # Used for fetching private RPC secrets
+  chains: []
 nameOverride: ''
 fullnameOverride: ''
 externalSecrets:

typescript/infra/src/rebalancer/helm.ts

@@ -20,7 +20,7 @@ import {
   getHelmReleaseName,
   removeHelmRelease,
 } from '../utils/helm.js';
-import { getInfraPath } from '../utils/utils.js';
+import { execCmdAndParseJson, getInfraPath } from '../utils/utils.js';
 
 export class RebalancerHelmManager extends HelmManager {
   static helmReleasePrefix: string = 'hyperlane-rebalancer';
@@ -31,6 +31,7 @@ export class RebalancerHelmManager extends HelmManager {
   );
 
   private rebalancerConfigContent: string = '';
+  private rebalancerChains: string[] = [];
 
   constructor(
     readonly warpRouteId: string,
@@ -67,6 +68,12 @@ export class RebalancerHelmManager extends HelmManager {
       throw new Error('No chains configured');
     }
 
+    // Store chains for helm values (used for private RPC secrets)
+    // Warp core config includes all strategy chains
+    this.rebalancerChains = [
+      ...new Set(warpCoreConfig.tokens.map((t) => t.chainName)),
+    ];
+
     // Store the config file content for helm values
     this.rebalancerConfigContent = fs.readFileSync(
       rebalancerConfigFile,
@@ -93,6 +100,8 @@ export class RebalancerHelmManager extends HelmManager {
         registryUri,
         rebalancerConfig: this.rebalancerConfigContent,
         withMetrics: this.withMetrics,
+        // Used for fetching private RPC secrets
+        chains: this.rebalancerChains,
       },
     };
   }
@@ -133,5 +142,111 @@ export class RebalancerHelmManager extends HelmManager {
     }
   }
 
+  /**
+   * Get all deployed rebalancers that include the given chain.
+   * Used by RPC rotation to refresh rebalancer pods when RPCs change.
+   */
+  static async getManagersForChain(
+    environment: DeployEnvironment,
+    chain: string,
+  ): Promise<RebalancerHelmManager[]> {
+    const deployedRebalancers = await getDeployedRebalancerWarpRouteIds(
+      environment,
+      RebalancerHelmManager.helmReleasePrefix,
+    );
+
+    const helmManagers: RebalancerHelmManager[] = [];
+
+    for (const { warpRouteId } of deployedRebalancers) {
+      let warpCoreConfig;
+      try {
+        warpCoreConfig = getWarpCoreConfig(warpRouteId);
+      } catch (e) {
+        continue;
+      }
+
+      const warpChains = warpCoreConfig.tokens.map((t) => t.chainName);
+      if (!warpChains.includes(chain)) {
+        continue;
+      }
+
+      // Create a minimal manager for RPC rotation (only needs helmReleaseName and namespace)
+      helmManagers.push(
+        new RebalancerHelmManager(
+          warpRouteId,
+          environment,
+          '', // registryCommit not needed for refresh
+          '', // rebalancerConfigFile not needed for refresh
+          '', // rebalanceStrategy not needed for refresh
+          false, // withMetrics not needed for refresh
+        ),
+      );
+    }
+
+    return helmManagers;
+  }
+
   // TODO: allow for a rebalancer to be uninstalled
 }
+
+export interface RebalancerPodInfo {
+  helmReleaseName: string;
+  warpRouteId: string;
+}
+
+/**
+ * Get deployed rebalancer warp route IDs by inspecting k8s pods.
+ */
+export async function getDeployedRebalancerWarpRouteIds(
+  namespace: string,
+  helmReleasePrefix: string,
+): Promise<RebalancerPodInfo[]> {
+  const podsResult = await execCmdAndParseJson(
+    `kubectl get pods -n ${namespace} -o json`,
+  );
+
+  const rebalancerPods: RebalancerPodInfo[] = [];
+
+  for (const pod of podsResult.items || []) {
+    const helmReleaseName =
+      pod.metadata?.labels?.['app.kubernetes.io/instance'];
+
+    if (!helmReleaseName?.startsWith(helmReleasePrefix)) {
+      continue;
+    }
+
+    let warpRouteId: string | undefined;
+
+    for (const container of pod.spec?.containers || []) {
+      // Check WARP_ROUTE_ID env var
+      const warpRouteIdEnv = (container.env || []).find(
+        (e: { name: string; value?: string }) => e.name === 'WARP_ROUTE_ID',
+      );
+      if (warpRouteIdEnv?.value) {
+        warpRouteId = warpRouteIdEnv.value;
+        break;
+      }
+
+      // Check --warpRouteId in command or args
+      const allArgs: string[] = [
+        ...(container.command || []),
+        ...(container.args || []),
+      ];
+      const warpRouteIdArgIndex = allArgs.indexOf('--warpRouteId');
+      if (warpRouteIdArgIndex !== -1 && allArgs[warpRouteIdArgIndex + 1]) {
+        warpRouteId = allArgs[warpRouteIdArgIndex + 1];
+        break;
+      }
+    }
+
+    if (warpRouteId) {
+      rebalancerPods.push({ helmReleaseName, warpRouteId });
+    } else {
+      rootLogger.warn(
+        `Could not extract warp route ID from rebalancer pod with helm release: ${helmReleaseName}. Skipping.`,
+      );
+    }
+  }
+
+  return rebalancerPods;
+}

typescript/infra/src/utils/rpcUrls.ts

@@ -21,6 +21,7 @@ import {
 import { DeployEnvironment } from '../config/environment.js';
 import { KeyFunderHelmManager } from '../funding/key-funder.js';
 import { KathyHelmManager } from '../helloworld/kathy.js';
+import { RebalancerHelmManager } from '../rebalancer/helm.js';
 import { WarpRouteMonitorHelmManager } from '../warp-monitor/helm.js';
 
 import { disableGCPSecretVersion } from './gcloud.js';
@@ -273,7 +274,7 @@ async function updateSecretAndDisablePrevious(
 
 /**
  * Interactively refreshes dependent k8s resources for the given chain in the given environment.
- * Prompts for core infrastructure, warp monitors, and CronJobs separately, then executes refreshes.
+ * Prompts for core infrastructure, warp monitors, rebalancers, and CronJobs separately.
  * CronJobs only get secret refresh (no pod restart) - they pick up new secrets on next run.
  * @param environment The environment to refresh resources in
  * @param chain The chain to refresh resources for
@@ -293,10 +294,15 @@ async function refreshDependentK8sResourcesInteractive(
   // Collect selections from all prompts
   const coreManagers = await selectCoreInfrastructure(environment, chain);
   const warpManagers = await selectWarpMonitors(environment, chain);
+  const rebalancerManagers = await selectRebalancers(environment, chain);
   const cronjobManagers = await selectCronJobs(environment);
 
   // Services get both secret and pod refresh
-  const serviceManagers = [...coreManagers, ...warpManagers];
+  const serviceManagers = [
+    ...coreManagers,
+    ...warpManagers,
+    ...rebalancerManagers,
+  ];
   // CronJobs only get secret refresh (they pick up new secrets on next scheduled run)
   const allManagersForSecrets = [...serviceManagers, ...cronjobManagers];
 
@@ -363,7 +369,7 @@ async function selectCoreInfrastructure(
     .filter((_, i) => selection.includes(i));
 }
 
-enum WarpMonitorRefreshChoice {
+enum RefreshChoice {
   ALL = 'all',
   SELECT = 'select',
   SKIP = 'skip',
@@ -393,25 +399,25 @@ async function selectWarpMonitors(
     choices: [
       {
         name: `Yes, refresh all ${warpMonitorManagers.length} monitors`,
-        value: WarpMonitorRefreshChoice.ALL,
+        value: RefreshChoice.ALL,
       },
       {
         name: 'Yes, let me select which ones',
-        value: WarpMonitorRefreshChoice.SELECT,
+        value: RefreshChoice.SELECT,
       },
       {
         name: 'No, skip warp monitors',
-        value: WarpMonitorRefreshChoice.SKIP,
+        value: RefreshChoice.SKIP,
       },
     ],
   });
 
-  if (choice === WarpMonitorRefreshChoice.SKIP) {
+  if (choice === RefreshChoice.SKIP) {
     console.log('Skipping warp monitor refresh');
     return [];
   }
 
-  if (choice === WarpMonitorRefreshChoice.ALL) {
+  if (choice === RefreshChoice.ALL) {
     return warpMonitorManagers;
   }
 
@@ -427,6 +433,66 @@ async function selectWarpMonitors(
   return warpMonitorManagers.filter((_, i) => selection.includes(i));
 }
 
+async function selectRebalancers(
+  environment: DeployEnvironment,
+  chain: string,
+): Promise<RebalancerHelmManager[]> {
+  const rebalancerManagers = await RebalancerHelmManager.getManagersForChain(
+    environment,
+    chain,
+  );
+
+  if (rebalancerManagers.length === 0) {
+    console.log(`No rebalancers found for ${chain}`);
+    return [];
+  }
+
+  console.log(
+    `Found ${rebalancerManagers.length} rebalancers that include ${chain}:`,
+  );
+  for (const manager of rebalancerManagers) {
+    console.log(`  - ${manager.helmReleaseName} (${manager.warpRouteId})`);
+  }
+
+  const choice = await select({
+    message: `Refresh rebalancers?`,
+    choices: [
+      {
+        name: `Yes, refresh all ${rebalancerManagers.length} rebalancers`,
+        value: RefreshChoice.ALL,
+      },
+      {
+        name: 'Yes, let me select which ones',
+        value: RefreshChoice.SELECT,
+      },
+      {
+        name: 'No, skip rebalancers',
+        value: RefreshChoice.SKIP,
+      },
+    ],
+  });
+
+  if (choice === RefreshChoice.SKIP) {
+    console.log('Skipping rebalancer refresh');
+    return [];
+  }
+
+  if (choice === RefreshChoice.ALL) {
+    return rebalancerManagers;
+  }
+
+  const selection = await checkbox({
+    message: 'Select rebalancers to refresh',
+    choices: rebalancerManagers.map((manager, i) => ({
+      name: manager.helmReleaseName,
+      value: i,
+      checked: true,
+    })),
+  });
+
+  return rebalancerManagers.filter((_, i) => selection.includes(i));
+}
+
 async function selectCronJobs(
   environment: DeployEnvironment,
 ): Promise<HelmManager<any>[]> {
@@ -476,12 +542,6 @@ async function selectCronJobs(
     .filter((_, i) => selection.includes(i));
 }
 
-/**
- * Test the provider at the given URL, returning false if the provider is unhealthy
- * or related to a different chain. No-op for non-Ethereum chains.
- * @param chain The chain to test the provider for
- * @param url The URL of the provider
- */
 async function testProvider(chain: ChainName, url: string): Promise<boolean> {
   const chainMetadata = getChain(chain);
   if (chainMetadata.protocol !== ProtocolType.Ethereum) {