diff --git a/typescript/infra/helm/rebalancer/templates/env-var-external-secret.yaml b/typescript/infra/helm/rebalancer/templates/env-var-external-secret.yaml index 71e2a9a5beb..16cddff01c5 100644 --- a/typescript/infra/helm/rebalancer/templates/env-var-external-secret.yaml +++ b/typescript/infra/helm/rebalancer/templates/env-var-external-secret.yaml @@ -20,10 +20,17 @@ spec: annotations: update-on-redeploy: "{{ now }}" data: - GCP_SECRET_OVERRIDES_ENABLED: "true" COINGECKO_API_KEY: {{ printf "'{{ .%s_coingecko_api_key | toString }}'" .Values.hyperlane.runEnv }} - # Extract only the privateKey field from the JSON + # Extract only the privateKey field from the JSON REBALANCER_KEY: {{ print "'{{ $json := .rebalancer_key | fromJson }}{{ $json.privateKey }}'" }} +{{/* + * For each network, create an RPC_URL_ environment variable. + * The rebalancer service reads these to override registry RPC URLs. + * Uses the first URL from the JSON array stored in GCP Secret Manager. + */}} + {{- range .Values.hyperlane.chains }} + RPC_URL_{{ . | upper | replace "-" "_" }}: {{ printf "'{{ index (.%s_rpcs | fromJson) 0 }}'" . }} + {{- end }} data: - secretKey: {{ printf "%s_coingecko_api_key" .Values.hyperlane.runEnv }} remoteRef: @@ -31,4 +38,13 @@ spec: - secretKey: rebalancer_key remoteRef: key: {{ printf "hyperlane-%s-key-rebalancer" .Values.hyperlane.runEnv }} +{{/* + * For each network, load the secret in GCP secret manager with the form: environment-rpc-endpoint-network, + * and associate it with the secret key networkname_rpcs. + */}} + {{- range .Values.hyperlane.chains }} + - secretKey: {{ printf "%s_rpcs" . }} + remoteRef: + key: {{ printf "%s-rpc-endpoints-%s" $.Values.hyperlane.runEnv . }} + {{- end }} diff --git a/typescript/infra/helm/rebalancer/values.yaml b/typescript/infra/helm/rebalancer/values.yaml index 46326345e20..32239f0b6b2 100644 --- a/typescript/infra/helm/rebalancer/values.yaml +++ b/typescript/infra/helm/rebalancer/values.yaml @@ -7,6 +7,8 @@ hyperlane: registryUri: '' rebalancerConfig: '' withMetrics: true + # Used for fetching private RPC secrets + chains: [] nameOverride: '' fullnameOverride: '' externalSecrets: diff --git a/typescript/infra/src/rebalancer/helm.ts b/typescript/infra/src/rebalancer/helm.ts index 269d9138cd0..cf594c4c55d 100644 --- a/typescript/infra/src/rebalancer/helm.ts +++ b/typescript/infra/src/rebalancer/helm.ts @@ -20,7 +20,7 @@ import { getHelmReleaseName, removeHelmRelease, } from '../utils/helm.js'; -import { getInfraPath } from '../utils/utils.js'; +import { execCmdAndParseJson, getInfraPath } from '../utils/utils.js'; export class RebalancerHelmManager extends HelmManager { static helmReleasePrefix: string = 'hyperlane-rebalancer'; @@ -31,6 +31,7 @@ export class RebalancerHelmManager extends HelmManager { ); private rebalancerConfigContent: string = ''; + private rebalancerChains: string[] = []; constructor( readonly warpRouteId: string, @@ -67,6 +68,12 @@ export class RebalancerHelmManager extends HelmManager { throw new Error('No chains configured'); } + // Store chains for helm values (used for private RPC secrets) + // Warp core config includes all strategy chains + this.rebalancerChains = [ + ...new Set(warpCoreConfig.tokens.map((t) => t.chainName)), + ]; + // Store the config file content for helm values this.rebalancerConfigContent = fs.readFileSync( rebalancerConfigFile, @@ -93,6 +100,8 @@ export class RebalancerHelmManager extends HelmManager { registryUri, rebalancerConfig: this.rebalancerConfigContent, withMetrics: this.withMetrics, + // Used for fetching private RPC secrets + chains: this.rebalancerChains, }, }; } @@ -133,5 +142,111 @@ export class RebalancerHelmManager extends HelmManager { } } + /** + * Get all deployed rebalancers that include the given chain. + * Used by RPC rotation to refresh rebalancer pods when RPCs change. + */ + static async getManagersForChain( + environment: DeployEnvironment, + chain: string, + ): Promise { + const deployedRebalancers = await getDeployedRebalancerWarpRouteIds( + environment, + RebalancerHelmManager.helmReleasePrefix, + ); + + const helmManagers: RebalancerHelmManager[] = []; + + for (const { warpRouteId } of deployedRebalancers) { + let warpCoreConfig; + try { + warpCoreConfig = getWarpCoreConfig(warpRouteId); + } catch (e) { + continue; + } + + const warpChains = warpCoreConfig.tokens.map((t) => t.chainName); + if (!warpChains.includes(chain)) { + continue; + } + + // Create a minimal manager for RPC rotation (only needs helmReleaseName and namespace) + helmManagers.push( + new RebalancerHelmManager( + warpRouteId, + environment, + '', // registryCommit not needed for refresh + '', // rebalancerConfigFile not needed for refresh + '', // rebalanceStrategy not needed for refresh + false, // withMetrics not needed for refresh + ), + ); + } + + return helmManagers; + } + // TODO: allow for a rebalancer to be uninstalled } + +export interface RebalancerPodInfo { + helmReleaseName: string; + warpRouteId: string; +} + +/** + * Get deployed rebalancer warp route IDs by inspecting k8s pods. + */ +export async function getDeployedRebalancerWarpRouteIds( + namespace: string, + helmReleasePrefix: string, +): Promise { + const podsResult = await execCmdAndParseJson( + `kubectl get pods -n ${namespace} -o json`, + ); + + const rebalancerPods: RebalancerPodInfo[] = []; + + for (const pod of podsResult.items || []) { + const helmReleaseName = + pod.metadata?.labels?.['app.kubernetes.io/instance']; + + if (!helmReleaseName?.startsWith(helmReleasePrefix)) { + continue; + } + + let warpRouteId: string | undefined; + + for (const container of pod.spec?.containers || []) { + // Check WARP_ROUTE_ID env var + const warpRouteIdEnv = (container.env || []).find( + (e: { name: string; value?: string }) => e.name === 'WARP_ROUTE_ID', + ); + if (warpRouteIdEnv?.value) { + warpRouteId = warpRouteIdEnv.value; + break; + } + + // Check --warpRouteId in command or args + const allArgs: string[] = [ + ...(container.command || []), + ...(container.args || []), + ]; + const warpRouteIdArgIndex = allArgs.indexOf('--warpRouteId'); + if (warpRouteIdArgIndex !== -1 && allArgs[warpRouteIdArgIndex + 1]) { + warpRouteId = allArgs[warpRouteIdArgIndex + 1]; + break; + } + } + + if (warpRouteId) { + rebalancerPods.push({ helmReleaseName, warpRouteId }); + } else { + rootLogger.warn( + `Could not extract warp route ID from rebalancer pod with helm release: ${helmReleaseName}. Skipping.`, + ); + } + } + + return rebalancerPods; +} diff --git a/typescript/infra/src/utils/rpcUrls.ts b/typescript/infra/src/utils/rpcUrls.ts index ff5348e6913..418820243f9 100644 --- a/typescript/infra/src/utils/rpcUrls.ts +++ b/typescript/infra/src/utils/rpcUrls.ts @@ -21,6 +21,7 @@ import { import { DeployEnvironment } from '../config/environment.js'; import { KeyFunderHelmManager } from '../funding/key-funder.js'; import { KathyHelmManager } from '../helloworld/kathy.js'; +import { RebalancerHelmManager } from '../rebalancer/helm.js'; import { WarpRouteMonitorHelmManager } from '../warp-monitor/helm.js'; import { disableGCPSecretVersion } from './gcloud.js'; @@ -273,7 +274,7 @@ async function updateSecretAndDisablePrevious( /** * Interactively refreshes dependent k8s resources for the given chain in the given environment. - * Prompts for core infrastructure, warp monitors, and CronJobs separately, then executes refreshes. + * Prompts for core infrastructure, warp monitors, rebalancers, and CronJobs separately. * CronJobs only get secret refresh (no pod restart) - they pick up new secrets on next run. * @param environment The environment to refresh resources in * @param chain The chain to refresh resources for @@ -293,10 +294,15 @@ async function refreshDependentK8sResourcesInteractive( // Collect selections from all prompts const coreManagers = await selectCoreInfrastructure(environment, chain); const warpManagers = await selectWarpMonitors(environment, chain); + const rebalancerManagers = await selectRebalancers(environment, chain); const cronjobManagers = await selectCronJobs(environment); // Services get both secret and pod refresh - const serviceManagers = [...coreManagers, ...warpManagers]; + const serviceManagers = [ + ...coreManagers, + ...warpManagers, + ...rebalancerManagers, + ]; // CronJobs only get secret refresh (they pick up new secrets on next scheduled run) const allManagersForSecrets = [...serviceManagers, ...cronjobManagers]; @@ -363,7 +369,7 @@ async function selectCoreInfrastructure( .filter((_, i) => selection.includes(i)); } -enum WarpMonitorRefreshChoice { +enum RefreshChoice { ALL = 'all', SELECT = 'select', SKIP = 'skip', @@ -393,25 +399,25 @@ async function selectWarpMonitors( choices: [ { name: `Yes, refresh all ${warpMonitorManagers.length} monitors`, - value: WarpMonitorRefreshChoice.ALL, + value: RefreshChoice.ALL, }, { name: 'Yes, let me select which ones', - value: WarpMonitorRefreshChoice.SELECT, + value: RefreshChoice.SELECT, }, { name: 'No, skip warp monitors', - value: WarpMonitorRefreshChoice.SKIP, + value: RefreshChoice.SKIP, }, ], }); - if (choice === WarpMonitorRefreshChoice.SKIP) { + if (choice === RefreshChoice.SKIP) { console.log('Skipping warp monitor refresh'); return []; } - if (choice === WarpMonitorRefreshChoice.ALL) { + if (choice === RefreshChoice.ALL) { return warpMonitorManagers; } @@ -427,6 +433,66 @@ async function selectWarpMonitors( return warpMonitorManagers.filter((_, i) => selection.includes(i)); } +async function selectRebalancers( + environment: DeployEnvironment, + chain: string, +): Promise { + const rebalancerManagers = await RebalancerHelmManager.getManagersForChain( + environment, + chain, + ); + + if (rebalancerManagers.length === 0) { + console.log(`No rebalancers found for ${chain}`); + return []; + } + + console.log( + `Found ${rebalancerManagers.length} rebalancers that include ${chain}:`, + ); + for (const manager of rebalancerManagers) { + console.log(` - ${manager.helmReleaseName} (${manager.warpRouteId})`); + } + + const choice = await select({ + message: `Refresh rebalancers?`, + choices: [ + { + name: `Yes, refresh all ${rebalancerManagers.length} rebalancers`, + value: RefreshChoice.ALL, + }, + { + name: 'Yes, let me select which ones', + value: RefreshChoice.SELECT, + }, + { + name: 'No, skip rebalancers', + value: RefreshChoice.SKIP, + }, + ], + }); + + if (choice === RefreshChoice.SKIP) { + console.log('Skipping rebalancer refresh'); + return []; + } + + if (choice === RefreshChoice.ALL) { + return rebalancerManagers; + } + + const selection = await checkbox({ + message: 'Select rebalancers to refresh', + choices: rebalancerManagers.map((manager, i) => ({ + name: manager.helmReleaseName, + value: i, + checked: true, + })), + }); + + return rebalancerManagers.filter((_, i) => selection.includes(i)); +} + async function selectCronJobs( environment: DeployEnvironment, ): Promise[]> { @@ -476,12 +542,6 @@ async function selectCronJobs( .filter((_, i) => selection.includes(i)); } -/** - * Test the provider at the given URL, returning false if the provider is unhealthy - * or related to a different chain. No-op for non-Ethereum chains. - * @param chain The chain to test the provider for - * @param url The URL of the provider - */ async function testProvider(chain: ChainName, url: string): Promise { const chainMetadata = getChain(chain); if (chainMetadata.protocol !== ProtocolType.Ethereum) { diff --git a/typescript/rebalancer/src/service.ts b/typescript/rebalancer/src/service.ts index df68333ebc4..bbbf58454f2 100644 --- a/typescript/rebalancer/src/service.ts +++ b/typescript/rebalancer/src/service.ts @@ -15,6 +15,7 @@ * - MONITOR_ONLY: Run in monitor-only mode without executing transactions (default: "false") * - LOG_LEVEL: Logging level (default: "info") - supported by pino * - REGISTRY_URI: Registry URI for chain metadata. Can include /tree/{commit} to pin version (default: GitHub registry) + * - RPC_URL_: Override RPC URL for a specific chain (e.g., RPC_URL_ETHEREUM, RPC_URL_ARBITRUM) * * Usage: * node dist/service.js @@ -24,7 +25,11 @@ import { Wallet } from 'ethers'; import { DEFAULT_GITHUB_REGISTRY } from '@hyperlane-xyz/registry'; import { getRegistry } from '@hyperlane-xyz/registry/fs'; -import { MultiProtocolProvider, MultiProvider } from '@hyperlane-xyz/sdk'; +import { + ChainMetadata, + MultiProtocolProvider, + MultiProvider, +} from '@hyperlane-xyz/sdk'; import { createServiceLogger, rootLogger } from '@hyperlane-xyz/utils'; import { RebalancerConfig } from './config/RebalancerConfig.js'; @@ -100,6 +105,9 @@ async function main(): Promise { `✅ Loaded metadata for ${Object.keys(chainMetadata).length} chains`, ); + // Apply RPC URL overrides from environment variables + applyRpcOverrides(chainMetadata); + // Create MultiProvider with signer const multiProvider = new MultiProvider(chainMetadata); const signer = new Wallet(privateKey); @@ -139,6 +147,31 @@ async function main(): Promise { } } +/** + * Applies RPC URL overrides from environment variables. + * Checks ALL chains in metadata for RPC_URL_ env vars + * (e.g., RPC_URL_ETHEREUM, RPC_URL_PARADEX) and overrides the registry URL. + * This ensures warp route chains not in the rebalancing strategy still get + * private RPCs for monitoring. + */ +function applyRpcOverrides( + chainMetadata: Record>, +): void { + for (const chain of Object.keys(chainMetadata)) { + const envVarName = `RPC_URL_${chain.toUpperCase().replace(/-/g, '_')}`; + const rpcUrl = process.env[envVarName]; + if (rpcUrl) { + rootLogger.debug( + { chain, envVarName }, + 'Using RPC from environment variable', + ); + chainMetadata[chain].rpcUrls = [ + { http: rpcUrl }, + ] as ChainMetadata['rpcUrls']; + } + } +} + // Run the service main().catch((error) => { const err = error as Error;