diff --git a/typescript/infra/config/docker.ts b/typescript/infra/config/docker.ts index 2a88ce7453b..adc5d75a695 100644 --- a/typescript/infra/config/docker.ts +++ b/typescript/infra/config/docker.ts @@ -36,8 +36,8 @@ export const mainnetDockerTags: MainnetDockerTags = { kathy: '74d999b-20260108-145131', checkWarpDeploy: '74d999b-20260108-145131', // standalone services - warpMonitor: '74d999b-20260108-145128', - rebalancer: '74d999b-20260108-145129', + warpMonitor: '6b6fd0b-20260123-121413', + rebalancer: '6b6fd0b-20260123-121418', }; export const testnetDockerTags: BaseDockerTags = { diff --git a/typescript/infra/config/registry.ts b/typescript/infra/config/registry.ts index 492590d7a6e..a9713b71051 100644 --- a/typescript/infra/config/registry.ts +++ b/typescript/infra/config/registry.ts @@ -104,6 +104,10 @@ export function getChainAddresses(): ChainMap { return getRegistry().getAddresses(); } +export function warpRouteExistsInRegistry(warpRouteId: string): boolean { + return !!getRegistry().getWarpRoute(warpRouteId); +} + export function getWarpCoreConfig(warpRouteId: string): WarpCoreConfig { const registry = getRegistry(); const warpRouteConfig = registry.getWarpRoute(warpRouteId); diff --git a/typescript/infra/helm/rebalancer/templates/_helpers.tpl b/typescript/infra/helm/rebalancer/templates/_helpers.tpl index 08584d764ab..6ad22eddf2c 100644 --- a/typescript/infra/helm/rebalancer/templates/_helpers.tpl +++ b/typescript/infra/helm/rebalancer/templates/_helpers.tpl @@ -74,6 +74,10 @@ The rebalancer container value: json - name: LOG_LEVEL value: info + {{- if .Values.warpRouteId }} + - name: WARP_ROUTE_ID + value: {{ .Values.warpRouteId }} + {{- end }} {{- if .Values.hyperlane.registryUri }} - name: REGISTRY_URI value: {{ .Values.hyperlane.registryUri }} diff --git a/typescript/infra/helm/rebalancer/values.yaml b/typescript/infra/helm/rebalancer/values.yaml index 32239f0b6b2..7e0dad36338 100644 --- a/typescript/infra/helm/rebalancer/values.yaml +++ b/typescript/infra/helm/rebalancer/values.yaml @@ -1,6 +1,7 @@ image: repository: gcr.io/abacus-labs-dev/hyperlane-rebalancer tag: +warpRouteId: '' hyperlane: runEnv: context: hyperlane diff --git a/typescript/infra/scripts/agent-utils.ts b/typescript/infra/scripts/agent-utils.ts index 98e9fc692f4..cb2e214f071 100644 --- a/typescript/infra/scripts/agent-utils.ts +++ b/typescript/infra/scripts/agent-utils.ts @@ -36,6 +36,7 @@ import { getChains, getEnvChains, getRegistry, + warpRouteExistsInRegistry, } from '../config/registry.js'; import { getCurrentKubernetesContext } from '../src/agents/index.js'; import { getCloudAgentKey } from '../src/agents/key-utils.js'; @@ -249,6 +250,14 @@ export function withDryRun(args: Argv) { .default('dryRun', false); } +export function withYes(args: Argv) { + return args + .describe('yes', 'Skip confirmations and use defaults') + .boolean('yes') + .alias('y', 'yes') + .default('yes', false); +} + export function withKnownWarpRouteIdRequired(args: Argv) { return withKnownWarpRouteId(args).demandOption('warpRouteId'); } @@ -441,6 +450,24 @@ export async function getWarpRouteIdsInteractive( return selection; } +export function filterOrphanedWarpRouteIds(warpRouteIds: string[]): { + validIds: string[]; + orphanedIds: string[]; +} { + const validIds: string[] = []; + const orphanedIds: string[] = []; + + for (const id of warpRouteIds) { + if (warpRouteExistsInRegistry(id)) { + validIds.push(id); + } else { + orphanedIds.push(id); + } + } + + return { validIds, orphanedIds }; +} + // not requiring to build coreConfig to get agentConfig export async function getAgentConfigsBasedOnArgs(argv?: { environment: DeployEnvironment; diff --git a/typescript/infra/scripts/rebalancer/deploy-rebalancer.ts b/typescript/infra/scripts/rebalancer/deploy-rebalancer.ts index fe459488441..b494d65e742 100644 --- a/typescript/infra/scripts/rebalancer/deploy-rebalancer.ts +++ b/typescript/infra/scripts/rebalancer/deploy-rebalancer.ts @@ -1,4 +1,4 @@ -import { input } from '@inquirer/prompts'; +import { checkbox, input } from '@inquirer/prompts'; import path from 'path'; import { @@ -9,16 +9,21 @@ import { } from '@hyperlane-xyz/utils'; import { DeployEnvironment } from '../../src/config/environment.js'; -import { RebalancerHelmManager } from '../../src/rebalancer/helm.js'; +import { + RebalancerHelmManager, + getDeployedRebalancerWarpRouteIds, +} from '../../src/rebalancer/helm.js'; +import { REBALANCER_HELM_RELEASE_PREFIX } from '../../src/utils/consts.js'; import { validateRegistryCommit } from '../../src/utils/git.js'; import { HelmCommand } from '../../src/utils/helm.js'; import { assertCorrectKubeContext, + filterOrphanedWarpRouteIds, getArgs, - getWarpRouteIdsInteractive, withMetrics, withRegistryCommit, withWarpRouteId, + withYes, } from '../agent-utils.js'; import { getEnvironmentConfig } from '../core-utils.js'; @@ -33,30 +38,102 @@ async function main() { warpRouteId, metrics, registryCommit: registryCommitArg, - } = await withMetrics(withRegistryCommit(withWarpRouteId(getArgs()))).parse(); + yes: skipConfirmation, + } = await withYes( + withMetrics(withRegistryCommit(withWarpRouteId(getArgs()))), + ).parse(); await assertCorrectKubeContext(getEnvironmentConfig(environment)); - let warpRouteIds; + let warpRouteIds: string[]; if (warpRouteId) { warpRouteIds = [warpRouteId]; } else { - warpRouteIds = await getWarpRouteIdsInteractive(environment); + const deployedPods = await getDeployedRebalancerWarpRouteIds( + environment, + REBALANCER_HELM_RELEASE_PREFIX, + ); + const deployedIds = [ + ...new Set( + deployedPods + .map((p) => p.warpRouteId) + .filter((id): id is string => !!id), + ), + ].sort(); + + if (deployedIds.length === 0) { + rootLogger.error( + 'No deployed rebalancers found. Use --warp-route-id to deploy a new one.', + ); + process.exit(1); + } + + warpRouteIds = await checkbox({ + message: 'Select rebalancers to redeploy', + choices: deployedIds.map((id) => ({ value: id })), + pageSize: 30, + }); + + if (warpRouteIds.length === 0) { + rootLogger.info('No rebalancers selected'); + process.exit(0); + } + } + + const { validIds: validWarpRouteIds, orphanedIds } = + filterOrphanedWarpRouteIds(warpRouteIds); + + if (orphanedIds.length > 0) { + rootLogger.warn( + `Skipping ${orphanedIds.length} orphaned rebalancers (warp route no longer in registry):\n${orphanedIds.map((id) => ` - ${id}`).join('\n')}`, + ); + rootLogger.warn('Run helm uninstall manually to remove these rebalancers'); } - const registryCommit = - registryCommitArg ?? - (await input({ - message: - 'Enter the registry version to use (can be a commit, branch or tag):', - })); - await validateRegistryCommit(registryCommit); + if (validWarpRouteIds.length === 0) { + if (warpRouteId && orphanedIds.includes(warpRouteId)) { + rootLogger.error( + `Warp route "${warpRouteId}" not found in registry. Verify the warp route ID is correct.`, + ); + process.exit(1); + } + rootLogger.info('No valid warp routes to deploy'); + process.exit(0); + } rootLogger.info( - `Deploying Rebalancer for the following Route IDs:\n${warpRouteIds.map((id) => ` - ${id}`).join('\n')}`, + `Deploying Rebalancer for the following Route IDs:\n${validWarpRouteIds.map((id) => ` - ${id}`).join('\n')}`, ); + // Cache validated commits to avoid re-validating the same commit + const validatedCommits = new Set(); + const deployRebalancer = async (warpRouteId: string) => { + let registryCommit: string; + if (registryCommitArg) { + registryCommit = registryCommitArg; + } else { + const defaultRegistryCommit = + await RebalancerHelmManager.getDeployedRegistryCommit( + warpRouteId, + environment, + ); + + if (skipConfirmation) { + registryCommit = defaultRegistryCommit ?? 'main'; + } else { + registryCommit = await input({ + message: `[${warpRouteId}] Enter registry version (commit, branch or tag):`, + default: defaultRegistryCommit, + }); + } + } + + if (!validatedCommits.has(registryCommit)) { + await validateRegistryCommit(registryCommit); + validatedCommits.add(registryCommit); + } + // Build path for config file - relative for local checks const configFileName = `${warpRouteId}-config.yaml`; const relativeConfigPath = path.join( @@ -83,7 +160,7 @@ async function main() { // TODO: Uninstall any stale rebalancer releases. - for (const id of warpRouteIds) { + for (const id of validWarpRouteIds) { rootLogger.info(`Deploying Rebalancer for Route ID: ${id}`); await deployRebalancer(id); } diff --git a/typescript/infra/scripts/warp-routes/deploy-warp-monitor.ts b/typescript/infra/scripts/warp-routes/deploy-warp-monitor.ts index 02fff9bc343..48e6d5ca429 100644 --- a/typescript/infra/scripts/warp-routes/deploy-warp-monitor.ts +++ b/typescript/infra/scripts/warp-routes/deploy-warp-monitor.ts @@ -1,4 +1,4 @@ -import { input } from '@inquirer/prompts'; +import { checkbox, input } from '@inquirer/prompts'; import { LogFormat, @@ -10,17 +10,22 @@ import { import { Contexts } from '../../config/contexts.js'; import { getWarpCoreConfig } from '../../config/registry.js'; +import { WARP_ROUTE_MONITOR_HELM_RELEASE_PREFIX } from '../../src/utils/consts.js'; import { validateRegistryCommit } from '../../src/utils/git.js'; import { HelmCommand } from '../../src/utils/helm.js'; -import { WarpRouteMonitorHelmManager } from '../../src/warp-monitor/helm.js'; +import { + WarpRouteMonitorHelmManager, + getDeployedWarpMonitorWarpRouteIds, +} from '../../src/warp-monitor/helm.js'; import { assertCorrectKubeContext, + filterOrphanedWarpRouteIds, getAgentConfig, getArgs, getMultiProtocolProvider, - getWarpRouteIdsInteractive, withRegistryCommit, withWarpRouteId, + withYes, } from '../agent-utils.js'; import { getEnvironmentConfig } from '../core-utils.js'; @@ -30,24 +35,73 @@ async function main() { environment, warpRouteId, registryCommit: registryCommitArg, - } = await withRegistryCommit(withWarpRouteId(getArgs())).argv; + yes: skipConfirmation, + } = await withYes(withRegistryCommit(withWarpRouteId(getArgs()))).argv; await timedAsync('assertCorrectKubeContext', () => assertCorrectKubeContext(getEnvironmentConfig(environment)), ); const envConfig = getEnvironmentConfig(environment); - // Get warp route IDs first to determine which chains we need - let warpRouteIds; + let warpRouteIds: string[]; if (warpRouteId) { warpRouteIds = [warpRouteId]; } else { - warpRouteIds = await getWarpRouteIdsInteractive(environment); + const deployedPods = await getDeployedWarpMonitorWarpRouteIds( + environment, + WARP_ROUTE_MONITOR_HELM_RELEASE_PREFIX, + ); + const deployedIds = [ + ...new Set( + deployedPods + .map((p) => p.warpRouteId) + .filter((id): id is string => !!id), + ), + ].sort(); + + if (deployedIds.length === 0) { + rootLogger.error( + 'No deployed warp monitors found. Use --warp-route-id to deploy a new one.', + ); + process.exit(1); + } + + warpRouteIds = await checkbox({ + message: 'Select warp monitors to redeploy', + choices: deployedIds.map((id) => ({ value: id })), + pageSize: 30, + }); + + if (warpRouteIds.length === 0) { + rootLogger.info('No warp monitors selected'); + process.exit(0); + } + } + + const { validIds: validWarpRouteIds, orphanedIds } = + filterOrphanedWarpRouteIds(warpRouteIds); + + if (orphanedIds.length > 0) { + rootLogger.warn( + `Skipping ${orphanedIds.length} orphaned monitors (warp route no longer in registry):\n${orphanedIds.map((id) => ` - ${id}`).join('\n')}`, + ); + rootLogger.warn('Run helm uninstall manually to remove these monitors'); + } + + if (validWarpRouteIds.length === 0) { + if (warpRouteId && orphanedIds.includes(warpRouteId)) { + rootLogger.error( + `Warp route "${warpRouteId}" not found in registry. Verify the warp route ID is correct.`, + ); + process.exit(1); + } + rootLogger.info('No valid warp routes to deploy'); + process.exit(0); } // Extract chains from warp routes to only fetch secrets for needed chains const warpRouteChains = new Set(); - for (const id of warpRouteIds) { + for (const id of validWarpRouteIds) { const warpConfig = getWarpCoreConfig(id); for (const token of warpConfig.tokens) { warpRouteChains.add(token.chainName); @@ -58,21 +112,8 @@ async function main() { `Loading secrets for ${chainsNeeded.length} chains: ${chainsNeeded.join(', ')}`, ); - const registryCommit = - registryCommitArg ?? - (await input({ - message: - 'Enter the registry version to use (can be a commit, branch or tag):', - })); - - // Only fetch secrets for the chains in the warp routes (optimization) - const [registry] = await timedAsync( - 'getRegistry + validateRegistryCommit', - () => - Promise.all([ - envConfig.getRegistry(true, chainsNeeded), - validateRegistryCommit(registryCommit), - ]), + const registry = await timedAsync('getRegistry', () => + envConfig.getRegistry(true, chainsNeeded), ); const multiProtocolProvider = await timedAsync( 'getMultiProtocolProvider', @@ -81,7 +122,34 @@ async function main() { const agentConfig = getAgentConfig(Contexts.Hyperlane, environment); + const validatedCommits = new Set(); + const deployWarpMonitor = async (warpRouteId: string) => { + let registryCommit: string; + if (registryCommitArg) { + registryCommit = registryCommitArg; + } else { + const defaultRegistryCommit = + await WarpRouteMonitorHelmManager.getDeployedRegistryCommit( + warpRouteId, + environment, + ); + + if (skipConfirmation) { + registryCommit = defaultRegistryCommit ?? 'main'; + } else { + registryCommit = await input({ + message: `[${warpRouteId}] Enter registry version (commit, branch or tag):`, + default: defaultRegistryCommit, + }); + } + } + + if (!validatedCommits.has(registryCommit)) { + await validateRegistryCommit(registryCommit); + validatedCommits.add(registryCommit); + } + const helmManager = new WarpRouteMonitorHelmManager( warpRouteId, environment, @@ -89,24 +157,14 @@ async function main() { registryCommit, ); await timedAsync(`runPreflightChecks(${warpRouteId})`, () => - helmManager.runPreflightChecks(multiProtocolProvider), + helmManager.runPreflightChecks(multiProtocolProvider, skipConfirmation), ); await timedAsync(`runHelmCommand(${warpRouteId})`, () => helmManager.runHelmCommand(HelmCommand.InstallOrUpgrade), ); }; - // Only run cleanup when deploying all warp routes (no specific ID provided). - // This cleanup is slow (~20s) because it reads all warp routes from the registry. - if (!warpRouteId) { - await timedAsync('uninstallUnknownWarpMonitorReleases', () => - WarpRouteMonitorHelmManager.uninstallUnknownWarpMonitorReleases( - environment, - ), - ); - } - - for (const id of warpRouteIds) { + for (const id of validWarpRouteIds) { rootLogger.info(`Deploying Warp Monitor for Warp Route ID: ${id}`); await deployWarpMonitor(id); } diff --git a/typescript/infra/src/rebalancer/helm.ts b/typescript/infra/src/rebalancer/helm.ts index cf594c4c55d..e1a7f233506 100644 --- a/typescript/infra/src/rebalancer/helm.ts +++ b/typescript/infra/src/rebalancer/helm.ts @@ -17,6 +17,7 @@ import { DeployEnvironment } from '../config/environment.js'; import { WARP_ROUTE_MONITOR_HELM_RELEASE_PREFIX } from '../utils/consts.js'; import { HelmManager, + getDeployedRegistryCommit, getHelmReleaseName, removeHelmRelease, } from '../utils/helm.js'; @@ -93,6 +94,7 @@ export class RebalancerHelmManager extends HelmManager { repository: DockerImageRepos.REBALANCER, tag: mainnetDockerTags.rebalancer, }, + warpRouteId: this.warpRouteId, withMetrics: this.withMetrics, fullnameOverride: this.helmReleaseName, hyperlane: { @@ -187,6 +189,17 @@ export class RebalancerHelmManager extends HelmManager { } // TODO: allow for a rebalancer to be uninstalled + + static getDeployedRegistryCommit( + warpRouteId: string, + environment: DeployEnvironment, + ): Promise { + return getDeployedRegistryCommit( + warpRouteId, + environment, + RebalancerHelmManager.helmReleasePrefix, + ); + } } export interface RebalancerPodInfo { @@ -239,6 +252,25 @@ export async function getDeployedRebalancerWarpRouteIds( } } + // Fallback: parse warpRouteId from configmap (for existing deployments without env var) + if (!warpRouteId) { + try { + const configMapName = `${helmReleaseName}-config`; + const cm = await execCmdAndParseJson( + `kubectl get configmap ${configMapName} -n ${namespace} -o json`, + ); + const configYaml = cm.data?.['rebalancer-config.yaml']; + if (configYaml) { + const match = configYaml.match(/^warpRouteId:\s*(.+)$/m); + warpRouteId = match?.[1]?.trim(); + } + } catch (e) { + rootLogger.debug( + `Failed to read configmap for ${helmReleaseName}: ${e}`, + ); + } + } + if (warpRouteId) { rebalancerPods.push({ helmReleaseName, warpRouteId }); } else { diff --git a/typescript/infra/src/utils/helm.ts b/typescript/infra/src/utils/helm.ts index 20bb37624d6..f43007d5b7a 100644 --- a/typescript/infra/src/utils/helm.ts +++ b/typescript/infra/src/utils/helm.ts @@ -463,3 +463,40 @@ export function getHelmReleaseName(id: string, prefix: string): string { } return name; } + +/** + * Extract registry commit from helm values. + * Supports standalone image (registryUri with /tree/{commit}) and legacy monorepo (registryCommit). + */ +export function extractRegistryCommitFromHelmValues( + values: HelmValues | null, +): string | undefined { + if (!values) return undefined; + + const registryUri = values?.hyperlane?.registryUri; + if (registryUri) { + const match = registryUri.match(/\/tree\/(.+)$/); + if (match?.[1]) return match[1]; + } + + return values?.hyperlane?.registryCommit; +} + +/** + * Get registry commit from a deployed helm release. + */ +export async function getDeployedRegistryCommit( + warpRouteId: string, + namespace: string, + helmReleasePrefix: string, +): Promise { + const helmReleaseName = getHelmReleaseName(warpRouteId, helmReleasePrefix); + try { + const values = await execCmdAndParseJson( + `helm get values ${helmReleaseName} --namespace ${namespace} -o json`, + ); + return extractRegistryCommitFromHelmValues(values); + } catch { + return undefined; + } +} diff --git a/typescript/infra/src/warp-monitor/helm.ts b/typescript/infra/src/warp-monitor/helm.ts index e2acfa8ee8a..8c12a438a90 100644 --- a/typescript/infra/src/warp-monitor/helm.ts +++ b/typescript/infra/src/warp-monitor/helm.ts @@ -19,6 +19,7 @@ import { DeployEnvironment } from '../config/environment.js'; import { REBALANCER_HELM_RELEASE_PREFIX } from '../utils/consts.js'; import { HelmManager, + getDeployedRegistryCommit, getHelmReleaseName, removeHelmRelease, } from '../utils/helm.js'; @@ -63,7 +64,10 @@ export class WarpRouteMonitorHelmManager extends HelmManager { return `${DEFAULT_GITHUB_REGISTRY}/tree/${this.registryCommit}`; } - async runPreflightChecks(multiProtocolProvider: MultiProtocolProvider) { + async runPreflightChecks( + multiProtocolProvider: MultiProtocolProvider, + skipConfirmation = false, + ) { const rebalancerReleaseName = getHelmReleaseName( this.warpRouteId, REBALANCER_HELM_RELEASE_PREFIX, @@ -95,7 +99,11 @@ export class WarpRouteMonitorHelmManager extends HelmManager { token.standard === TokenStandard.SealevelHypCollateral || token.standard === TokenStandard.SealevelHypSynthetic ) { - await this.ensureAtaPayerBalanceSufficient(warpCore, token); + await this.ensureAtaPayerBalanceSufficient( + warpCore, + token, + skipConfirmation, + ); } } } @@ -126,6 +134,17 @@ export class WarpRouteMonitorHelmManager extends HelmManager { ); } + static getDeployedRegistryCommit( + warpRouteId: string, + namespace: string, + ): Promise { + return getDeployedRegistryCommit( + warpRouteId, + namespace, + WarpRouteMonitorHelmManager.helmReleasePrefix, + ); + } + // Gets all Warp Monitor Helm Releases in the given namespace. static async getWarpMonitorHelmReleases( namespace: string, @@ -211,7 +230,11 @@ export class WarpRouteMonitorHelmManager extends HelmManager { return helmManagers; } - async ensureAtaPayerBalanceSufficient(warpCore: WarpCore, token: IToken) { + async ensureAtaPayerBalanceSufficient( + warpCore: WarpCore, + token: IToken, + skipConfirmation = false, + ) { if (!ataPayerAlertThreshold[token.chainName]) { rootLogger.warn( `No ATA payer alert threshold set for chain: ${token.chainName}. Skipping balance check.`, @@ -249,9 +272,11 @@ export class WarpRouteMonitorHelmManager extends HelmManager { token.chainName }`, ); - await confirm({ - message: 'Continue?', - }); + if (!skipConfirmation) { + await confirm({ + message: 'Continue?', + }); + } } else { rootLogger.info( `ATA payer balance for ${