From 3546a6680f7ba7907a5a7330550046c70309597d Mon Sep 17 00:00:00 2001 From: David Chu Date: Thu, 19 Feb 2026 14:38:39 -0800 Subject: [PATCH 1/4] shadaj fixes --- hydro_deploy/core/src/aws.rs | 95 +++++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 39 deletions(-) diff --git a/hydro_deploy/core/src/aws.rs b/hydro_deploy/core/src/aws.rs index e7c3b72b06b9..3d184a058e58 100644 --- a/hydro_deploy/core/src/aws.rs +++ b/hydro_deploy/core/src/aws.rs @@ -40,23 +40,30 @@ impl LaunchedSshHost for LaunchedEc2Instance { } } +#[derive(Debug)] +pub struct NetworkResources { + vpc: String, + subnet: String, + security_group: String, +} + #[derive(Debug)] pub struct AwsNetwork { pub region: String, - pub existing_vpc: OnceLock, + pub existing_network: OnceLock, id: String, } impl AwsNetwork { - pub fn new(region: impl Into, existing_vpc: Option) -> Arc { + pub fn new(region: impl Into, existing_vpc: Option) -> Arc { Arc::new(Self { region: region.into(), - existing_vpc: existing_vpc.map(From::from).unwrap_or_default(), + existing_network: existing_vpc.map(From::from).unwrap_or_default(), id: nanoid!(8, &TERRAFORM_ALPHABET), }) } - fn collect_resources(&self, resource_batch: &mut ResourceBatch) -> String { + fn collect_resources(&self, resource_batch: &mut ResourceBatch) -> NetworkResources { resource_batch .terraform .terraform @@ -77,29 +84,35 @@ impl AwsNetwork { ); let vpc_network = format!("hydro-vpc-network-{}", self.id); - - if let Some(existing) = self.existing_vpc.get() { - if resource_batch - .terraform - .resource - .get("aws_vpc") - .is_some_and(|map| map.contains_key(existing)) - { - format!("aws_vpc.{existing}") - } else { - resource_batch + let subnet_key = format!("{vpc_network}-subnet"); + let sg_key = format!("{vpc_network}-default-sg"); + + if let Some(existing) = self.existing_network.get() { + // Resolve an existing resource: reuse if already in terraform resources, + // otherwise create a data source lookup. + let mut resolve = |resource_type: &str, existing_id: &str, data_key: String| { + if resource_batch .terraform - .data - .entry("aws_vpc".to_owned()) - .or_default() - .insert( - vpc_network.clone(), - json!({ - "id": existing, - }), - ); + .resource + .get(resource_type) + .is_some_and(|map| map.contains_key(existing_id)) + { + format!("{resource_type}.{existing_id}") + } else { + resource_batch + .terraform + .data + .entry(resource_type.to_owned()) + .or_default() + .insert(data_key.clone(), json!({ "id": existing_id })); + format!("data.{resource_type}.{data_key}") + } + }; - format!("data.aws_vpc.{vpc_network}") + NetworkResources { + vpc: resolve("aws_vpc", &existing.vpc, vpc_network), + subnet: resolve("aws_subnet", &existing.subnet, subnet_key), + security_group: resolve("aws_security_group", &existing.security_group, sg_key), } } else { resource_batch @@ -137,7 +150,6 @@ impl AwsNetwork { ); // Create subnet - let subnet_key = format!("{vpc_network}-subnet"); resource_batch .terraform .resource @@ -202,14 +214,13 @@ impl AwsNetwork { ); // Create security group that allows internal communication - let sg_key = format!("{vpc_network}-default-sg"); resource_batch .terraform .resource .entry("aws_security_group".to_owned()) .or_default() .insert( - sg_key, + sg_key.clone(), json!({ "name": format!("{vpc_network}-default-allow-internal"), "description": "Allow internal communication between instances", @@ -265,9 +276,19 @@ impl AwsNetwork { }), ); - let out = format!("aws_vpc.{vpc_network}"); - self.existing_vpc.set(vpc_network).unwrap(); - out + let resources = NetworkResources { + vpc: format!("aws_vpc.{vpc_network}"), + subnet: format!("aws_subnet.{subnet_key}"), + security_group: format!("aws_security_group.{sg_key}"), + }; + self.existing_network + .set(NetworkResources { + vpc: vpc_network, + subnet: subnet_key, + security_group: sg_key, + }) + .unwrap(); + resources } } } @@ -612,7 +633,7 @@ impl Host for AwsEc2Host { return; } - let vpc_path = self.network.collect_resources(resource_batch); + let network_resources = self.network.collect_resources(resource_batch); let iam_instance_profile = self .iam_instance_profile @@ -705,12 +726,8 @@ impl Host for AwsEc2Host { instance_name.push_str(&display_name); } - let network_id = self.network.id.clone(); - let vpc_ref = format!("${{{}.id}}", vpc_path); - let default_sg_ref = format!( - "${{aws_security_group.hydro-vpc-network-{}-default-sg.id}}", - network_id - ); + let vpc_ref = format!("${{{}.id}}", network_resources.vpc); + let default_sg_ref = format!("${{{}.id}}", network_resources.security_group); // Create additional security group for external ports if needed let mut security_groups = vec![default_sg_ref]; @@ -764,7 +781,7 @@ impl Host for AwsEc2Host { } drop(external_ports); - let subnet_ref = format!("${{aws_subnet.hydro-vpc-network-{}-subnet.id}}", network_id); + let subnet_ref = format!("${{{}.id}}", network_resources.subnet); let iam_instance_profile_ref = iam_instance_profile.map(|key| format!("${{{key}.name}}")); // Write the CloudWatch Agent config file. From a6ebd79a98df90e5612c6603b023cf058270e6aa Mon Sep 17 00:00:00 2001 From: David Chu Date: Mon, 23 Feb 2026 21:38:56 +0000 Subject: [PATCH 2/4] Working again --- hydro_deploy/core/src/aws.rs | 50 ++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/hydro_deploy/core/src/aws.rs b/hydro_deploy/core/src/aws.rs index 3d184a058e58..ba637d6ce424 100644 --- a/hydro_deploy/core/src/aws.rs +++ b/hydro_deploy/core/src/aws.rs @@ -41,6 +41,7 @@ impl LaunchedSshHost for LaunchedEc2Instance { } #[derive(Debug)] +#[derive(Clone)] pub struct NetworkResources { vpc: String, subnet: String, @@ -50,7 +51,7 @@ pub struct NetworkResources { #[derive(Debug)] pub struct AwsNetwork { pub region: String, - pub existing_network: OnceLock, + pub existing_network: Mutex>, id: String, } @@ -58,7 +59,7 @@ impl AwsNetwork { pub fn new(region: impl Into, existing_vpc: Option) -> Arc { Arc::new(Self { region: region.into(), - existing_network: existing_vpc.map(From::from).unwrap_or_default(), + existing_network: Mutex::new(existing_vpc), id: nanoid!(8, &TERRAFORM_ALPHABET), }) } @@ -87,7 +88,7 @@ impl AwsNetwork { let subnet_key = format!("{vpc_network}-subnet"); let sg_key = format!("{vpc_network}-default-sg"); - if let Some(existing) = self.existing_network.get() { + if let Some(existing) = self.existing_network.lock().unwrap().clone() { // Resolve an existing resource: reuse if already in terraform resources, // otherwise create a data source lookup. let mut resolve = |resource_type: &str, existing_id: &str, data_key: String| { @@ -281,16 +282,44 @@ impl AwsNetwork { subnet: format!("aws_subnet.{subnet_key}"), security_group: format!("aws_security_group.{sg_key}"), }; - self.existing_network - .set(NetworkResources { - vpc: vpc_network, - subnet: subnet_key, - security_group: sg_key, - }) - .unwrap(); + + // Add outputs so we can retrieve actual AWS IDs after apply + resource_batch.terraform.output.insert( + format!("hydro-network-{}-vpc-id", self.id), + TerraformOutput { value: format!("${{aws_vpc.{vpc_network}.id}}") }, + ); + resource_batch.terraform.output.insert( + format!("hydro-network-{}-subnet-id", self.id), + TerraformOutput { value: format!("${{aws_subnet.{subnet_key}.id}}") }, + ); + resource_batch.terraform.output.insert( + format!("hydro-network-{}-sg-id", self.id), + TerraformOutput { value: format!("${{aws_security_group.{sg_key}.id}}") }, + ); + + *self.existing_network.lock().unwrap() = Some(NetworkResources { + vpc: vpc_network, + subnet: subnet_key, + security_group: sg_key, + }); resources } } + + pub fn update_from_outputs(&self, resource_result: &ResourceResult) { + let outputs = &resource_result.terraform.outputs; + if let (Some(vpc), Some(subnet), Some(sg)) = ( + outputs.get(&format!("hydro-network-{}-vpc-id", self.id)), + outputs.get(&format!("hydro-network-{}-subnet-id", self.id)), + outputs.get(&format!("hydro-network-{}-sg-id", self.id)), + ) { + *self.existing_network.lock().unwrap() = Some(NetworkResources { + vpc: vpc.value.clone(), + subnet: subnet.value.clone(), + security_group: sg.value.clone(), + }); + } + } } /// Represents a IAM role, IAM policy attachments, and instance profile for one or multiple EC2 instances. @@ -911,6 +940,7 @@ echo -e "{cwa_config_esc}" > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwa .get_or_init(|| { let id = self.id; + self.network.update_from_outputs(resource_result); let internal_ip = resource_result .terraform .outputs From 585ca4567b3662b98840931da3d47fdacec2718c Mon Sep 17 00:00:00 2001 From: David Chu Date: Mon, 23 Feb 2026 21:40:04 +0000 Subject: [PATCH 3/4] fetch actual resource ID --- hydro_deploy/core/src/aws.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/hydro_deploy/core/src/aws.rs b/hydro_deploy/core/src/aws.rs index ba637d6ce424..f43d0c68db10 100644 --- a/hydro_deploy/core/src/aws.rs +++ b/hydro_deploy/core/src/aws.rs @@ -40,8 +40,7 @@ impl LaunchedSshHost for LaunchedEc2Instance { } } -#[derive(Debug)] -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct NetworkResources { vpc: String, subnet: String, @@ -286,15 +285,21 @@ impl AwsNetwork { // Add outputs so we can retrieve actual AWS IDs after apply resource_batch.terraform.output.insert( format!("hydro-network-{}-vpc-id", self.id), - TerraformOutput { value: format!("${{aws_vpc.{vpc_network}.id}}") }, + TerraformOutput { + value: format!("${{aws_vpc.{vpc_network}.id}}"), + }, ); resource_batch.terraform.output.insert( format!("hydro-network-{}-subnet-id", self.id), - TerraformOutput { value: format!("${{aws_subnet.{subnet_key}.id}}") }, + TerraformOutput { + value: format!("${{aws_subnet.{subnet_key}.id}}"), + }, ); resource_batch.terraform.output.insert( format!("hydro-network-{}-sg-id", self.id), - TerraformOutput { value: format!("${{aws_security_group.{sg_key}.id}}") }, + TerraformOutput { + value: format!("${{aws_security_group.{sg_key}.id}}"), + }, ); *self.existing_network.lock().unwrap() = Some(NetworkResources { From d0d9ad5cb0ccdc2a4c4d567dec5a479b777f1bad Mon Sep 17 00:00:00 2001 From: David Chu Date: Tue, 24 Feb 2026 00:02:37 +0000 Subject: [PATCH 4/4] Cleaning --- hydro_deploy/core/src/aws.rs | 41 +++++++++++++++++------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/hydro_deploy/core/src/aws.rs b/hydro_deploy/core/src/aws.rs index f43d0c68db10..fc24e4fbd7b1 100644 --- a/hydro_deploy/core/src/aws.rs +++ b/hydro_deploy/core/src/aws.rs @@ -50,7 +50,8 @@ pub struct NetworkResources { #[derive(Debug)] pub struct AwsNetwork { pub region: String, - pub existing_network: Mutex>, + pub existing_network_key: OnceLock, + pub existing_network_id: OnceLock, id: String, } @@ -58,7 +59,8 @@ impl AwsNetwork { pub fn new(region: impl Into, existing_vpc: Option) -> Arc { Arc::new(Self { region: region.into(), - existing_network: Mutex::new(existing_vpc), + existing_network_key: OnceLock::new(), + existing_network_id: existing_vpc.map(From::from).unwrap_or_default(), id: nanoid!(8, &TERRAFORM_ALPHABET), }) } @@ -87,26 +89,15 @@ impl AwsNetwork { let subnet_key = format!("{vpc_network}-subnet"); let sg_key = format!("{vpc_network}-default-sg"); - if let Some(existing) = self.existing_network.lock().unwrap().clone() { - // Resolve an existing resource: reuse if already in terraform resources, - // otherwise create a data source lookup. + if let Some(existing) = self.existing_network_id.get() { let mut resolve = |resource_type: &str, existing_id: &str, data_key: String| { - if resource_batch + resource_batch .terraform - .resource - .get(resource_type) - .is_some_and(|map| map.contains_key(existing_id)) - { - format!("{resource_type}.{existing_id}") - } else { - resource_batch - .terraform - .data - .entry(resource_type.to_owned()) - .or_default() - .insert(data_key.clone(), json!({ "id": existing_id })); - format!("data.{resource_type}.{data_key}") - } + .data + .entry(resource_type.to_owned()) + .or_default() + .insert(data_key.clone(), json!({ "id": existing_id })); + format!("data.{resource_type}.{data_key}") }; NetworkResources { @@ -114,6 +105,12 @@ impl AwsNetwork { subnet: resolve("aws_subnet", &existing.subnet, subnet_key), security_group: resolve("aws_security_group", &existing.security_group, sg_key), } + } else if let Some(existing) = self.existing_network_key.get() { + NetworkResources { + vpc: format!("aws_vpc.{}", existing.vpc), + subnet: format!("aws_subnet.{}", existing.subnet), + security_group: format!("aws_security_group.{}", existing.security_group), + } } else { resource_batch .terraform @@ -302,7 +299,7 @@ impl AwsNetwork { }, ); - *self.existing_network.lock().unwrap() = Some(NetworkResources { + let _ = self.existing_network_key.set(NetworkResources { vpc: vpc_network, subnet: subnet_key, security_group: sg_key, @@ -318,7 +315,7 @@ impl AwsNetwork { outputs.get(&format!("hydro-network-{}-subnet-id", self.id)), outputs.get(&format!("hydro-network-{}-sg-id", self.id)), ) { - *self.existing_network.lock().unwrap() = Some(NetworkResources { + let _ = self.existing_network_id.set(NetworkResources { vpc: vpc.value.clone(), subnet: subnet.value.clone(), security_group: sg.value.clone(),