From e239dfc0e0212a47218e0f3ee00c48c16a4b8390 Mon Sep 17 00:00:00 2001 From: hilderic Date: Tue, 17 Mar 2020 16:44:38 +0100 Subject: [PATCH 01/23] yorc.nodes.aws.VPC type added --- data/tosca/yorc-aws-types.yml | 44 ++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/data/tosca/yorc-aws-types.yml b/data/tosca/yorc-aws-types.yml index d4f0181e4..b8a85cf64 100644 --- a/data/tosca/yorc-aws-types.yml +++ b/data/tosca/yorc-aws-types.yml @@ -3,7 +3,7 @@ tosca_definitions_version: yorc_tosca_simple_yaml_1_0 metadata: template_name: yorc-aws-types template_author: yorc - template_version: 1.1.0 + template_version: 1.2.0 imports: - yorc: @@ -101,4 +101,46 @@ node_types: required: false default: false + yorc.nodes.aws.VPC: + derived_from: tosca.nodes.Network + # See https://www.terraform.io/docs/providers/aws/r/vpc.html + properties: + instance_tenancy: + type: string + description: > + You can run instances in your VPC on single-tenant, dedicated hardware. + Select Dedicated to ensure that instances launched in this VPC are dedicated tenancy instances, regardless of the tenancy attribute specified at launch. + Select Default to ensure that instances launched in this VPC use the tenancy attribute specified at launch + default: default + constraints : + - valid_values: [default, dedicated] + enable_dns_support: + type: boolean + description: A boolean flag to enable/disable DNS support in the VPC. Defaults true. + default: true + required: false + enable_dns_hostnames: + type: boolean + default: false + description : A boolean flag to enable/disable DNS hostnames in the VPC. Defaults false. + required: false + enable_classiclink: + type: boolean + default: false + description: A boolean flag to enable/disable ClassicLink for the VPC. Only valid in regions and accounts that support EC2 Classic. + required: false + enable_classiclink_dns_support: + type: boolean + required: false + description: A boolean flag to enable/disable ClassicLink DNS Support for the VPC. Only valid in regions and accounts that support EC2 Classic + assign_generated_ipv6_cidr_block: + type: boolean + required: false + description: Requests an Amazon-provided IPv6 CIDR block with a /56 prefix length for the VPC. You cannot specify the range of IP addresses, or the size of the CIDR block. + tags: + type: map + description: A mapping of tags to assign to the resource. + required: false + entry_schema: + type: string From abb3328a364626a16ee6fa28aa03035037186164 Mon Sep 17 00:00:00 2001 From: hilderic Date: Wed, 18 Mar 2020 17:22:18 +0100 Subject: [PATCH 02/23] yorc.nodes.aws.Subnet --- data/tosca/yorc-aws-types.yml | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/data/tosca/yorc-aws-types.yml b/data/tosca/yorc-aws-types.yml index b8a85cf64..c4d9db513 100644 --- a/data/tosca/yorc-aws-types.yml +++ b/data/tosca/yorc-aws-types.yml @@ -105,6 +105,9 @@ node_types: derived_from: tosca.nodes.Network # See https://www.terraform.io/docs/providers/aws/r/vpc.html properties: + cidr_block: + type: string + required: true instance_tenancy: type: string description: > @@ -144,3 +147,40 @@ node_types: entry_schema: type: string + yorc.nodes.aws.Subnet: + derived_from: tosca.nodes.Network + # See : https://www.terraform.io/docs/providers/aws/r/subnet.html + properties: + availability_zone: + type: string + required: false + description: The AZ for the subnet + availability_zone_id: + type: string + required: false + description: The AZ for the subnet + cidr_block: + type: string + required: true + ipv6_cidr_block : + type: string + required: false + map_public_ip_on_launch: + type: boolean + required: false + description: The AZ for the subnet + assign_ipv6_address_on_creation: + type: boolean + required: false + description: > + Specify true to indicate that network interfaces created in the specified subnet should be assigned an IPv6 address + vpc_id: + type: string + required: true + tags: + type: map + description: A mapping of tags to assign to the resource. + required: false + entry_schema: + type: string + From 2ac04974f1ca568f22a70f02923dc466cbaf0d02 Mon Sep 17 00:00:00 2001 From: hilderic Date: Thu, 19 Mar 2020 16:34:54 +0100 Subject: [PATCH 03/23] vpc [WIP_UNSTABLE] --- prov/terraform/aws/consul_test.go | 3 + prov/terraform/aws/testdata/simpleVPC.yaml | 20 +++++ prov/terraform/aws/vpc.go | 85 ++++++++++++++++++++++ prov/terraform/aws/vpc_test.go | 44 +++++++++++ 4 files changed, 152 insertions(+) create mode 100644 prov/terraform/aws/testdata/simpleVPC.yaml create mode 100644 prov/terraform/aws/vpc.go create mode 100644 prov/terraform/aws/vpc_test.go diff --git a/prov/terraform/aws/consul_test.go b/prov/terraform/aws/consul_test.go index 8bce7649a..5e4753243 100644 --- a/prov/terraform/aws/consul_test.go +++ b/prov/terraform/aws/consul_test.go @@ -76,5 +76,8 @@ func TestRunConsulAWSPackageTests(t *testing.T) { t.Run("simpleAWSInstanceWithPersistentDisk", func(t *testing.T) { testSimpleAWSInstanceWithPersistentDisk(t, cfg, srv) }) + t.Run("simpleVPC", func(t *testing.T) { + testSimpleVPC(t, cfg) + }) }) } diff --git a/prov/terraform/aws/testdata/simpleVPC.yaml b/prov/terraform/aws/testdata/simpleVPC.yaml new file mode 100644 index 000000000..2aa8761f0 --- /dev/null +++ b/prov/terraform/aws/testdata/simpleVPC.yaml @@ -0,0 +1,20 @@ +tosca_definitions_version: alien_dsl_2_0_0 + +metadata: + template_name: VPC_creation_test + template_version: 1.0 + template_author: tester + +description: "" + +imports: + - + - + +topology_template: + node_templates: + Network: + type: yorc.nodes.aws.VPC + properties: + cidr_block: 10.0.0.0/16 + assign_generated_ipv6_cidr_block: true \ No newline at end of file diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go new file mode 100644 index 000000000..d391699ee --- /dev/null +++ b/prov/terraform/aws/vpc.go @@ -0,0 +1,85 @@ +// Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "strings" + + "github.com/pkg/errors" + "github.com/ystia/yorc/v4/deployments" + "github.com/ystia/yorc/v4/prov/terraform/commons" +) + +func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams) error { + err := verifyThatNodeIsTypeOf(ctx, nodeParams, "yorc.nodes.aws.VPC") + if err != nil { + return err + } + + vpc := &vpc{} + + stringParams := []struct { + pAttr *string + propertyName string + mandatory bool + }{ + {&vpc.CidrBlock, "cidr_block", true}, + {&vpc.InstanceTenancy, "instance_tenancy", false}, + {&vpc.EnableDNSSupport, "enable_dns_support", false}, + {&vpc.EnableDNSHostnames, "enable_dns_hostnames", false}, + {&vpc.EnableClassiclink, "enable_classiclink", false}, + {&vpc.EnableClassiclinkDNSSupport, "enable_classiclink_dns_support", false}, + {&vpc.AssignGeneratedIpv6CidrBlock, "assign_generated_ipv6_cidr_block", false}, + } + + for _, stringParam := range stringParams { + if *stringParam.pAttr, err = deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, stringParam.propertyName, stringParam.mandatory); err != nil { + return errors.Wrapf(err, "failed to generate private network for deploymentID:%q, nodeName:%q", nodeParams.deploymentID, nodeParams.nodeName) + } + } + + // Get tags map + tagsVal, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "tags") + if tagsVal != nil && tagsVal.RawString() != "" { + d, ok := tagsVal.Value.(map[string]interface{}) + if !ok { + return errors.New("failed to retrieve tags map from Tosca Value: not expected type") + } + + vpc.Tags = make(map[string]string, len(d)) + for k, v := range d { + v, ok := v.(string) + if !ok { + return errors.Errorf("failed to retrieve string value from tags map from Tosca Value:%q not expected type", v) + } + vpc.Tags[k] = v + } + } + + // Create the name for the resource + var name = nil + if vpc.tags["Name"] != nil { + name := vpc.tags["Name"] + } else { + name := strings.ToLower(nodeParams.nodeName + "-" + instanceName) + } + + commons.AddResource(nodeParams.infrastructure, "aws_vpc", name, vpc) + + // Terraform Outputs + + return nil +} diff --git a/prov/terraform/aws/vpc_test.go b/prov/terraform/aws/vpc_test.go new file mode 100644 index 000000000..db66a4873 --- /dev/null +++ b/prov/terraform/aws/vpc_test.go @@ -0,0 +1,44 @@ +// Copyright 2018 Bull S.A.S. Atos Technologies - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "github.com/ystia/yorc/v4/config" + "github.com/ystia/yorc/v4/prov/terraform/commons" +) + +func testSimpleVPC(t *testing.T, cfg config.Configuration) { + t.Parallel() + deploymentID := loadTestYaml(t) + ctx := context.Background() + infrastructure := commons.Infrastructure{} + g := awsGenerator{} + + nodeParams := nodeParams{ + deploymentID: deploymentID, + nodeName: "Network", + infrastructure: &infrastructure, + } + + err := g.generateVPC(ctx, nodeParams) + + require.NoError(t, err, "Unexpected error attempting to generate vpc for %s", deploymentID) + require.Len(t, infrastructure.Resource["aws_vpc"], 1, "Expected one vpc") + +} From b5d5a8894db0fc7c10321b7cb39b71c86792bb70 Mon Sep 17 00:00:00 2001 From: hilderic Date: Thu, 19 Mar 2020 16:37:16 +0100 Subject: [PATCH 04/23] VPC in aws rsc --- prov/terraform/aws/resources.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/prov/terraform/aws/resources.go b/prov/terraform/aws/resources.go index bed3a7211..6fb370d57 100644 --- a/prov/terraform/aws/resources.go +++ b/prov/terraform/aws/resources.go @@ -71,3 +71,16 @@ type VolumeAttachment struct { InstanceID string `json:"instance_id,omitempty"` VolumeID string `json:"volume_id,omitempty"` } + +// VPC represents a virtual private cloud +// see : https://www.terraform.io/docs/providers/aws/r/vpc.html +type VPC struct { + CidrBlock string `json:"cidr_block,omitempty"` + InstanceTenancy string `json:"instance_tenancy,omitempty"` + EnableDNSSupport string `json:"enable_dns_support,omitempty"` + EnableDNSHostnames string `json:"enable_dns_hostnames,omitempty"` + EnableClassiclink string `json:"enable_classiclink,omitempty"` + EnableClassiclinkDNSSupport string `json:"enable_classiclink_dns_support,omitempty"` + AssignGeneratedIpv6CidrBlock string `json:"assign_generated_ipv6_cidr_block,omitempty"` + Tags string `json:"tags,omitempty"` +} From 3018a874b08cb6a7051b659f25e42f62c6e1d976 Mon Sep 17 00:00:00 2001 From: hilderic Date: Thu, 19 Mar 2020 17:27:19 +0100 Subject: [PATCH 05/23] vpc test => ok --- prov/terraform/aws/resources.go | 16 ++++++++-------- prov/terraform/aws/vpc.go | 10 +++++----- prov/terraform/aws/vpc_test.go | 11 +++++++++++ 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/prov/terraform/aws/resources.go b/prov/terraform/aws/resources.go index 6fb370d57..58b068d44 100644 --- a/prov/terraform/aws/resources.go +++ b/prov/terraform/aws/resources.go @@ -75,12 +75,12 @@ type VolumeAttachment struct { // VPC represents a virtual private cloud // see : https://www.terraform.io/docs/providers/aws/r/vpc.html type VPC struct { - CidrBlock string `json:"cidr_block,omitempty"` - InstanceTenancy string `json:"instance_tenancy,omitempty"` - EnableDNSSupport string `json:"enable_dns_support,omitempty"` - EnableDNSHostnames string `json:"enable_dns_hostnames,omitempty"` - EnableClassiclink string `json:"enable_classiclink,omitempty"` - EnableClassiclinkDNSSupport string `json:"enable_classiclink_dns_support,omitempty"` - AssignGeneratedIpv6CidrBlock string `json:"assign_generated_ipv6_cidr_block,omitempty"` - Tags string `json:"tags,omitempty"` + CidrBlock string `json:"cidr_block,omitempty"` + InstanceTenancy string `json:"instance_tenancy,omitempty"` + EnableDNSSupport string `json:"enable_dns_support,omitempty"` + EnableDNSHostnames string `json:"enable_dns_hostnames,omitempty"` + EnableClassiclink string `json:"enable_classiclink,omitempty"` + EnableClassiclinkDNSSupport string `json:"enable_classiclink_dns_support,omitempty"` + AssignGeneratedIpv6CidrBlock string `json:"assign_generated_ipv6_cidr_block,omitempty"` + Tags map[string]string `json:"tags,omitempty"` } diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go index d391699ee..92de6bfd6 100644 --- a/prov/terraform/aws/vpc.go +++ b/prov/terraform/aws/vpc.go @@ -29,7 +29,7 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams) e return err } - vpc := &vpc{} + vpc := &VPC{} stringParams := []struct { pAttr *string @@ -70,11 +70,11 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams) e } // Create the name for the resource - var name = nil - if vpc.tags["Name"] != nil { - name := vpc.tags["Name"] + var name = "" + if vpc.Tags["Name"] != "" { + name = vpc.Tags["Name"] } else { - name := strings.ToLower(nodeParams.nodeName + "-" + instanceName) + name = strings.ToLower(nodeParams.deploymentID + "_" + nodeParams.nodeName) } commons.AddResource(nodeParams.infrastructure, "aws_vpc", name, vpc) diff --git a/prov/terraform/aws/vpc_test.go b/prov/terraform/aws/vpc_test.go index db66a4873..585132c81 100644 --- a/prov/terraform/aws/vpc_test.go +++ b/prov/terraform/aws/vpc_test.go @@ -18,6 +18,7 @@ import ( "context" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/ystia/yorc/v4/config" "github.com/ystia/yorc/v4/prov/terraform/commons" @@ -29,6 +30,7 @@ func testSimpleVPC(t *testing.T, cfg config.Configuration) { ctx := context.Background() infrastructure := commons.Infrastructure{} g := awsGenerator{} + networkName := "simplevpc_network" nodeParams := nodeParams{ deploymentID: deploymentID, @@ -41,4 +43,13 @@ func testSimpleVPC(t *testing.T, cfg config.Configuration) { require.NoError(t, err, "Unexpected error attempting to generate vpc for %s", deploymentID) require.Len(t, infrastructure.Resource["aws_vpc"], 1, "Expected one vpc") + instancesMap := infrastructure.Resource["aws_vpc"].(map[string]interface{}) + require.Len(t, instancesMap, 1) + require.Contains(t, instancesMap, networkName) + + vpc, ok := instancesMap[networkName].(*VPC) + require.True(t, ok, "%s is not a VPC", networkName) + assert.Equal(t, "10.0.0.0/16", vpc.CidrBlock) + assert.Equal(t, "true", vpc.AssignGeneratedIpv6CidrBlock) + } From e312cfba55b82f48c2f1337d81e5c61a6482d934 Mon Sep 17 00:00:00 2001 From: hilderic Date: Fri, 20 Mar 2020 16:26:45 +0100 Subject: [PATCH 06/23] generating VPC => ok --- prov/terraform/aws/generator.go | 5 +++++ prov/terraform/aws/vpc.go | 14 +++++++++++--- prov/terraform/aws/vpc_test.go | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/prov/terraform/aws/generator.go b/prov/terraform/aws/generator.go index 0fa261a26..e8c712f3b 100644 --- a/prov/terraform/aws/generator.go +++ b/prov/terraform/aws/generator.go @@ -141,6 +141,11 @@ func (g *awsGenerator) generateInstances(ctx context.Context, cfg config.Configu if err != nil { return err } + case "yorc.nodes.aws.VPC": + err = g.generateVPC(ctx, *nodeParams, instanceName, outputs) + if err != nil { + return err + } default: return errors.Errorf("Unsupported node type '%s' for node '%s' in deployment '%s'", nodeType, nodeParams.nodeName, nodeParams.deploymentID) } diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go index 92de6bfd6..30fdb6e6f 100644 --- a/prov/terraform/aws/vpc.go +++ b/prov/terraform/aws/vpc.go @@ -16,14 +16,17 @@ package aws import ( "context" + "fmt" + "path" "strings" "github.com/pkg/errors" "github.com/ystia/yorc/v4/deployments" + "github.com/ystia/yorc/v4/helper/consulutil" "github.com/ystia/yorc/v4/prov/terraform/commons" ) -func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams) error { +func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams, instanceName string, outputs map[string]string) error { err := verifyThatNodeIsTypeOf(ctx, nodeParams, "yorc.nodes.aws.VPC") if err != nil { return err @@ -74,12 +77,17 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams) e if vpc.Tags["Name"] != "" { name = vpc.Tags["Name"] } else { - name = strings.ToLower(nodeParams.deploymentID + "_" + nodeParams.nodeName) + name = strings.ToLower(nodeParams.deploymentID + "-" + nodeParams.nodeName) } commons.AddResource(nodeParams.infrastructure, "aws_vpc", name, vpc) - // Terraform Outputs + // Terraform output + nodeKey := path.Join(consulutil.DeploymentKVPrefix, nodeParams.deploymentID, "topology", "instances", nodeParams.nodeName, instanceName) + idKey := nodeParams.nodeName + "-" + instanceName + "-id" + idValue := fmt.Sprintf("${aws_vpc.%s.id}", name) + commons.AddOutput(nodeParams.infrastructure, idKey, &commons.Output{Value: idValue}) + outputs[path.Join(nodeKey, "/attributes/vpc_id")] = idKey return nil } diff --git a/prov/terraform/aws/vpc_test.go b/prov/terraform/aws/vpc_test.go index 585132c81..4e114078c 100644 --- a/prov/terraform/aws/vpc_test.go +++ b/prov/terraform/aws/vpc_test.go @@ -38,7 +38,7 @@ func testSimpleVPC(t *testing.T, cfg config.Configuration) { infrastructure: &infrastructure, } - err := g.generateVPC(ctx, nodeParams) + err := g.generateVPC(ctx, nodeParams, "instance0", make(map[string]string)) require.NoError(t, err, "Unexpected error attempting to generate vpc for %s", deploymentID) require.Len(t, infrastructure.Resource["aws_vpc"], 1, "Expected one vpc") From 76acd95c206d342cfeb9793dfa00373113df7a9a Mon Sep 17 00:00:00 2001 From: hilderic Date: Mon, 23 Mar 2020 18:15:48 +0100 Subject: [PATCH 07/23] Subnet --- prov/terraform/aws/consul_test.go | 4 +- prov/terraform/aws/resources.go | 13 +++ prov/terraform/aws/testdata/simpleSubnet.yaml | 29 +++++++ prov/terraform/aws/vpc.go | 87 +++++++++++++++++++ prov/terraform/aws/vpc_test.go | 37 +++++++- 5 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 prov/terraform/aws/testdata/simpleSubnet.yaml diff --git a/prov/terraform/aws/consul_test.go b/prov/terraform/aws/consul_test.go index 5e4753243..a735aab71 100644 --- a/prov/terraform/aws/consul_test.go +++ b/prov/terraform/aws/consul_test.go @@ -19,7 +19,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "github.com/ystia/yorc/v4/locations" "github.com/ystia/yorc/v4/testutil" ) @@ -79,5 +78,8 @@ func TestRunConsulAWSPackageTests(t *testing.T) { t.Run("simpleVPC", func(t *testing.T) { testSimpleVPC(t, cfg) }) + t.Run("simpleSubnet", func(t *testing.T) { + testSimpleSubnet(t, srv, cfg) + }) }) } diff --git a/prov/terraform/aws/resources.go b/prov/terraform/aws/resources.go index 58b068d44..7691d9dbd 100644 --- a/prov/terraform/aws/resources.go +++ b/prov/terraform/aws/resources.go @@ -84,3 +84,16 @@ type VPC struct { AssignGeneratedIpv6CidrBlock string `json:"assign_generated_ipv6_cidr_block,omitempty"` Tags map[string]string `json:"tags,omitempty"` } + +// Subnet reprents a network in a VPC +// see : https://www.terraform.io/docs/providers/aws/r/subnet.html +type Subnet struct { + AvailabilityZone string `json:"availability_zone,omitempty"` + AvailabilityZoneID string `json:"availability_zone_id,omitempty"` + CidrBlock string `json:"cidr_block,omitempty"` + Ipv6CidrBlock string `json:"ipv6_cidr_block,omitempty"` + MapPublicIPOnLaunch string `json:"map_public_ip_on_launch,omitempty"` + AssignIpv6AddressOnCreation string `json:"assign_ipv6_address_on_creation,omitempty"` + VPCId string `json:"vpc_id,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} diff --git a/prov/terraform/aws/testdata/simpleSubnet.yaml b/prov/terraform/aws/testdata/simpleSubnet.yaml new file mode 100644 index 000000000..1a33e711e --- /dev/null +++ b/prov/terraform/aws/testdata/simpleSubnet.yaml @@ -0,0 +1,29 @@ +tosca_definitions_version: alien_dsl_2_0_0 + +metadata: + template_name: VPC_creation_test + template_version: 1.0 + template_author: tester + +description: "" + +imports: + - + - + +topology_template: + node_templates: + Network_VPC: + type: yorc.nodes.aws.VPC + properties: + cidr_block: 10.0.0.0/16 + Network_Subnet: + type: yorc.nodes.aws.Subnet + properties: + cidr_block: 10.0.0.0/24 + requirements: + - Network_Subnet_Network_VPC: + type_requirement: dependency + node: Network_VPC + capability: tosca.capabilities.Node + relationship: tosca.relationships.DependsOn diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go index 30fdb6e6f..edccb11a8 100644 --- a/prov/terraform/aws/vpc.go +++ b/prov/terraform/aws/vpc.go @@ -79,6 +79,8 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams, i } else { name = strings.ToLower(nodeParams.deploymentID + "-" + nodeParams.nodeName) } + // Name must respect regular expression + name = strings.Replace(strings.ToLower(name), "_", "-", -1) commons.AddResource(nodeParams.infrastructure, "aws_vpc", name, vpc) @@ -91,3 +93,88 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams, i outputs[path.Join(nodeKey, "/attributes/vpc_id")] = idKey return nil } + +func (g *awsGenerator) generateSubnet(ctx context.Context, nodeParams nodeParams, instanceName string, outputs map[string]string) error { + err := verifyThatNodeIsTypeOf(ctx, nodeParams, "yorc.nodes.aws.Subnet") + if err != nil { + return err + } + + subnet := &Subnet{} + + stringParams := []struct { + pAttr *string + propertyName string + mandatory bool + }{ + {&subnet.AvailabilityZone, "availability_zone", false}, + {&subnet.AvailabilityZoneID, "availability_zone_id", false}, + {&subnet.CidrBlock, "cidr_block", true}, + {&subnet.Ipv6CidrBlock, "ipv6_cidr_block", true}, + {&subnet.MapPublicIPOnLaunch, "map_public_ip_on_launch", false}, + {&subnet.AssignIpv6AddressOnCreation, "assign_ipv6_address_on_creation", false}, + {&subnet.VPCId, "vpc_id", false}, + } + + for _, stringParam := range stringParams { + if *stringParam.pAttr, err = deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, stringParam.propertyName, stringParam.mandatory); err != nil { + return errors.Wrapf(err, "failed to generate private network for deploymentID:%q, nodeName:%q", nodeParams.deploymentID, nodeParams.nodeName) + } + } + + // Get tags map + tagsVal, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "tags") + if tagsVal != nil && tagsVal.RawString() != "" { + d, ok := tagsVal.Value.(map[string]interface{}) + if !ok { + return errors.New("failed to retrieve tags map from Tosca Value: not expected type") + } + + subnet.Tags = make(map[string]string, len(d)) + for k, v := range d { + v, ok := v.(string) + if !ok { + return errors.Errorf("failed to retrieve string value from tags map from Tosca Value:%q not expected type", v) + } + subnet.Tags[k] = v + } + } + + // Create the name for the resource + var name = "" + if subnet.Tags["Name"] != "" { + name = subnet.Tags["Name"] + } else { + name = strings.ToLower(nodeParams.deploymentID + "-" + nodeParams.nodeName) + } + // Name must respect regular expression + name = strings.Replace(strings.ToLower(name), "_", "-", -1) + + // Get VPC ID if not given (if it's the case, it should be in a dependency relationship) + if subnet.VPCId == "" { + hasDep, vpcNodeName, err := deployments.HasAnyRequirementFromNodeType(ctx, nodeParams.deploymentID, nodeParams.nodeName, "dependency", "yorc.nodes.aws.VPC") + if err != nil { + return err + } + if !hasDep { + return errors.Errorf("failed to retrieve dependency btw any network and the subnet with name:%q", name) + } + + subnet.VPCId, err = deployments.LookupInstanceAttributeValue(ctx, nodeParams.deploymentID, vpcNodeName, instanceName, "vpc_id") + if err != nil { + return err + } + } + + commons.AddResource(nodeParams.infrastructure, "aws_subnet", name, subnet) + + // Terraform output + nodeKey := path.Join(consulutil.DeploymentKVPrefix, nodeParams.deploymentID, "topology", "instances", nodeParams.nodeName, instanceName) + + idKey := nodeParams.nodeName + "-" + instanceName + "-id" + idValue := fmt.Sprintf("${aws_subnet.%s.id}", name) + commons.AddOutput(nodeParams.infrastructure, idKey, &commons.Output{Value: idValue}) + outputs[path.Join(nodeKey, "/attributes/subnet_id")] = idKey + + return nil +} diff --git a/prov/terraform/aws/vpc_test.go b/prov/terraform/aws/vpc_test.go index 4e114078c..25c553b07 100644 --- a/prov/terraform/aws/vpc_test.go +++ b/prov/terraform/aws/vpc_test.go @@ -16,11 +16,14 @@ package aws import ( "context" + "path" "testing" + "github.com/hashicorp/consul/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/ystia/yorc/v4/config" + "github.com/ystia/yorc/v4/helper/consulutil" "github.com/ystia/yorc/v4/prov/terraform/commons" ) @@ -30,7 +33,7 @@ func testSimpleVPC(t *testing.T, cfg config.Configuration) { ctx := context.Background() infrastructure := commons.Infrastructure{} g := awsGenerator{} - networkName := "simplevpc_network" + networkName := "simplevpc-network" nodeParams := nodeParams{ deploymentID: deploymentID, @@ -53,3 +56,35 @@ func testSimpleVPC(t *testing.T, cfg config.Configuration) { assert.Equal(t, "true", vpc.AssignGeneratedIpv6CidrBlock) } + +func testSimpleSubnet(t *testing.T, srv1 *testutil.TestServer, cfg config.Configuration) { + t.Parallel() + deploymentID := loadTestYaml(t) + ctx := context.Background() + infrastructure := commons.Infrastructure{} + g := awsGenerator{} + subnetName := "simplesubnet-network-subnet" // deploymentID + "-" + nodeName (with _ changed to -) + + nodeParams := nodeParams{ + deploymentID: deploymentID, + nodeName: "Network_Subnet", + infrastructure: &infrastructure, + } + + srv1.PopulateKV(t, map[string][]byte{ + path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/instances/Network_VPC/0/attributes/vpc_id"): []byte("vpc_id"), + }) + + err := g.generateSubnet(ctx, nodeParams, "0", make(map[string]string)) + require.NoError(t, err, "Unexpected error attempting to generate sub-network for %s", deploymentID) + + instancesMap := infrastructure.Resource["aws_subnet"].(map[string]interface{}) + require.Len(t, instancesMap, 1) + require.Contains(t, instancesMap, subnetName) + + subnet, ok := instancesMap[subnetName].(*Subnet) + require.True(t, ok, "%s is not a Subnet", subnetName) + assert.Equal(t, "10.0.0.0/24", subnet.CidrBlock) + assert.Equal(t, "vpc_id", subnet.VPCId) + +} From 5f6f7ae5aeb5fc7893f14431ed191f1743b7b0ec Mon Sep 17 00:00:00 2001 From: hilderic Date: Wed, 25 Mar 2020 15:18:19 +0100 Subject: [PATCH 08/23] Adding networkInterfaces --- prov/terraform/aws/aws_instance.go | 25 ++++++++ prov/terraform/aws/aws_instance_test.go | 30 ++++++++++ prov/terraform/aws/consul_test.go | 3 + prov/terraform/aws/resources.go | 27 ++++++--- .../simpleAWSInstanceWithVPCandSubnet.yaml | 59 +++++++++++++++++++ 5 files changed, 135 insertions(+), 9 deletions(-) create mode 100644 prov/terraform/aws/testdata/simpleAWSInstanceWithVPCandSubnet.yaml diff --git a/prov/terraform/aws/aws_instance.go b/prov/terraform/aws/aws_instance.go index 39f0dc54c..0e9f03fcb 100644 --- a/prov/terraform/aws/aws_instance.go +++ b/prov/terraform/aws/aws_instance.go @@ -141,6 +141,31 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi if placementGroup != nil { instance.PlacementGroup = placementGroup.RawString() } + + // Network Interfaces + netInterfaces := []NetworkInterface{} + hasNetwork, _, err := deployments.HasAnyRequirementFromNodeType(ctx, deploymentID, nodeName, "network", "yorc.nodes.aws.Subnet") + if err != nil { + return err + } + if hasNetwork { + networkRequirements, err := deployments.GetRequirementsByTypeForNode(ctx, deploymentID, nodeName, "network") + if err != nil { + return err + } + + for _, networkReq := range networkRequirements { + networkInterface := &NetworkInterface{} + networkInterface.SubnetID, err = deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "subnet_id") + if err != nil { + return err + } + netInterfaces = append(netInterfaces, *networkInterface) + } + + } + instance.NetworkInterfaces = netInterfaces + // Add the AWS instance commons.AddResource(infrastructure, "aws_instance", instance.Tags.Name, &instance) diff --git a/prov/terraform/aws/aws_instance_test.go b/prov/terraform/aws/aws_instance_test.go index ce99a7299..035f9f841 100644 --- a/prov/terraform/aws/aws_instance_test.go +++ b/prov/terraform/aws/aws_instance_test.go @@ -373,3 +373,33 @@ func testSimpleAWSInstanceWithPersistentDisk(t *testing.T, cfg config.Configurat assert.Equal(t, "/dev/sdf", attachedDisk.DeviceName) } + +func simpleAWSInstanceWithVPCandSubnet(t *testing.T, cfg config.Configuration, srv *testutil.TestServer) { + t.Parallel() + + deploymentID := loadTestYaml(t) + g := awsGenerator{} + infrastructure := commons.Infrastructure{} + ctx := context.Background() + env := make([]string, 0) + outputs := make(map[string]string) + + srv.PopulateKV(t, map[string][]byte{ + path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/nodes/Network_Subnet/type"): []byte("yorc.nodes.aws.Subnet"), + path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/instances/Network_Subnet/0/attributes/subnet_id"): []byte("subnet_id"), + }) + + err := g.generateAWSInstance(ctx, cfg, deploymentID, "AWS_Compute", "0", &infrastructure, outputs, &env) + require.Nil(t, err) + require.Len(t, infrastructure.Resource["aws_instance"], 1) + + instancesMap := infrastructure.Resource["aws_instance"].(map[string]interface{}) + require.Contains(t, instancesMap, "AWS_Compute-0") + + compute, ok := instancesMap["AWS_Compute-0"].(*ComputeInstance) + require.True(t, ok, "%s is not a ComputeInstance", "AWS_Compute-0") + + assert.Len(t, compute.NetworkInterfaces, 1, "1 NetworkInterface expected") + assert.Equal(t, "subnet_id", compute.NetworkInterfaces[0].SubnetID, "Subnetwork is not retrieved") + +} diff --git a/prov/terraform/aws/consul_test.go b/prov/terraform/aws/consul_test.go index a735aab71..366c616e0 100644 --- a/prov/terraform/aws/consul_test.go +++ b/prov/terraform/aws/consul_test.go @@ -63,6 +63,9 @@ func TestRunConsulAWSPackageTests(t *testing.T) { t.Run("simpleAWSInstanceWithMalformedEIP", func(t *testing.T) { testSimpleAWSInstanceWithMalformedEIP(t, cfg) }) + t.Run("simpleAWSInstanceWithVPCandSubnet", func(t *testing.T) { + simpleAWSInstanceWithVPCandSubnet(t, cfg, srv) + }) t.Run("generateTerraformInfraForAWSNode", func(t *testing.T) { testGenerateTerraformInfraForAWSNode(t, cfg, locationMgr) }) diff --git a/prov/terraform/aws/resources.go b/prov/terraform/aws/resources.go index 7691d9dbd..707797b74 100644 --- a/prov/terraform/aws/resources.go +++ b/prov/terraform/aws/resources.go @@ -16,15 +16,16 @@ package aws // A ComputeInstance represent an AWS compute type ComputeInstance struct { - ImageID string `json:"ami,omitempty"` - InstanceType string `json:"instance_type,omitempty"` - AvailabilityZone string `json:"availability_zone,omitempty"` - PlacementGroup string `json:"placement_group,omitempty"` - SecurityGroups []string `json:"security_groups,omitempty"` - KeyName string `json:"key_name,omitempty"` - Tags Tags `json:"tags,omitempty"` - ElasticIps []string `json:"-"` - RootBlockDevice BlockDevice `json:"root_block_device,omitempty"` + ImageID string `json:"ami,omitempty"` + InstanceType string `json:"instance_type,omitempty"` + AvailabilityZone string `json:"availability_zone,omitempty"` + PlacementGroup string `json:"placement_group,omitempty"` + SecurityGroups []string `json:"security_groups,omitempty"` + KeyName string `json:"key_name,omitempty"` + Tags Tags `json:"tags,omitempty"` + ElasticIps []string `json:"-"` + RootBlockDevice BlockDevice `json:"root_block_device,omitempty"` + NetworkInterfaces []NetworkInterface `json:"network_interface,omitempty"` Provisioners map[string]interface{} `json:"provisioner,omitempty"` } @@ -97,3 +98,11 @@ type Subnet struct { VPCId string `json:"vpc_id,omitempty"` Tags map[string]string `json:"tags,omitempty"` } + +// NetworkInterface create an ENI (Elastic Network Interface) +// see : https://www.terraform.io/docs/providers/aws/r/network_interface.html#attachment +type NetworkInterface struct { + SubnetID string `json:"subnet_id,omitempty"` + Description string `json:"description,omitempty"` + PrivateIps string `json:"private_ips,omitempty"` +} diff --git a/prov/terraform/aws/testdata/simpleAWSInstanceWithVPCandSubnet.yaml b/prov/terraform/aws/testdata/simpleAWSInstanceWithVPCandSubnet.yaml new file mode 100644 index 000000000..a698ee006 --- /dev/null +++ b/prov/terraform/aws/testdata/simpleAWSInstanceWithVPCandSubnet.yaml @@ -0,0 +1,59 @@ +tosca_definitions_version: alien_dsl_2_0_0 + +metadata: + template_name: NetworkWithCustomSubnetsTest + template_version: 1.0 + template_author: tester + +description: "" + +imports: + - + - + - + +topology_template: + node_templates: + Network_VPC: + type: yorc.nodes.aws.VPC + properties: + cidr_block: 10.0.0.0/16 + AWS_Compute: + type: yorc.nodes.aws.Compute + properties: + image_id: "ami-16dffe73" + instance_type: "t2.micro" + key_name: "yorc-keypair" + security_groups: "yorc-securityGroup" + availability_zone: "us-east-2c" + placement_group: "myPlacement" + requirements: + - AWS_Compute_network: + type_requirement: network + node: Network_Subnet + capability: tosca.capabilities.Connectivity + relationship: tosca.relationships.Network + capabilities: + scalable: + properties: + min_instances: 1 + max_instances: 1 + default_instances: 1 + endpoint: + properties: + secure: true + protocol: tcp + network_name: PRIVATE + initiator: source + credentials: {user: centos} + Network_Subnet: + type: yorc.nodes.aws.Subnet + properties: + cidr_block: 10.10.0.0/24 + availability_zone: "us-east-2c" + requirements: + - VPC_Requirements: + type_requirement: dependency + node: Network_VPC + capability: tosca.capabilities.Node + relationship: tosca.relationships.DependsOn \ No newline at end of file From 24106e6dda435575eb1d90d0a823abf8e71a6f37 Mon Sep 17 00:00:00 2001 From: hilderic Date: Fri, 27 Mar 2020 17:21:08 +0100 Subject: [PATCH 09/23] ENI creation => OK --- prov/terraform/aws/aws_instance.go | 17 ++++++++++++++++- prov/terraform/aws/generator.go | 6 ++++++ prov/terraform/aws/resources.go | 8 +++++--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/prov/terraform/aws/aws_instance.go b/prov/terraform/aws/aws_instance.go index 0e9f03fcb..1c6f76b1b 100644 --- a/prov/terraform/aws/aws_instance.go +++ b/prov/terraform/aws/aws_instance.go @@ -154,17 +154,32 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi return err } + // Start at 1, (0 reserved for the default network interface) + i := 1 for _, networkReq := range networkRequirements { networkInterface := &NetworkInterface{} networkInterface.SubnetID, err = deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "subnet_id") if err != nil { return err } + networkInterface.SecurityGroups = instance.SecurityGroups netInterfaces = append(netInterfaces, *networkInterface) + name := strings.ToLower("network-inteface-" + strconv.Itoa(i)) + name = strings.Replace(strings.ToLower(name), "_", "-", -1) + commons.AddResource(infrastructure, "aws_network_interface", name, networkInterface) + + networkInterface.Attachment = map[string]string{ + "instance": "${aws_instance." + instance.Tags.Name + ".id}", + "device_index": strconv.Itoa(i), + } + + i++ } + // No security groups on instance level when defining customs ENI + // (So the instance will have be in the default security groups on creation) + instance.SecurityGroups = nil } - instance.NetworkInterfaces = netInterfaces // Add the AWS instance commons.AddResource(infrastructure, "aws_instance", instance.Tags.Name, &instance) diff --git a/prov/terraform/aws/generator.go b/prov/terraform/aws/generator.go index e8c712f3b..35cf02096 100644 --- a/prov/terraform/aws/generator.go +++ b/prov/terraform/aws/generator.go @@ -146,6 +146,12 @@ func (g *awsGenerator) generateInstances(ctx context.Context, cfg config.Configu if err != nil { return err } + + case "yorc.nodes.aws.Subnet": + err = g.generateSubnet(ctx, *nodeParams, instanceName, outputs) + if err != nil { + return err + } default: return errors.Errorf("Unsupported node type '%s' for node '%s' in deployment '%s'", nodeType, nodeParams.nodeName, nodeParams.deploymentID) } diff --git a/prov/terraform/aws/resources.go b/prov/terraform/aws/resources.go index 707797b74..acd04f3b9 100644 --- a/prov/terraform/aws/resources.go +++ b/prov/terraform/aws/resources.go @@ -102,7 +102,9 @@ type Subnet struct { // NetworkInterface create an ENI (Elastic Network Interface) // see : https://www.terraform.io/docs/providers/aws/r/network_interface.html#attachment type NetworkInterface struct { - SubnetID string `json:"subnet_id,omitempty"` - Description string `json:"description,omitempty"` - PrivateIps string `json:"private_ips,omitempty"` + SubnetID string `json:"subnet_id,omitempty"` + Description string `json:"description,omitempty"` + PrivateIps string `json:"private_ips,omitempty"` + SecurityGroups []string `json:"security_groups,omitempty"` + Attachment map[string]string `json:"attachment,omitempty"` } From c153828492237bd14c814f4eb1e6034d95be8404 Mon Sep 17 00:00:00 2001 From: hilderic Date: Fri, 27 Mar 2020 18:36:28 +0100 Subject: [PATCH 10/23] fix for connectionCheck --- prov/terraform/aws/aws_instance.go | 19 +++++++++++++------ prov/terraform/aws/resources.go | 20 ++++++++++---------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/prov/terraform/aws/aws_instance.go b/prov/terraform/aws/aws_instance.go index 1c6f76b1b..67a192f76 100644 --- a/prov/terraform/aws/aws_instance.go +++ b/prov/terraform/aws/aws_instance.go @@ -143,7 +143,6 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi } // Network Interfaces - netInterfaces := []NetworkInterface{} hasNetwork, _, err := deployments.HasAnyRequirementFromNodeType(ctx, deploymentID, nodeName, "network", "yorc.nodes.aws.Subnet") if err != nil { return err @@ -155,7 +154,7 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi } // Start at 1, (0 reserved for the default network interface) - i := 1 + i := 0 for _, networkReq := range networkRequirements { networkInterface := &NetworkInterface{} networkInterface.SubnetID, err = deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "subnet_id") @@ -163,14 +162,22 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi return err } networkInterface.SecurityGroups = instance.SecurityGroups - netInterfaces = append(netInterfaces, *networkInterface) name := strings.ToLower("network-inteface-" + strconv.Itoa(i)) name = strings.Replace(strings.ToLower(name), "_", "-", -1) commons.AddResource(infrastructure, "aws_network_interface", name, networkInterface) - networkInterface.Attachment = map[string]string{ - "instance": "${aws_instance." + instance.Tags.Name + ".id}", - "device_index": strconv.Itoa(i), + // First interface will be considered as default one + if i == 0 { + instance.NetworkInterface = map[string]string{ + "network_interface_id": "${aws_network_interface." + name + ".id}", + "device_index": strconv.Itoa(i), + } + } else { + // Others interfaces are considered attachment + networkInterface.Attachment = map[string]string{ + "instance": "${aws_instance." + instance.Tags.Name + ".id}", + "device_index": strconv.Itoa(i), + } } i++ diff --git a/prov/terraform/aws/resources.go b/prov/terraform/aws/resources.go index acd04f3b9..f1de69ebd 100644 --- a/prov/terraform/aws/resources.go +++ b/prov/terraform/aws/resources.go @@ -16,16 +16,16 @@ package aws // A ComputeInstance represent an AWS compute type ComputeInstance struct { - ImageID string `json:"ami,omitempty"` - InstanceType string `json:"instance_type,omitempty"` - AvailabilityZone string `json:"availability_zone,omitempty"` - PlacementGroup string `json:"placement_group,omitempty"` - SecurityGroups []string `json:"security_groups,omitempty"` - KeyName string `json:"key_name,omitempty"` - Tags Tags `json:"tags,omitempty"` - ElasticIps []string `json:"-"` - RootBlockDevice BlockDevice `json:"root_block_device,omitempty"` - NetworkInterfaces []NetworkInterface `json:"network_interface,omitempty"` + ImageID string `json:"ami,omitempty"` + InstanceType string `json:"instance_type,omitempty"` + AvailabilityZone string `json:"availability_zone,omitempty"` + PlacementGroup string `json:"placement_group,omitempty"` + SecurityGroups []string `json:"security_groups,omitempty"` + KeyName string `json:"key_name,omitempty"` + Tags Tags `json:"tags,omitempty"` + ElasticIps []string `json:"-"` + RootBlockDevice BlockDevice `json:"root_block_device,omitempty"` + NetworkInterface map[string]string `json:"network_interface,omitempty"` Provisioners map[string]interface{} `json:"provisioner,omitempty"` } From 0424e32063793d880549619b6ef58e7370e19310 Mon Sep 17 00:00:00 2001 From: hilderic Date: Mon, 30 Mar 2020 15:12:51 +0200 Subject: [PATCH 11/23] fix test --- prov/terraform/aws/aws_instance.go | 3 ++- prov/terraform/aws/aws_instance_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/prov/terraform/aws/aws_instance.go b/prov/terraform/aws/aws_instance.go index 67a192f76..45a07479a 100644 --- a/prov/terraform/aws/aws_instance.go +++ b/prov/terraform/aws/aws_instance.go @@ -164,7 +164,6 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi networkInterface.SecurityGroups = instance.SecurityGroups name := strings.ToLower("network-inteface-" + strconv.Itoa(i)) name = strings.Replace(strings.ToLower(name), "_", "-", -1) - commons.AddResource(infrastructure, "aws_network_interface", name, networkInterface) // First interface will be considered as default one if i == 0 { @@ -180,6 +179,8 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi } } + commons.AddResource(infrastructure, "aws_network_interface", name, networkInterface) + i++ } diff --git a/prov/terraform/aws/aws_instance_test.go b/prov/terraform/aws/aws_instance_test.go index 035f9f841..23046c47f 100644 --- a/prov/terraform/aws/aws_instance_test.go +++ b/prov/terraform/aws/aws_instance_test.go @@ -399,7 +399,7 @@ func simpleAWSInstanceWithVPCandSubnet(t *testing.T, cfg config.Configuration, s compute, ok := instancesMap["AWS_Compute-0"].(*ComputeInstance) require.True(t, ok, "%s is not a ComputeInstance", "AWS_Compute-0") - assert.Len(t, compute.NetworkInterfaces, 1, "1 NetworkInterface expected") - assert.Equal(t, "subnet_id", compute.NetworkInterfaces[0].SubnetID, "Subnetwork is not retrieved") + assert.Equal(t, "${aws_network_interface.network-inteface-0.id}", compute.NetworkInterface["network_interface_id"], "Subnetwork is not retrieved") + assert.Equal(t, "0", compute.NetworkInterface["device_index"], "Subnetwork is not retrieved") } From 51961bf38c8c8f2bfd77029f34d3fdeece40ed50 Mon Sep 17 00:00:00 2001 From: hilderic Date: Mon, 30 Mar 2020 15:26:32 +0200 Subject: [PATCH 12/23] getTagsMap fn added --- prov/terraform/aws/utils.go | 26 ++++++++++++++++++++++++++ prov/terraform/aws/vpc.go | 37 ++++++------------------------------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/prov/terraform/aws/utils.go b/prov/terraform/aws/utils.go index a6304bd46..22b63d5ee 100644 --- a/prov/terraform/aws/utils.go +++ b/prov/terraform/aws/utils.go @@ -16,6 +16,7 @@ package aws import ( "context" + "github.com/pkg/errors" "github.com/ystia/yorc/v4/deployments" ) @@ -30,3 +31,28 @@ func verifyThatNodeIsTypeOf(ctx context.Context, nodeParams nodeParams, nodeType } return nil } + +func getTagsMap(ctx context.Context, nodeParams nodeParams) (map[string]string, error) { + tagsVal, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "tags") + if err != nil { + return nil, err + } + var tags map[string]string + if tagsVal != nil && tagsVal.RawString() != "" { + d, ok := tagsVal.Value.(map[string]interface{}) + if !ok { + return nil, errors.New("failed to retrieve tags map from Tosca Value: not expected type") + } + + tags = make(map[string]string, len(d)) + for k, v := range d { + v, ok := v.(string) + if !ok { + return nil, errors.Errorf("failed to retrieve string value from tags map from Tosca Value:%q not expected type", v) + } + tags[k] = v + } + } + + return tags, nil +} diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go index edccb11a8..01da4956a 100644 --- a/prov/terraform/aws/vpc.go +++ b/prov/terraform/aws/vpc.go @@ -55,21 +55,9 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams, i } // Get tags map - tagsVal, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "tags") - if tagsVal != nil && tagsVal.RawString() != "" { - d, ok := tagsVal.Value.(map[string]interface{}) - if !ok { - return errors.New("failed to retrieve tags map from Tosca Value: not expected type") - } - - vpc.Tags = make(map[string]string, len(d)) - for k, v := range d { - v, ok := v.(string) - if !ok { - return errors.Errorf("failed to retrieve string value from tags map from Tosca Value:%q not expected type", v) - } - vpc.Tags[k] = v - } + vpc.Tags, err = getTagsMap(ctx, nodeParams) + if err != nil { + return nil } // Create the name for the resource @@ -122,22 +110,9 @@ func (g *awsGenerator) generateSubnet(ctx context.Context, nodeParams nodeParams } } - // Get tags map - tagsVal, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "tags") - if tagsVal != nil && tagsVal.RawString() != "" { - d, ok := tagsVal.Value.(map[string]interface{}) - if !ok { - return errors.New("failed to retrieve tags map from Tosca Value: not expected type") - } - - subnet.Tags = make(map[string]string, len(d)) - for k, v := range d { - v, ok := v.(string) - if !ok { - return errors.Errorf("failed to retrieve string value from tags map from Tosca Value:%q not expected type", v) - } - subnet.Tags[k] = v - } + subnet.Tags, err = getTagsMap(ctx, nodeParams) + if err != nil { + return nil } // Create the name for the resource From 68dd3ac102a3076b6cebdedf5550208d11311ded Mon Sep 17 00:00:00 2001 From: hilderic Date: Mon, 30 Mar 2020 16:16:39 +0200 Subject: [PATCH 13/23] fix string to bool params --- prov/terraform/aws/resources.go | 14 +-- prov/terraform/aws/testdata/simpleVPC.yaml | 3 +- prov/terraform/aws/vpc.go | 126 ++++++++++++++------- prov/terraform/aws/vpc_test.go | 4 +- 4 files changed, 98 insertions(+), 49 deletions(-) diff --git a/prov/terraform/aws/resources.go b/prov/terraform/aws/resources.go index f1de69ebd..74cf41585 100644 --- a/prov/terraform/aws/resources.go +++ b/prov/terraform/aws/resources.go @@ -78,11 +78,11 @@ type VolumeAttachment struct { type VPC struct { CidrBlock string `json:"cidr_block,omitempty"` InstanceTenancy string `json:"instance_tenancy,omitempty"` - EnableDNSSupport string `json:"enable_dns_support,omitempty"` - EnableDNSHostnames string `json:"enable_dns_hostnames,omitempty"` - EnableClassiclink string `json:"enable_classiclink,omitempty"` - EnableClassiclinkDNSSupport string `json:"enable_classiclink_dns_support,omitempty"` - AssignGeneratedIpv6CidrBlock string `json:"assign_generated_ipv6_cidr_block,omitempty"` + EnableDNSSupport bool `json:"enable_dns_support,omitempty"` + EnableDNSHostnames bool `json:"enable_dns_hostnames,omitempty"` + EnableClassiclink bool `json:"enable_classiclink,omitempty"` + EnableClassiclinkDNSSupport bool `json:"enable_classiclink_dns_support,omitempty"` + AssignGeneratedIpv6CidrBlock bool `json:"assign_generated_ipv6_cidr_block,omitempty"` Tags map[string]string `json:"tags,omitempty"` } @@ -93,8 +93,8 @@ type Subnet struct { AvailabilityZoneID string `json:"availability_zone_id,omitempty"` CidrBlock string `json:"cidr_block,omitempty"` Ipv6CidrBlock string `json:"ipv6_cidr_block,omitempty"` - MapPublicIPOnLaunch string `json:"map_public_ip_on_launch,omitempty"` - AssignIpv6AddressOnCreation string `json:"assign_ipv6_address_on_creation,omitempty"` + MapPublicIPOnLaunch bool `json:"map_public_ip_on_launch,omitempty"` + AssignIpv6AddressOnCreation bool `json:"assign_ipv6_address_on_creation,omitempty"` VPCId string `json:"vpc_id,omitempty"` Tags map[string]string `json:"tags,omitempty"` } diff --git a/prov/terraform/aws/testdata/simpleVPC.yaml b/prov/terraform/aws/testdata/simpleVPC.yaml index 2aa8761f0..72ba78430 100644 --- a/prov/terraform/aws/testdata/simpleVPC.yaml +++ b/prov/terraform/aws/testdata/simpleVPC.yaml @@ -17,4 +17,5 @@ topology_template: type: yorc.nodes.aws.VPC properties: cidr_block: 10.0.0.0/16 - assign_generated_ipv6_cidr_block: true \ No newline at end of file + assign_generated_ipv6_cidr_block: true + tags: {"tag1": "foo"} \ No newline at end of file diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go index 01da4956a..306f9f50f 100644 --- a/prov/terraform/aws/vpc.go +++ b/prov/terraform/aws/vpc.go @@ -33,25 +33,9 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams, i } vpc := &VPC{} - - stringParams := []struct { - pAttr *string - propertyName string - mandatory bool - }{ - {&vpc.CidrBlock, "cidr_block", true}, - {&vpc.InstanceTenancy, "instance_tenancy", false}, - {&vpc.EnableDNSSupport, "enable_dns_support", false}, - {&vpc.EnableDNSHostnames, "enable_dns_hostnames", false}, - {&vpc.EnableClassiclink, "enable_classiclink", false}, - {&vpc.EnableClassiclinkDNSSupport, "enable_classiclink_dns_support", false}, - {&vpc.AssignGeneratedIpv6CidrBlock, "assign_generated_ipv6_cidr_block", false}, - } - - for _, stringParam := range stringParams { - if *stringParam.pAttr, err = deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, stringParam.propertyName, stringParam.mandatory); err != nil { - return errors.Wrapf(err, "failed to generate private network for deploymentID:%q, nodeName:%q", nodeParams.deploymentID, nodeParams.nodeName) - } + err = g.getVPCProperties(ctx, nodeParams, vpc) + if err != nil { + return err } // Get tags map @@ -89,25 +73,9 @@ func (g *awsGenerator) generateSubnet(ctx context.Context, nodeParams nodeParams } subnet := &Subnet{} - - stringParams := []struct { - pAttr *string - propertyName string - mandatory bool - }{ - {&subnet.AvailabilityZone, "availability_zone", false}, - {&subnet.AvailabilityZoneID, "availability_zone_id", false}, - {&subnet.CidrBlock, "cidr_block", true}, - {&subnet.Ipv6CidrBlock, "ipv6_cidr_block", true}, - {&subnet.MapPublicIPOnLaunch, "map_public_ip_on_launch", false}, - {&subnet.AssignIpv6AddressOnCreation, "assign_ipv6_address_on_creation", false}, - {&subnet.VPCId, "vpc_id", false}, - } - - for _, stringParam := range stringParams { - if *stringParam.pAttr, err = deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, stringParam.propertyName, stringParam.mandatory); err != nil { - return errors.Wrapf(err, "failed to generate private network for deploymentID:%q, nodeName:%q", nodeParams.deploymentID, nodeParams.nodeName) - } + err = g.getSubnetProperties(ctx, nodeParams, subnet) + if err != nil { + return err } subnet.Tags, err = getTagsMap(ctx, nodeParams) @@ -125,7 +93,7 @@ func (g *awsGenerator) generateSubnet(ctx context.Context, nodeParams nodeParams // Name must respect regular expression name = strings.Replace(strings.ToLower(name), "_", "-", -1) - // Get VPC ID if not given (if it's the case, it should be in a dependency relationship) + // If not VPCId defined, it should be a dependancy, find it if subnet.VPCId == "" { hasDep, vpcNodeName, err := deployments.HasAnyRequirementFromNodeType(ctx, nodeParams.deploymentID, nodeParams.nodeName, "dependency", "yorc.nodes.aws.VPC") if err != nil { @@ -153,3 +121,83 @@ func (g *awsGenerator) generateSubnet(ctx context.Context, nodeParams nodeParams return nil } + +func (g *awsGenerator) getVPCProperties(ctx context.Context, nodeParams nodeParams, vpc *VPC) error { + stringParams := []struct { + pAttr *string + propertyName string + mandatory bool + }{ + {&vpc.CidrBlock, "cidr_block", true}, + {&vpc.InstanceTenancy, "instance_tenancy", false}, + } + + for _, stringParam := range stringParams { + val, err := deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, stringParam.propertyName, stringParam.mandatory) + if err != nil { + return err + } + *stringParam.pAttr = val + } + + // Get bool properties + boolProps := []struct { + pAttr *bool + propertyName string + }{ + {&vpc.EnableDNSSupport, "enable_dns_support"}, + {&vpc.EnableDNSHostnames, "enable_dns_hostnames"}, + {&vpc.EnableClassiclink, "enable_classiclink"}, + {&vpc.EnableClassiclinkDNSSupport, "enable_classiclink_dns_support"}, + {&vpc.AssignGeneratedIpv6CidrBlock, "assign_generated_ipv6_cidr_block"}, + } + for _, boolProps := range boolProps { + val, err := deployments.GetBooleanNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, boolProps.propertyName) + if err != nil { + return err + } + *boolProps.pAttr = val + } + + return nil +} + +func (g *awsGenerator) getSubnetProperties(ctx context.Context, nodeParams nodeParams, subnet *Subnet) error { + stringParams := []struct { + pAttr *string + propertyName string + mandatory bool + }{ + {&subnet.AvailabilityZone, "availability_zone", false}, + {&subnet.AvailabilityZoneID, "availability_zone_id", false}, + {&subnet.CidrBlock, "cidr_block", true}, + {&subnet.Ipv6CidrBlock, "ipv6_cidr_block", true}, + {&subnet.VPCId, "vpc_id", false}, + } + + for _, stringParam := range stringParams { + val, err := deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, stringParam.propertyName, stringParam.mandatory) + if err != nil { + return err + } + *stringParam.pAttr = val + } + + // Get bool properties + boolProps := []struct { + pAttr *bool + propertyName string + }{ + {&subnet.AssignIpv6AddressOnCreation, "assign_ipv6_address_on_creation"}, + {&subnet.MapPublicIPOnLaunch, "map_public_ip_on_launch"}, + } + for _, boolProps := range boolProps { + val, err := deployments.GetBooleanNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, boolProps.propertyName) + if err != nil { + return err + } + *boolProps.pAttr = val + } + + return nil +} diff --git a/prov/terraform/aws/vpc_test.go b/prov/terraform/aws/vpc_test.go index 25c553b07..9801882a1 100644 --- a/prov/terraform/aws/vpc_test.go +++ b/prov/terraform/aws/vpc_test.go @@ -53,7 +53,8 @@ func testSimpleVPC(t *testing.T, cfg config.Configuration) { vpc, ok := instancesMap[networkName].(*VPC) require.True(t, ok, "%s is not a VPC", networkName) assert.Equal(t, "10.0.0.0/16", vpc.CidrBlock) - assert.Equal(t, "true", vpc.AssignGeneratedIpv6CidrBlock) + assert.Equal(t, true, vpc.AssignGeneratedIpv6CidrBlock) + assert.Equal(t, "foo", vpc.Tags["tag1"]) } @@ -86,5 +87,4 @@ func testSimpleSubnet(t *testing.T, srv1 *testutil.TestServer, cfg config.Config require.True(t, ok, "%s is not a Subnet", subnetName) assert.Equal(t, "10.0.0.0/24", subnet.CidrBlock) assert.Equal(t, "vpc_id", subnet.VPCId) - } From c40d9a4b70357fcf9d84a57f4a42dfc51eb0a192 Mon Sep 17 00:00:00 2001 From: hilderic Date: Fri, 3 Apr 2020 09:08:35 +0200 Subject: [PATCH 14/23] yorc.nodes.aws.Subnet fixes --- data/tosca/yorc-aws-types.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/data/tosca/yorc-aws-types.yml b/data/tosca/yorc-aws-types.yml index c4d9db513..a60036c24 100644 --- a/data/tosca/yorc-aws-types.yml +++ b/data/tosca/yorc-aws-types.yml @@ -158,17 +158,18 @@ node_types: availability_zone_id: type: string required: false - description: The AZ for the subnet + description: The AZ ID for the subnet cidr_block: type: string required: true ipv6_cidr_block : type: string required: false + description: The IPv6 network range for the subnet, in CIDR notation. The subnet size must use a /64 prefix length. map_public_ip_on_launch: type: boolean required: false - description: The AZ for the subnet + description: Specify true to indicate that instances launched into the subnet should be assigned a public IP address. assign_ipv6_address_on_creation: type: boolean required: false @@ -176,7 +177,7 @@ node_types: Specify true to indicate that network interfaces created in the specified subnet should be assigned an IPv6 address vpc_id: type: string - required: true + required: false tags: type: map description: A mapping of tags to assign to the resource. From 5b7793df5ebc1fdd2de1e8b383d32ad9d478e1e4 Mon Sep 17 00:00:00 2001 From: hilderic Date: Tue, 7 Apr 2020 13:14:37 +0200 Subject: [PATCH 15/23] Instance can have an already existing subnet --- data/tosca/yorc-aws-types.yml | 5 +++++ prov/terraform/aws/aws_instance.go | 10 ++++++++-- prov/terraform/aws/aws_instance_test.go | 1 + prov/terraform/aws/resources.go | 1 + prov/terraform/aws/testdata/simpleAWSInstance.yaml | 1 + 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/data/tosca/yorc-aws-types.yml b/data/tosca/yorc-aws-types.yml index a60036c24..e8061151e 100644 --- a/data/tosca/yorc-aws-types.yml +++ b/data/tosca/yorc-aws-types.yml @@ -30,6 +30,11 @@ node_types: description: > Comma-separated list of security groups to add to the Compute required: true + subnet_id: + type: string + description: > + The VPC Subnet ID to launch in. + required: false availability_zone: type: string required: false diff --git a/prov/terraform/aws/aws_instance.go b/prov/terraform/aws/aws_instance.go index 45a07479a..ecd1811cc 100644 --- a/prov/terraform/aws/aws_instance.go +++ b/prov/terraform/aws/aws_instance.go @@ -142,7 +142,14 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi instance.PlacementGroup = placementGroup.RawString() } - // Network Interfaces + // Network + subnetID, err := deployments.GetNodePropertyValue(ctx, deploymentID, nodeName, "subnet_id") + if err != nil { + return err + } + if subnetID != nil { + instance.SubnetID = subnetID.RawString() + } hasNetwork, _, err := deployments.HasAnyRequirementFromNodeType(ctx, deploymentID, nodeName, "network", "yorc.nodes.aws.Subnet") if err != nil { return err @@ -153,7 +160,6 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi return err } - // Start at 1, (0 reserved for the default network interface) i := 0 for _, networkReq := range networkRequirements { networkInterface := &NetworkInterface{} diff --git a/prov/terraform/aws/aws_instance_test.go b/prov/terraform/aws/aws_instance_test.go index 23046c47f..ee3b3c087 100644 --- a/prov/terraform/aws/aws_instance_test.go +++ b/prov/terraform/aws/aws_instance_test.go @@ -101,6 +101,7 @@ func testSimpleAWSInstance(t *testing.T, cfg config.Configuration) { require.Equal(t, "ComputeAWS-0", compute.Tags.Name) require.Equal(t, "us-east-2c", compute.AvailabilityZone) require.Equal(t, "myPlacement", compute.PlacementGroup) + require.Equal(t, "subnet-f578918e", compute.SubnetID) require.Equal(t, true, compute.RootBlockDevice.DeleteOnTermination) require.Len(t, compute.SecurityGroups, 1) require.Contains(t, compute.SecurityGroups, "yorc-securityGroup") diff --git a/prov/terraform/aws/resources.go b/prov/terraform/aws/resources.go index 74cf41585..2ae98f20e 100644 --- a/prov/terraform/aws/resources.go +++ b/prov/terraform/aws/resources.go @@ -21,6 +21,7 @@ type ComputeInstance struct { AvailabilityZone string `json:"availability_zone,omitempty"` PlacementGroup string `json:"placement_group,omitempty"` SecurityGroups []string `json:"security_groups,omitempty"` + SubnetID string `json:"subnet_id,omitempty"` KeyName string `json:"key_name,omitempty"` Tags Tags `json:"tags,omitempty"` ElasticIps []string `json:"-"` diff --git a/prov/terraform/aws/testdata/simpleAWSInstance.yaml b/prov/terraform/aws/testdata/simpleAWSInstance.yaml index 4f15ed5cc..8f34bef81 100644 --- a/prov/terraform/aws/testdata/simpleAWSInstance.yaml +++ b/prov/terraform/aws/testdata/simpleAWSInstance.yaml @@ -18,6 +18,7 @@ topology_template: instance_type: "t2.micro" key_name: "yorc-keypair" security_groups: "yorc-securityGroup" + subnet_id: subnet-f578918e availability_zone: "us-east-2c" placement_group: "myPlacement" capabilities: From f6f018ae304af53908306295c4c348311fc082fe Mon Sep 17 00:00:00 2001 From: hilderic Date: Tue, 14 Apr 2020 11:38:01 +0200 Subject: [PATCH 16/23] Implementing security groups WIP --- data/tosca/yorc-aws-types.yml | 167 ++++++++++++++---- prov/terraform/aws/consul_test.go | 3 + prov/terraform/aws/testdata/simpleSubnet.yaml | 29 --- .../aws/testdata/simpleVPCWithSubnet.yaml | 27 +++ prov/terraform/aws/utils.go | 2 +- prov/terraform/aws/vpc.go | 102 +++++++++-- prov/terraform/aws/vpc_test.go | 60 ++++--- 7 files changed, 277 insertions(+), 113 deletions(-) create mode 100644 prov/terraform/aws/testdata/simpleVPCWithSubnet.yaml diff --git a/data/tosca/yorc-aws-types.yml b/data/tosca/yorc-aws-types.yml index e8061151e..293441581 100644 --- a/data/tosca/yorc-aws-types.yml +++ b/data/tosca/yorc-aws-types.yml @@ -8,6 +8,81 @@ metadata: imports: - yorc: +data_types: + yorc.datatypes.aws.SubnetType: + derived_from: tosca.datatypes.Root + # See : https://www.terraform.io/docs/providers/aws/r/subnet.html + properties: + availability_zone: + type: string + required: false + description: The AZ for the subnet + availability_zone_id: + type: string + required: false + description: The AZ ID for the subnet + cidr_block: + type: string + required: true + ipv6_cidr_block : + type: string + required: false + description: The IPv6 network range for the subnet, in CIDR notation. The subnet size must use a /64 prefix length. + map_public_ip_on_launch: + type: boolean + required: false + description: Specify true to indicate that instances launched into the subnet should be assigned a public IP address. + assign_ipv6_address_on_creation: + type: boolean + required: false + description: > + Specify true to indicate that network interfaces created in the specified subnet should be assigned an IPv6 address + tags: + type: map + description: A mapping of tags to assign to the resource. + required: false + entry_schema: + type: string + + yorc.datatypes.aws.SecurityRuleType: + derived_from: tosca.datatypes.Root + # See : https://www.terraform.io/docs/providers/aws/r/subnet.html + properties: + from_port: + type: string + required: true + to_port: + type: string + required: true + protocol: + type: string + required: true + + yorc.datatypes.aws.SecurityGroupType: + derived_from: tosca.datatypes.Root + # See : https://www.terraform.io/docs/providers/aws/r/subnet.html + properties: + name: + type: string + required: true + ingress: + type: yorc.datatypes.aws.SecurityRuleType + required: true + egress: + type: yorc.datatypes.aws.SecurityRuleType + required: true + +relationship_types: + yorc.relationships.aws.Network: + derived_from: tosca.relationships.Network + properties: + subnet_id: + type: string + required: true + security_group: + type: string + required: true + node_types: yorc.nodes.aws.Compute: derived_from: yorc.nodes.Compute @@ -29,7 +104,7 @@ node_types: type: string description: > Comma-separated list of security groups to add to the Compute - required: true + required: false subnet_id: type: string description: > @@ -106,6 +181,44 @@ node_types: required: false default: false + yorc.nodes.aws.Subnet: + derived_from: tosca.nodes.Network + # See : https://www.terraform.io/docs/providers/aws/r/subnet.html + properties: + availability_zone: + type: string + required: false + description: The AZ for the subnet + availability_zone_id: + type: string + required: false + description: The AZ ID for the subnet + cidr_block: + type: string + required: true + ipv6_cidr_block : + type: string + required: false + description: The IPv6 network range for the subnet, in CIDR notation. The subnet size must use a /64 prefix length. + map_public_ip_on_launch: + type: boolean + required: false + description: Specify true to indicate that instances launched into the subnet should be assigned a public IP address. + assign_ipv6_address_on_creation: + type: boolean + required: false + description: > + Specify true to indicate that network interfaces created in the specified subnet should be assigned an IPv6 address + vpc_id: + type: string + required: true + tags: + type: map + description: A mapping of tags to assign to the resource. + required: false + entry_schema: + type: string + yorc.nodes.aws.VPC: derived_from: tosca.nodes.Network # See https://www.terraform.io/docs/providers/aws/r/vpc.html @@ -151,42 +264,22 @@ node_types: required: false entry_schema: type: string - - yorc.nodes.aws.Subnet: - derived_from: tosca.nodes.Network - # See : https://www.terraform.io/docs/providers/aws/r/subnet.html - properties: - availability_zone: - type: string - required: false - description: The AZ for the subnet - availability_zone_id: - type: string - required: false - description: The AZ ID for the subnet - cidr_block: - type: string + subnets: + type: list + description: Subnetworks in this VPC required: true - ipv6_cidr_block : - type: string - required: false - description: The IPv6 network range for the subnet, in CIDR notation. The subnet size must use a /64 prefix length. - map_public_ip_on_launch: - type: boolean - required: false - description: Specify true to indicate that instances launched into the subnet should be assigned a public IP address. - assign_ipv6_address_on_creation: - type: boolean - required: false - description: > - Specify true to indicate that network interfaces created in the specified subnet should be assigned an IPv6 address - vpc_id: - type: string - required: false - tags: - type: map - description: A mapping of tags to assign to the resource. - required: false entry_schema: - type: string + type: yorc.datatypes.aws.SubnetType + constraints: + - greater_or_equal: 1 + security_groups: + type: list + description: SecurityGroups in this VPC + required: true + entry_schema: + type: yorc.datatypes.aws.SecurityGroupType + constraints: + - greater_or_equal: 1 + + diff --git a/prov/terraform/aws/consul_test.go b/prov/terraform/aws/consul_test.go index 366c616e0..407ac782b 100644 --- a/prov/terraform/aws/consul_test.go +++ b/prov/terraform/aws/consul_test.go @@ -84,5 +84,8 @@ func TestRunConsulAWSPackageTests(t *testing.T) { t.Run("simpleSubnet", func(t *testing.T) { testSimpleSubnet(t, srv, cfg) }) + t.Run("simpleVPCWithSubnet", func(t *testing.T) { + testSimpleVPCWithSubnet(t, srv, cfg) + }) }) } diff --git a/prov/terraform/aws/testdata/simpleSubnet.yaml b/prov/terraform/aws/testdata/simpleSubnet.yaml index 1a33e711e..e69de29bb 100644 --- a/prov/terraform/aws/testdata/simpleSubnet.yaml +++ b/prov/terraform/aws/testdata/simpleSubnet.yaml @@ -1,29 +0,0 @@ -tosca_definitions_version: alien_dsl_2_0_0 - -metadata: - template_name: VPC_creation_test - template_version: 1.0 - template_author: tester - -description: "" - -imports: - - - - - -topology_template: - node_templates: - Network_VPC: - type: yorc.nodes.aws.VPC - properties: - cidr_block: 10.0.0.0/16 - Network_Subnet: - type: yorc.nodes.aws.Subnet - properties: - cidr_block: 10.0.0.0/24 - requirements: - - Network_Subnet_Network_VPC: - type_requirement: dependency - node: Network_VPC - capability: tosca.capabilities.Node - relationship: tosca.relationships.DependsOn diff --git a/prov/terraform/aws/testdata/simpleVPCWithSubnet.yaml b/prov/terraform/aws/testdata/simpleVPCWithSubnet.yaml new file mode 100644 index 000000000..d5f9d4e78 --- /dev/null +++ b/prov/terraform/aws/testdata/simpleVPCWithSubnet.yaml @@ -0,0 +1,27 @@ +tosca_definitions_version: alien_dsl_2_0_0 + +metadata: + template_name: VPC_creation_test + template_version: 1.0 + template_author: tester + +description: "" + +imports: + - + - + +topology_template: + node_templates: + Network_VPC: + type: yorc.nodes.aws.VPC + properties: + cidr_block: 10.0.0.0/16 + Network_Subnet: + type: yorc.nodes.aws.Subnet + properties: + cidr_block: 10.0.0.0/24 + subnets: + - availability_zone: "us-east-2a" + cidr_block: "10.0.0.0/24" + map_public_ip_on_launch: true \ No newline at end of file diff --git a/prov/terraform/aws/utils.go b/prov/terraform/aws/utils.go index 22b63d5ee..cb26184e9 100644 --- a/prov/terraform/aws/utils.go +++ b/prov/terraform/aws/utils.go @@ -32,7 +32,7 @@ func verifyThatNodeIsTypeOf(ctx context.Context, nodeParams nodeParams, nodeType return nil } -func getTagsMap(ctx context.Context, nodeParams nodeParams) (map[string]string, error) { +func getTagsMap(ctx context.Context, nodeParams nodeParams, nestedKeys ...string) (map[string]string, error) { tagsVal, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "tags") if err != nil { return nil, err diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go index 306f9f50f..309489be6 100644 --- a/prov/terraform/aws/vpc.go +++ b/prov/terraform/aws/vpc.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "path" + "strconv" "strings" "github.com/pkg/errors" @@ -56,6 +57,8 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams, i commons.AddResource(nodeParams.infrastructure, "aws_vpc", name, vpc) + g.generatedVPCSubnets(ctx, nodeParams, name) + // Terraform output nodeKey := path.Join(consulutil.DeploymentKVPrefix, nodeParams.deploymentID, "topology", "instances", nodeParams.nodeName, instanceName) @@ -93,22 +96,6 @@ func (g *awsGenerator) generateSubnet(ctx context.Context, nodeParams nodeParams // Name must respect regular expression name = strings.Replace(strings.ToLower(name), "_", "-", -1) - // If not VPCId defined, it should be a dependancy, find it - if subnet.VPCId == "" { - hasDep, vpcNodeName, err := deployments.HasAnyRequirementFromNodeType(ctx, nodeParams.deploymentID, nodeParams.nodeName, "dependency", "yorc.nodes.aws.VPC") - if err != nil { - return err - } - if !hasDep { - return errors.Errorf("failed to retrieve dependency btw any network and the subnet with name:%q", name) - } - - subnet.VPCId, err = deployments.LookupInstanceAttributeValue(ctx, nodeParams.deploymentID, vpcNodeName, instanceName, "vpc_id") - if err != nil { - return err - } - } - commons.AddResource(nodeParams.infrastructure, "aws_subnet", name, subnet) // Terraform output @@ -172,7 +159,7 @@ func (g *awsGenerator) getSubnetProperties(ctx context.Context, nodeParams nodeP {&subnet.AvailabilityZoneID, "availability_zone_id", false}, {&subnet.CidrBlock, "cidr_block", true}, {&subnet.Ipv6CidrBlock, "ipv6_cidr_block", true}, - {&subnet.VPCId, "vpc_id", false}, + {&subnet.VPCId, "vpc_id", true}, } for _, stringParam := range stringParams { @@ -201,3 +188,84 @@ func (g *awsGenerator) getSubnetProperties(ctx context.Context, nodeParams nodeP return nil } + +func (g *awsGenerator) generatedVPCSubnets(ctx context.Context, nodeParams nodeParams, vpcName string) error { + subNetsRaw, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "subnets") + if err != nil { + return err + } + + if subNetsRaw == nil || subNetsRaw.RawString() == "" { + return nil + } + + list, ok := subNetsRaw.Value.([]interface{}) + if !ok { + return errors.New("failed to retrieve yorc.datatypes.aws.SubnetType Tosca Value: not expected type") + } + + for i := range list { + g.generatedVPCSubnet(ctx, nodeParams, vpcName, i) + } + + return nil +} + +func (g *awsGenerator) generatedVPCSubnet(ctx context.Context, nodeParams nodeParams, vpcName string, i int) error { + ind := strconv.Itoa(i) + subnet := &Subnet{} + + params := []struct { + pStringAtt *string + pBoolAtt *bool + propertyName string + mandatory bool + }{ + {&subnet.AvailabilityZone, nil, "availability_zone", false}, + {&subnet.AvailabilityZoneID, nil, "availability_zone_id", false}, + {&subnet.CidrBlock, nil, "cidr_block", true}, + {&subnet.Ipv6CidrBlock, nil, "ipv6_cidr_block", false}, + {nil, &subnet.AssignIpv6AddressOnCreation, "assign_ipv6_address_on_creation", false}, + {nil, &subnet.MapPublicIPOnLaunch, "map_public_ip_on_launch", false}, + } + + for _, params := range params { + val, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "subnets", ind, params.propertyName) + if err != nil { + return err + } + if val == nil && params.mandatory { + return errors.New("Missing mandatory params " + params.propertyName) + } + if params.pStringAtt != nil { + *params.pStringAtt = val.RawString() + } else if params.pBoolAtt != nil { + res, err := strconv.ParseBool(val.RawString()) + if err != nil { + return err + } + *params.pBoolAtt = res + } + + subnet.Tags, err = getTagsMap(ctx, nodeParams, ind, "tags") + if err != nil { + return err + } + + subnet.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", vpcName) + + // Create the name for the resource + var name = "" + if subnet.Tags["Name"] != "" { + name = subnet.Tags["Name"] + } else { + name = strings.ToLower(nodeParams.deploymentID + "-" + nodeParams.nodeName + "-" + vpcName) + } + // Name must respect regular expression + name = strings.Replace(strings.ToLower(name), "_", "-", -1) + + commons.AddResource(nodeParams.infrastructure, "aws_subnet", name, subnet) + } + + return nil +} diff --git a/prov/terraform/aws/vpc_test.go b/prov/terraform/aws/vpc_test.go index 9801882a1..696dacabc 100644 --- a/prov/terraform/aws/vpc_test.go +++ b/prov/terraform/aws/vpc_test.go @@ -16,14 +16,12 @@ package aws import ( "context" - "path" "testing" "github.com/hashicorp/consul/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/ystia/yorc/v4/config" - "github.com/ystia/yorc/v4/helper/consulutil" "github.com/ystia/yorc/v4/prov/terraform/commons" ) @@ -59,32 +57,36 @@ func testSimpleVPC(t *testing.T, cfg config.Configuration) { } func testSimpleSubnet(t *testing.T, srv1 *testutil.TestServer, cfg config.Configuration) { - t.Parallel() - deploymentID := loadTestYaml(t) - ctx := context.Background() - infrastructure := commons.Infrastructure{} - g := awsGenerator{} - subnetName := "simplesubnet-network-subnet" // deploymentID + "-" + nodeName (with _ changed to -) - - nodeParams := nodeParams{ - deploymentID: deploymentID, - nodeName: "Network_Subnet", - infrastructure: &infrastructure, - } - - srv1.PopulateKV(t, map[string][]byte{ - path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/instances/Network_VPC/0/attributes/vpc_id"): []byte("vpc_id"), - }) - - err := g.generateSubnet(ctx, nodeParams, "0", make(map[string]string)) - require.NoError(t, err, "Unexpected error attempting to generate sub-network for %s", deploymentID) - - instancesMap := infrastructure.Resource["aws_subnet"].(map[string]interface{}) - require.Len(t, instancesMap, 1) - require.Contains(t, instancesMap, subnetName) + // t.Parallel() + // deploymentID := loadTestYaml(t) + // ctx := context.Background() + // infrastructure := commons.Infrastructure{} + // g := awsGenerator{} + // subnetName := "simplesubnet-network-subnet" // deploymentID + "-" + nodeName (with _ changed to -) + + // nodeParams := nodeParams{ + // deploymentID: deploymentID, + // nodeName: "Network_Subnet", + // infrastructure: &infrastructure, + // } + + // srv1.PopulateKV(t, map[string][]byte{ + // path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/instances/Network_VPC/0/attributes/vpc_id"): []byte("vpc_id"), + // }) + + // err := g.generateSubnet(ctx, nodeParams, "0", make(map[string]string)) + // require.NoError(t, err, "Unexpected error attempting to generate sub-network for %s", deploymentID) + + // instancesMap := infrastructure.Resource["aws_subnet"].(map[string]interface{}) + // require.Len(t, instancesMap, 1) + // require.Contains(t, instancesMap, subnetName) + + // subnet, ok := instancesMap[subnetName].(*Subnet) + // require.True(t, ok, "%s is not a Subnet", subnetName) + // assert.Equal(t, "10.0.0.0/24", subnet.CidrBlock) + // assert.Equal(t, "vpc_id", subnet.VPCId) +} - subnet, ok := instancesMap[subnetName].(*Subnet) - require.True(t, ok, "%s is not a Subnet", subnetName) - assert.Equal(t, "10.0.0.0/24", subnet.CidrBlock) - assert.Equal(t, "vpc_id", subnet.VPCId) +func testSimpleVPCWithSubnet(t *testing.T, srv1 *testutil.TestServer, cfg config.Configuration) { + // TODO } From f639f6d61a58b488fe2a1e69df2daba9a89de496 Mon Sep 17 00:00:00 2001 From: hilderic Date: Mon, 20 Apr 2020 11:28:53 +0200 Subject: [PATCH 17/23] Default VPC system - NeedCleanup --- data/tosca/yorc-aws-types.yml | 8 +- prov/terraform/aws/aws_instance.go | 34 +++-- prov/terraform/aws/resources.go | 34 +++++ prov/terraform/aws/testdata/simpleVPC.yaml | 14 ++ .../aws/testdata/simpleVPCWithSubnet.yaml | 27 ---- prov/terraform/aws/vpc.go | 138 ++++++++++++++++-- 6 files changed, 201 insertions(+), 54 deletions(-) delete mode 100644 prov/terraform/aws/testdata/simpleVPCWithSubnet.yaml diff --git a/data/tosca/yorc-aws-types.yml b/data/tosca/yorc-aws-types.yml index 293441581..1f1129c39 100644 --- a/data/tosca/yorc-aws-types.yml +++ b/data/tosca/yorc-aws-types.yml @@ -267,19 +267,15 @@ node_types: subnets: type: list description: Subnetworks in this VPC - required: true + required: false entry_schema: type: yorc.datatypes.aws.SubnetType - constraints: - - greater_or_equal: 1 security_groups: type: list description: SecurityGroups in this VPC - required: true + required: false entry_schema: type: yorc.datatypes.aws.SecurityGroupType - constraints: - - greater_or_equal: 1 diff --git a/prov/terraform/aws/aws_instance.go b/prov/terraform/aws/aws_instance.go index ecd1811cc..7e75aa1f9 100644 --- a/prov/terraform/aws/aws_instance.go +++ b/prov/terraform/aws/aws_instance.go @@ -75,18 +75,16 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi } instance.KeyName = keyName.RawString() - // security_groups needs to contain a least one occurrence secGroups, err := deployments.GetNodePropertyValue(ctx, deploymentID, nodeName, "security_groups") if err != nil { return err } - if secGroups == nil || secGroups.RawString() == "" { - return errors.Errorf("Missing mandatory parameter 'security_groups' node type for %s", nodeName) - } - for _, secGroup := range strings.Split(strings.NewReplacer("\"", "", "'", "").Replace(secGroups.RawString()), ",") { - secGroup = strings.TrimSpace(secGroup) - instance.SecurityGroups = append(instance.SecurityGroups, secGroup) + if secGroups.RawString() != "" { + for _, secGroup := range strings.Split(strings.NewReplacer("\"", "", "'", "").Replace(secGroups.RawString()), ",") { + secGroup = strings.TrimSpace(secGroup) + instance.SecurityGroups = append(instance.SecurityGroups, secGroup) + } } // Get connection info (user, private key) @@ -150,7 +148,7 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi if subnetID != nil { instance.SubnetID = subnetID.RawString() } - hasNetwork, _, err := deployments.HasAnyRequirementFromNodeType(ctx, deploymentID, nodeName, "network", "yorc.nodes.aws.Subnet") + hasNetwork, _, err := deployments.HasAnyRequirementFromNodeType(ctx, deploymentID, nodeName, "network", "yorc.nodes.aws.VPC") if err != nil { return err } @@ -163,11 +161,27 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi i := 0 for _, networkReq := range networkRequirements { networkInterface := &NetworkInterface{} - networkInterface.SubnetID, err = deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "subnet_id") + // TODO : Check if subnet and security group are defined in relationship + // networkInterface.SubnetID, err = deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "subnet_id") + // if err != nil { + // return err + // } + + // If not, use default one + defaultSubnetID, err := deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "default_subnet_id") if err != nil { return err } - networkInterface.SecurityGroups = instance.SecurityGroups + networkInterface.SubnetID = defaultSubnetID + + // With default Security Group + defaultSecurityGroup, err := deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "default_security_group") + if err != nil { + return err + } + networkInterface.SecurityGroups = make([]string, 1) + networkInterface.SecurityGroups = append(networkInterface.SecurityGroups, defaultSecurityGroup) + name := strings.ToLower("network-inteface-" + strconv.Itoa(i)) name = strings.Replace(strings.ToLower(name), "_", "-", -1) diff --git a/prov/terraform/aws/resources.go b/prov/terraform/aws/resources.go index 2ae98f20e..2bf6abce1 100644 --- a/prov/terraform/aws/resources.go +++ b/prov/terraform/aws/resources.go @@ -109,3 +109,37 @@ type NetworkInterface struct { SecurityGroups []string `json:"security_groups,omitempty"` Attachment map[string]string `json:"attachment,omitempty"` } + +// todo +type SecurityRule struct { + FromPort string `json:"from_port,omitempty"` + ToPort string `json:"to_port,omitempty"` + Protocol string `json:"protocol,omitempty"` + CidrBlock []string `json:"cidr_blocks,omitempty"` +} + +// todo +type SecurityGroups struct { + VPCId string `json:"vpc_id,omitempty"` + Egress SecurityRule `json:"egress,omitempty"` + Ingress SecurityRule `json:"ingress,omitempty"` + Name string `json:"name,omitempty"` +} + +// todo +type RouteTable struct { + VPCId string `json:"vpc_id,omitempty"` + Route map[string]string `json:"route,omitempty"` + DependsOn []string `json:"depends_on,omitempty"` +} + +type DefaultRouteTable struct { + DefaultRouteTableID string `json:"default_route_table_id,omitempty"` + Route map[string]string `json:"route,omitempty"` + DependsOn []string `json:"depends_on,omitempty"` +} + +// todo +type InternetGateway struct { + VPCId string `json:"vpc_id,omitempty"` +} diff --git a/prov/terraform/aws/testdata/simpleVPC.yaml b/prov/terraform/aws/testdata/simpleVPC.yaml index 72ba78430..64a963b38 100644 --- a/prov/terraform/aws/testdata/simpleVPC.yaml +++ b/prov/terraform/aws/testdata/simpleVPC.yaml @@ -18,4 +18,18 @@ topology_template: properties: cidr_block: 10.0.0.0/16 assign_generated_ipv6_cidr_block: true + subnets: + - availability_zone: "us-east-2a" + cidr_block: "10.0.0.0/24" + map_public_ip_on_launch: true + security_groups: + - name: groupOpen + ingress: + from_port: 0 + to_port: 0 + protocol: "-1" + egress: + from_port: 0 + protocol: 1 + to_port: 0 tags: {"tag1": "foo"} \ No newline at end of file diff --git a/prov/terraform/aws/testdata/simpleVPCWithSubnet.yaml b/prov/terraform/aws/testdata/simpleVPCWithSubnet.yaml deleted file mode 100644 index d5f9d4e78..000000000 --- a/prov/terraform/aws/testdata/simpleVPCWithSubnet.yaml +++ /dev/null @@ -1,27 +0,0 @@ -tosca_definitions_version: alien_dsl_2_0_0 - -metadata: - template_name: VPC_creation_test - template_version: 1.0 - template_author: tester - -description: "" - -imports: - - - - - -topology_template: - node_templates: - Network_VPC: - type: yorc.nodes.aws.VPC - properties: - cidr_block: 10.0.0.0/16 - Network_Subnet: - type: yorc.nodes.aws.Subnet - properties: - cidr_block: 10.0.0.0/24 - subnets: - - availability_zone: "us-east-2a" - cidr_block: "10.0.0.0/24" - map_public_ip_on_launch: true \ No newline at end of file diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go index 309489be6..c6eb39e24 100644 --- a/prov/terraform/aws/vpc.go +++ b/prov/terraform/aws/vpc.go @@ -17,6 +17,8 @@ package aws import ( "context" "fmt" + "io/ioutil" + "net/http" "path" "strconv" "strings" @@ -57,11 +59,29 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams, i commons.AddResource(nodeParams.infrastructure, "aws_vpc", name, vpc) - g.generatedVPCSubnets(ctx, nodeParams, name) - // Terraform output nodeKey := path.Join(consulutil.DeploymentKVPrefix, nodeParams.deploymentID, "topology", "instances", nodeParams.nodeName, instanceName) + g.generateVPCSubnets(ctx, nodeParams, name, vpc, nodeKey, outputs) + g.generateVPCSecurityGroups(ctx, nodeParams, name, nodeKey, outputs) + + internetGateway := &InternetGateway{} + internetGateway.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", name) + internetGatewayName := nodeParams.nodeName + "_defaultInternetGateway" + commons.AddResource(nodeParams.infrastructure, "aws_internet_gateway", internetGatewayName, internetGateway) + + routeTable := DefaultRouteTable{} + // routeTable.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", name) + routeTable.DefaultRouteTableID = fmt.Sprintf("${aws_vpc.%s.default_route_table_id}", name) + routeTable.Route = map[string]string{ + "cidr_block": "0.0.0.0/0", + "gateway_id": fmt.Sprintf("${aws_internet_gateway.%s.id}", internetGatewayName), + } + routeTable.DependsOn = []string{ + fmt.Sprintf("aws_internet_gateway.%s", internetGatewayName), + } + commons.AddResource(nodeParams.infrastructure, "aws_default_route_table", nodeParams.nodeName+"_defaultRouteTable", routeTable) + idKey := nodeParams.nodeName + "-" + instanceName + "-id" idValue := fmt.Sprintf("${aws_vpc.%s.id}", name) commons.AddOutput(nodeParams.infrastructure, idKey, &commons.Output{Value: idValue}) @@ -189,26 +209,47 @@ func (g *awsGenerator) getSubnetProperties(ctx context.Context, nodeParams nodeP return nil } -func (g *awsGenerator) generatedVPCSubnets(ctx context.Context, nodeParams nodeParams, vpcName string) error { +func (g *awsGenerator) generateVPCSubnets(ctx context.Context, nodeParams nodeParams, vpcName string, vpc *VPC, nodeKey string, outputs map[string]string) error { subNetsRaw, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "subnets") if err != nil { return err } if subNetsRaw == nil || subNetsRaw.RawString() == "" { - return nil - } + // Generate default subnet + g.generateDefaultSubnet(ctx, nodeParams, vpcName, vpc, nodeKey, outputs) + } else { + list, ok := subNetsRaw.Value.([]interface{}) + if !ok { + return errors.New("failed to retrieve yorc.datatypes.aws.SubnetType Tosca Value: not expected type") + } - list, ok := subNetsRaw.Value.([]interface{}) - if !ok { - return errors.New("failed to retrieve yorc.datatypes.aws.SubnetType Tosca Value: not expected type") + for i := range list { + g.generatedVPCSubnet(ctx, nodeParams, vpcName, i) + } } - for i := range list { - g.generatedVPCSubnet(ctx, nodeParams, vpcName, i) + return nil +} + +func (g *awsGenerator) generateDefaultSubnet(ctx context.Context, nodeParams nodeParams, vpcName string, vpc *VPC, nodeKey string, outputs map[string]string) { + subnet := &Subnet{} + subnet.MapPublicIPOnLaunch = true + subnet.CidrBlock = vpc.CidrBlock + subnet.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", vpcName) + subnet.Tags = map[string]string{ + "name": "DefaultSubnet", } - return nil + name := strings.ToLower(nodeParams.deploymentID + "-" + nodeParams.nodeName + "-" + "defaultsubnet") + name = strings.Replace(strings.ToLower(name), "_", "-", -1) + + commons.AddResource(nodeParams.infrastructure, "aws_subnet", name, subnet) + + idKey := nodeParams.nodeName + "-defaultSubnet" + idValue := fmt.Sprintf("${aws_subnet.%s.id}", name) + commons.AddOutput(nodeParams.infrastructure, idKey, &commons.Output{Value: idValue}) + outputs[path.Join(nodeKey, "/attributes/default_subnet_id")] = idKey } func (g *awsGenerator) generatedVPCSubnet(ctx context.Context, nodeParams nodeParams, vpcName string, i int) error { @@ -269,3 +310,78 @@ func (g *awsGenerator) generatedVPCSubnet(ctx context.Context, nodeParams nodePa return nil } + +func (g *awsGenerator) generateVPCSecurityGroups(ctx context.Context, nodeParams nodeParams, vpcName string, nodeKey string, outputs map[string]string) error { + securityGroupsRaw, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "security_groups") + if err != nil { + return err + } + + if securityGroupsRaw == nil || securityGroupsRaw.RawString() == "" { + err := g.generateDefaultSecurityGroup(ctx, nodeParams, vpcName, nodeKey, outputs) + if err != nil { + return err + } + } else { + list, ok := securityGroupsRaw.Value.([]interface{}) + if !ok { + return errors.New("failed to retrieve yorc.datatypes.aws.SecurityGroupType Tosca Value: not expected type") + } + + for i := range list { + g.generateVPCSecurityGroup(ctx, nodeParams, vpcName, i) + } + } + + return nil +} + +func (g *awsGenerator) generateDefaultSecurityGroup(ctx context.Context, nodeParams nodeParams, vpcName string, nodeKey string, outputs map[string]string) error { + url := "https://api.ipify.org" + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + ip, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + securityGroup := &SecurityGroups{} + securityGroup.Egress = SecurityRule{"0", "0", "-1", []string{string(ip) + "/32"}} + securityGroup.Ingress = SecurityRule{"0", "0", "-1", []string{string(ip) + "/32"}} + securityGroup.Name = "Default of " + vpcName + securityGroup.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", vpcName) + + name := strings.ToLower(nodeParams.deploymentID + "-" + nodeParams.nodeName + "-" + "defaultSecurityGroup") + name = strings.Replace(strings.ToLower(name), "_", "-", -1) + + commons.AddResource(nodeParams.infrastructure, "aws_security_group", name, securityGroup) + + idKey := nodeParams.nodeName + "-defaultSecurityGroup" + idValue := fmt.Sprintf("${aws_security_group.%s.id}", name) + commons.AddOutput(nodeParams.infrastructure, idKey, &commons.Output{Value: idValue}) + outputs[path.Join(nodeKey, "/attributes/default_security_group")] = idKey + + return nil +} + +func (g *awsGenerator) generateVPCSecurityGroup(ctx context.Context, nodeParams nodeParams, vpcName string, i int) error { + ind := strconv.Itoa(i) + securityGroup := &SecurityGroups{} + + val, err := deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "security_groups", ind, "name") + if err != nil { + return err + } + securityGroup.Name = val.RawString() + + val, err = deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "security_groups", ind, "ingress", "protocol") + if err != nil { + return err + } + fmt.Println(val) + + return nil +} From 6d2699f5d67e10198a7458e64ae2b68fee3e825f Mon Sep 17 00:00:00 2001 From: hilderic Date: Mon, 20 Apr 2020 16:00:04 +0200 Subject: [PATCH 18/23] new test + vpc nested sg fix --- deployments/nodes.go | 8 +- prov/terraform/aws/aws_instance_test.go | 41 +++---- prov/terraform/aws/consul_test.go | 11 +- prov/terraform/aws/testdata/VPC.yaml | 21 ++++ ...VPC.yaml => VPCWithNestedSubnetAndSG.yaml} | 8 +- prov/terraform/aws/testdata/simpleSubnet.yaml | 0 prov/terraform/aws/vpc.go | 88 +++++++++++---- prov/terraform/aws/vpc_test.go | 103 ++++++++++++++++-- 8 files changed, 214 insertions(+), 66 deletions(-) create mode 100644 prov/terraform/aws/testdata/VPC.yaml rename prov/terraform/aws/testdata/{simpleVPC.yaml => VPCWithNestedSubnetAndSG.yaml} (89%) delete mode 100644 prov/terraform/aws/testdata/simpleSubnet.yaml diff --git a/deployments/nodes.go b/deployments/nodes.go index e2cfee53e..8c72bad62 100644 --- a/deployments/nodes.go +++ b/deployments/nodes.go @@ -431,9 +431,9 @@ func GetNodePropertyValue(ctx context.Context, deploymentID, nodeName, propertyN // GetStringNodeProperty returns the string value of a property. // If this value is empty and the argument mandatory is true, an error is returned -func GetStringNodeProperty(ctx context.Context, deploymentID, nodeName, propertyName string, mandatory bool) (string, error) { +func GetStringNodeProperty(ctx context.Context, deploymentID, nodeName, propertyName string, mandatory bool, nestedKeys ...string) (string, error) { - result, err := GetNodePropertyValue(ctx, deploymentID, nodeName, propertyName) + result, err := GetNodePropertyValue(ctx, deploymentID, nodeName, propertyName, nestedKeys...) if err != nil { return "", err } @@ -449,9 +449,9 @@ func GetStringNodeProperty(ctx context.Context, deploymentID, nodeName, property } // GetBooleanNodeProperty returns the boolean value of a property (default: false) -func GetBooleanNodeProperty(ctx context.Context, deploymentID, nodeName, propertyName string) (bool, error) { +func GetBooleanNodeProperty(ctx context.Context, deploymentID, nodeName, propertyName string, nestedKeys ...string) (bool, error) { var result bool - va, err := GetNodePropertyValue(ctx, deploymentID, nodeName, propertyName) + va, err := GetNodePropertyValue(ctx, deploymentID, nodeName, propertyName, nestedKeys...) if err != nil { return result, err } diff --git a/prov/terraform/aws/aws_instance_test.go b/prov/terraform/aws/aws_instance_test.go index ee3b3c087..8b0b1978e 100644 --- a/prov/terraform/aws/aws_instance_test.go +++ b/prov/terraform/aws/aws_instance_test.go @@ -376,31 +376,32 @@ func testSimpleAWSInstanceWithPersistentDisk(t *testing.T, cfg config.Configurat } func simpleAWSInstanceWithVPCandSubnet(t *testing.T, cfg config.Configuration, srv *testutil.TestServer) { - t.Parallel() + // TODO + // t.Parallel() - deploymentID := loadTestYaml(t) - g := awsGenerator{} - infrastructure := commons.Infrastructure{} - ctx := context.Background() - env := make([]string, 0) - outputs := make(map[string]string) + // deploymentID := loadTestYaml(t) + // g := awsGenerator{} + // infrastructure := commons.Infrastructure{} + // ctx := context.Background() + // env := make([]string, 0) + // outputs := make(map[string]string) - srv.PopulateKV(t, map[string][]byte{ - path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/nodes/Network_Subnet/type"): []byte("yorc.nodes.aws.Subnet"), - path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/instances/Network_Subnet/0/attributes/subnet_id"): []byte("subnet_id"), - }) + // srv.PopulateKV(t, map[string][]byte{ + // path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/nodes/Network_Subnet/type"): []byte("yorc.nodes.aws.Subnet"), + // path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/instances/Network_Subnet/0/attributes/subnet_id"): []byte("subnet_id"), + // }) - err := g.generateAWSInstance(ctx, cfg, deploymentID, "AWS_Compute", "0", &infrastructure, outputs, &env) - require.Nil(t, err) - require.Len(t, infrastructure.Resource["aws_instance"], 1) + // err := g.generateAWSInstance(ctx, cfg, deploymentID, "AWS_Compute", "0", &infrastructure, outputs, &env) + // require.Nil(t, err) + // require.Len(t, infrastructure.Resource["aws_instance"], 1) - instancesMap := infrastructure.Resource["aws_instance"].(map[string]interface{}) - require.Contains(t, instancesMap, "AWS_Compute-0") + // instancesMap := infrastructure.Resource["aws_instance"].(map[string]interface{}) + // require.Contains(t, instancesMap, "AWS_Compute-0") - compute, ok := instancesMap["AWS_Compute-0"].(*ComputeInstance) - require.True(t, ok, "%s is not a ComputeInstance", "AWS_Compute-0") + // compute, ok := instancesMap["AWS_Compute-0"].(*ComputeInstance) + // require.True(t, ok, "%s is not a ComputeInstance", "AWS_Compute-0") - assert.Equal(t, "${aws_network_interface.network-inteface-0.id}", compute.NetworkInterface["network_interface_id"], "Subnetwork is not retrieved") - assert.Equal(t, "0", compute.NetworkInterface["device_index"], "Subnetwork is not retrieved") + // assert.Equal(t, "${aws_network_interface.network-inteface-0.id}", compute.NetworkInterface["network_interface_id"], "Subnetwork is not retrieved") + // assert.Equal(t, "0", compute.NetworkInterface["device_index"], "Subnetwork is not retrieved") } diff --git a/prov/terraform/aws/consul_test.go b/prov/terraform/aws/consul_test.go index 407ac782b..8af2cdcb4 100644 --- a/prov/terraform/aws/consul_test.go +++ b/prov/terraform/aws/consul_test.go @@ -78,14 +78,11 @@ func TestRunConsulAWSPackageTests(t *testing.T) { t.Run("simpleAWSInstanceWithPersistentDisk", func(t *testing.T) { testSimpleAWSInstanceWithPersistentDisk(t, cfg, srv) }) - t.Run("simpleVPC", func(t *testing.T) { - testSimpleVPC(t, cfg) + t.Run("VPC", func(t *testing.T) { + testVPC(t, cfg) }) - t.Run("simpleSubnet", func(t *testing.T) { - testSimpleSubnet(t, srv, cfg) - }) - t.Run("simpleVPCWithSubnet", func(t *testing.T) { - testSimpleVPCWithSubnet(t, srv, cfg) + t.Run("VPCWithNestedSubnetAndSG", func(t *testing.T) { + testVPCWithNestedSubnetAndSG(t, srv, cfg) }) }) } diff --git a/prov/terraform/aws/testdata/VPC.yaml b/prov/terraform/aws/testdata/VPC.yaml new file mode 100644 index 000000000..72ba78430 --- /dev/null +++ b/prov/terraform/aws/testdata/VPC.yaml @@ -0,0 +1,21 @@ +tosca_definitions_version: alien_dsl_2_0_0 + +metadata: + template_name: VPC_creation_test + template_version: 1.0 + template_author: tester + +description: "" + +imports: + - + - + +topology_template: + node_templates: + Network: + type: yorc.nodes.aws.VPC + properties: + cidr_block: 10.0.0.0/16 + assign_generated_ipv6_cidr_block: true + tags: {"tag1": "foo"} \ No newline at end of file diff --git a/prov/terraform/aws/testdata/simpleVPC.yaml b/prov/terraform/aws/testdata/VPCWithNestedSubnetAndSG.yaml similarity index 89% rename from prov/terraform/aws/testdata/simpleVPC.yaml rename to prov/terraform/aws/testdata/VPCWithNestedSubnetAndSG.yaml index 64a963b38..2d890a4f9 100644 --- a/prov/terraform/aws/testdata/simpleVPC.yaml +++ b/prov/terraform/aws/testdata/VPCWithNestedSubnetAndSG.yaml @@ -25,11 +25,11 @@ topology_template: security_groups: - name: groupOpen ingress: - from_port: 0 - to_port: 0 - protocol: "-1" + from_port: 1 + to_port: 15 + protocol: "TCP" egress: from_port: 0 - protocol: 1 + protocol: "-1" to_port: 0 tags: {"tag1": "foo"} \ No newline at end of file diff --git a/prov/terraform/aws/testdata/simpleSubnet.yaml b/prov/terraform/aws/testdata/simpleSubnet.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go index c6eb39e24..ed04fa265 100644 --- a/prov/terraform/aws/vpc.go +++ b/prov/terraform/aws/vpc.go @@ -44,7 +44,7 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams, i // Get tags map vpc.Tags, err = getTagsMap(ctx, nodeParams) if err != nil { - return nil + return err } // Create the name for the resource @@ -59,29 +59,19 @@ func (g *awsGenerator) generateVPC(ctx context.Context, nodeParams nodeParams, i commons.AddResource(nodeParams.infrastructure, "aws_vpc", name, vpc) - // Terraform output nodeKey := path.Join(consulutil.DeploymentKVPrefix, nodeParams.deploymentID, "topology", "instances", nodeParams.nodeName, instanceName) - g.generateVPCSubnets(ctx, nodeParams, name, vpc, nodeKey, outputs) - g.generateVPCSecurityGroups(ctx, nodeParams, name, nodeKey, outputs) - - internetGateway := &InternetGateway{} - internetGateway.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", name) - internetGatewayName := nodeParams.nodeName + "_defaultInternetGateway" - commons.AddResource(nodeParams.infrastructure, "aws_internet_gateway", internetGatewayName, internetGateway) - - routeTable := DefaultRouteTable{} - // routeTable.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", name) - routeTable.DefaultRouteTableID = fmt.Sprintf("${aws_vpc.%s.default_route_table_id}", name) - routeTable.Route = map[string]string{ - "cidr_block": "0.0.0.0/0", - "gateway_id": fmt.Sprintf("${aws_internet_gateway.%s.id}", internetGatewayName), + err = g.generateVPCSubnets(ctx, nodeParams, name, vpc, nodeKey, outputs) + if err != nil { + return err } - routeTable.DependsOn = []string{ - fmt.Sprintf("aws_internet_gateway.%s", internetGatewayName), + err = g.generateVPCSecurityGroups(ctx, nodeParams, name, nodeKey, outputs) + if err != nil { + return err } - commons.AddResource(nodeParams.infrastructure, "aws_default_route_table", nodeParams.nodeName+"_defaultRouteTable", routeTable) + g.generateIGandRT(ctx, nodeParams, name) + // Terraform output idKey := nodeParams.nodeName + "-" + instanceName + "-id" idValue := fmt.Sprintf("${aws_vpc.%s.id}", name) commons.AddOutput(nodeParams.infrastructure, idKey, &commons.Output{Value: idValue}) @@ -281,11 +271,11 @@ func (g *awsGenerator) generatedVPCSubnet(ctx context.Context, nodeParams nodePa if params.pStringAtt != nil { *params.pStringAtt = val.RawString() } else if params.pBoolAtt != nil { - res, err := strconv.ParseBool(val.RawString()) + val, err := deployments.GetBooleanNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, "subnets", ind, params.propertyName) if err != nil { return err } - *params.pBoolAtt = res + *params.pBoolAtt = val } subnet.Tags, err = getTagsMap(ctx, nodeParams, ind, "tags") @@ -351,7 +341,7 @@ func (g *awsGenerator) generateDefaultSecurityGroup(ctx context.Context, nodePar securityGroup := &SecurityGroups{} securityGroup.Egress = SecurityRule{"0", "0", "-1", []string{string(ip) + "/32"}} securityGroup.Ingress = SecurityRule{"0", "0", "-1", []string{string(ip) + "/32"}} - securityGroup.Name = "Default of " + vpcName + securityGroup.Name = "Default " + vpcName securityGroup.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", vpcName) name := strings.ToLower(nodeParams.deploymentID + "-" + nodeParams.nodeName + "-" + "defaultSecurityGroup") @@ -377,11 +367,61 @@ func (g *awsGenerator) generateVPCSecurityGroup(ctx context.Context, nodeParams } securityGroup.Name = val.RawString() - val, err = deployments.GetNodePropertyValue(ctx, nodeParams.deploymentID, nodeParams.nodeName, "security_groups", ind, "ingress", "protocol") + protocol, err := deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, "security_groups", true, ind, "ingress", "protocol") + if err != nil { + return err + } + + fromPort, err := deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, "security_groups", true, ind, "ingress", "from_port") + if err != nil { + return err + } + + toPort, err := deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, "security_groups", true, ind, "ingress", "to_port") + if err != nil { + return err + } + + securityGroup.Ingress = SecurityRule{FromPort: fromPort, ToPort: toPort, Protocol: protocol} + + protocol, err = deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, "security_groups", true, ind, "egress", "protocol") + if err != nil { + return err + } + + fromPort, err = deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, "security_groups", true, ind, "egress", "from_port") if err != nil { return err } - fmt.Println(val) + + toPort, err = deployments.GetStringNodeProperty(ctx, nodeParams.deploymentID, nodeParams.nodeName, "security_groups", true, ind, "egress", "to_port") + if err != nil { + return err + } + + securityGroup.Egress = SecurityRule{FromPort: fromPort, ToPort: toPort, Protocol: protocol} + + commons.AddResource(nodeParams.infrastructure, "aws_security_group", securityGroup.Name, securityGroup) return nil } + +// Generate an InternetGateway and Routable so the VPC is accesible through internet +func (g *awsGenerator) generateIGandRT(ctx context.Context, nodeParams nodeParams, vpcName string) { + internetGateway := &InternetGateway{} + internetGateway.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", vpcName) + internetGatewayName := nodeParams.nodeName + "_defaultInternetGateway" + commons.AddResource(nodeParams.infrastructure, "aws_internet_gateway", internetGatewayName, internetGateway) + + routeTable := DefaultRouteTable{} + // routeTable.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", name) + routeTable.DefaultRouteTableID = fmt.Sprintf("${aws_vpc.%s.default_route_table_id}", vpcName) + routeTable.Route = map[string]string{ + "cidr_block": "0.0.0.0/0", + "gateway_id": fmt.Sprintf("${aws_internet_gateway.%s.id}", internetGatewayName), + } + routeTable.DependsOn = []string{ + fmt.Sprintf("aws_internet_gateway.%s", internetGatewayName), + } + commons.AddResource(nodeParams.infrastructure, "aws_default_route_table", nodeParams.nodeName+"_defaultRouteTable", routeTable) +} diff --git a/prov/terraform/aws/vpc_test.go b/prov/terraform/aws/vpc_test.go index 696dacabc..b6128eb3d 100644 --- a/prov/terraform/aws/vpc_test.go +++ b/prov/terraform/aws/vpc_test.go @@ -16,6 +16,10 @@ package aws import ( "context" + "fmt" + "io/ioutil" + "net/http" + "strings" "testing" "github.com/hashicorp/consul/testutil" @@ -25,13 +29,13 @@ import ( "github.com/ystia/yorc/v4/prov/terraform/commons" ) -func testSimpleVPC(t *testing.T, cfg config.Configuration) { +func testVPC(t *testing.T, cfg config.Configuration) { t.Parallel() deploymentID := loadTestYaml(t) ctx := context.Background() infrastructure := commons.Infrastructure{} g := awsGenerator{} - networkName := "simplevpc-network" + networkName := "vpc-network" nodeParams := nodeParams{ deploymentID: deploymentID, @@ -44,6 +48,7 @@ func testSimpleVPC(t *testing.T, cfg config.Configuration) { require.NoError(t, err, "Unexpected error attempting to generate vpc for %s", deploymentID) require.Len(t, infrastructure.Resource["aws_vpc"], 1, "Expected one vpc") + // VPC resource test instancesMap := infrastructure.Resource["aws_vpc"].(map[string]interface{}) require.Len(t, instancesMap, 1) require.Contains(t, instancesMap, networkName) @@ -54,9 +59,97 @@ func testSimpleVPC(t *testing.T, cfg config.Configuration) { assert.Equal(t, true, vpc.AssignGeneratedIpv6CidrBlock) assert.Equal(t, "foo", vpc.Tags["tag1"]) + // Default subnet test + defaultSubnetName := strings.ToLower(nodeParams.deploymentID + "-" + nodeParams.nodeName + "-" + "defaultsubnet") + instancesMap = infrastructure.Resource["aws_subnet"].(map[string]interface{}) + require.Len(t, instancesMap, 1) + require.Contains(t, instancesMap, defaultSubnetName) + + subnet, ok := instancesMap[defaultSubnetName].(*Subnet) + require.True(t, ok, "%s is not a Subnet", defaultSubnetName) + assert.Equal(t, "10.0.0.0/16", subnet.CidrBlock) + assert.Equal(t, true, subnet.MapPublicIPOnLaunch) + + defaultSubnetKey := nodeParams.nodeName + "-defaultSubnet" + require.NotNil(t, infrastructure.Output[defaultSubnetKey], "Expected related output to default subnet ID") + output := infrastructure.Output[defaultSubnetKey] + defaultSubnetName = strings.Replace(strings.ToLower(defaultSubnetName), "_", "-", -1) + require.Equal(t, output.Value, fmt.Sprintf("${aws_subnet.%s.id}", defaultSubnetName)) + + // Default Security Group test + defaultSecurityGroupName := strings.ToLower(nodeParams.deploymentID + "-" + nodeParams.nodeName + "-" + "defaultSecurityGroup") + instancesMap = infrastructure.Resource["aws_security_group"].(map[string]interface{}) + require.Len(t, instancesMap, 1) + require.Contains(t, instancesMap, defaultSecurityGroupName) + + securityGroup, ok := instancesMap[defaultSecurityGroupName].(*SecurityGroups) + require.True(t, ok, "%s is not a Subnet", defaultSubnetName) + url := "https://api.ipify.org" + resp, err := http.Get(url) + require.NoError(t, err, "Error GET with api.ipify", deploymentID) + defer resp.Body.Close() + ip, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err, "Can't read IP on ipify response", deploymentID) + assert.Equal(t, SecurityRule{"0", "0", "-1", []string{string(ip) + "/32"}}, securityGroup.Egress) } -func testSimpleSubnet(t *testing.T, srv1 *testutil.TestServer, cfg config.Configuration) { +func testVPCWithNestedSubnetAndSG(t *testing.T, srv1 *testutil.TestServer, cfg config.Configuration) { + t.Parallel() + deploymentID := loadTestYaml(t) + ctx := context.Background() + infrastructure := commons.Infrastructure{} + g := awsGenerator{} + networkName := "vpcwithnestedsubnetandsg-network" + + nodeParams := nodeParams{ + deploymentID: deploymentID, + nodeName: "Network", + infrastructure: &infrastructure, + } + + err := g.generateVPC(ctx, nodeParams, "instance0", make(map[string]string)) + + require.NoError(t, err, "Unexpected error attempting to generate vpc for %s", deploymentID) + require.Len(t, infrastructure.Resource["aws_vpc"], 1, "Expected one vpc") + + // VPC resource test + instancesMap := infrastructure.Resource["aws_vpc"].(map[string]interface{}) + require.Len(t, instancesMap, 1) + require.Contains(t, instancesMap, networkName) + + vpc, ok := instancesMap[networkName].(*VPC) + require.True(t, ok, "%s is not a VPC", networkName) + assert.Equal(t, "10.0.0.0/16", vpc.CidrBlock) + assert.Equal(t, true, vpc.AssignGeneratedIpv6CidrBlock) + assert.Equal(t, "foo", vpc.Tags["tag1"]) + + // Generated SubnetTest + subnetName := strings.ToLower(nodeParams.deploymentID + "-" + nodeParams.nodeName + "-" + networkName) + subnetName = strings.Replace(strings.ToLower(subnetName), "_", "-", -1) + instancesMap = infrastructure.Resource["aws_subnet"].(map[string]interface{}) + require.Len(t, instancesMap, 1) + require.Contains(t, instancesMap, subnetName) + + subnet, ok := instancesMap[subnetName].(*Subnet) + require.True(t, ok, "%s is not a subnet", subnet) + assert.Equal(t, "10.0.0.0/24", subnet.CidrBlock) + assert.Equal(t, "us-east-2a", subnet.AvailabilityZone) + assert.Equal(t, true, subnet.MapPublicIPOnLaunch) + + // Generated SecurityGroups + securityGroupName := "groupOpen" + instancesMap = infrastructure.Resource["aws_security_group"].(map[string]interface{}) + require.Len(t, instancesMap, 1) + require.Contains(t, instancesMap, securityGroupName) + + securityGroup, ok := instancesMap[securityGroupName].(*SecurityGroups) + require.True(t, ok, "%s is not a securityGroup", securityGroup) + assert.Equal(t, SecurityRule{FromPort: "0", ToPort: "0", Protocol: "-1"}, securityGroup.Egress) + assert.Equal(t, SecurityRule{FromPort: "1", ToPort: "15", Protocol: "TCP"}, securityGroup.Ingress) + +} + +func testSubnet(t *testing.T, srv1 *testutil.TestServer, cfg config.Configuration) { // t.Parallel() // deploymentID := loadTestYaml(t) // ctx := context.Background() @@ -86,7 +179,3 @@ func testSimpleSubnet(t *testing.T, srv1 *testutil.TestServer, cfg config.Config // assert.Equal(t, "10.0.0.0/24", subnet.CidrBlock) // assert.Equal(t, "vpc_id", subnet.VPCId) } - -func testSimpleVPCWithSubnet(t *testing.T, srv1 *testutil.TestServer, cfg config.Configuration) { - // TODO -} From 33de3d6a11f8e59ee52a08551b6432cf57623941 Mon Sep 17 00:00:00 2001 From: hilderic Date: Tue, 21 Apr 2020 11:57:41 +0200 Subject: [PATCH 19/23] Subnet test + small fix --- prov/terraform/aws/consul_test.go | 3 ++ prov/terraform/aws/testdata/Subnet.yaml | 22 +++++++++++ prov/terraform/aws/vpc_test.go | 52 ++++++++++++------------- 3 files changed, 49 insertions(+), 28 deletions(-) create mode 100644 prov/terraform/aws/testdata/Subnet.yaml diff --git a/prov/terraform/aws/consul_test.go b/prov/terraform/aws/consul_test.go index 8af2cdcb4..ea1093882 100644 --- a/prov/terraform/aws/consul_test.go +++ b/prov/terraform/aws/consul_test.go @@ -81,6 +81,9 @@ func TestRunConsulAWSPackageTests(t *testing.T) { t.Run("VPC", func(t *testing.T) { testVPC(t, cfg) }) + t.Run("Subnet", func(t *testing.T) { + testSubnet(t, srv, cfg) + }) t.Run("VPCWithNestedSubnetAndSG", func(t *testing.T) { testVPCWithNestedSubnetAndSG(t, srv, cfg) }) diff --git a/prov/terraform/aws/testdata/Subnet.yaml b/prov/terraform/aws/testdata/Subnet.yaml new file mode 100644 index 000000000..d57e68b5d --- /dev/null +++ b/prov/terraform/aws/testdata/Subnet.yaml @@ -0,0 +1,22 @@ +tosca_definitions_version: alien_dsl_2_0_0 + +metadata: + template_name: Subnets_creation_test + template_version: 1.0 + template_author: tester + +description: "" + +imports: + - + - + +topology_template: + node_templates: + Network_Subnet: + type: yorc.nodes.aws.Subnet + properties: + vpc_id: "vpc_id" + cidr_block: 10.0.0.0/24 + assign_generated_ipv6_cidr_block: true + tags: {"tag1": "foo"} \ No newline at end of file diff --git a/prov/terraform/aws/vpc_test.go b/prov/terraform/aws/vpc_test.go index b6128eb3d..6256aa2a0 100644 --- a/prov/terraform/aws/vpc_test.go +++ b/prov/terraform/aws/vpc_test.go @@ -150,32 +150,28 @@ func testVPCWithNestedSubnetAndSG(t *testing.T, srv1 *testutil.TestServer, cfg c } func testSubnet(t *testing.T, srv1 *testutil.TestServer, cfg config.Configuration) { - // t.Parallel() - // deploymentID := loadTestYaml(t) - // ctx := context.Background() - // infrastructure := commons.Infrastructure{} - // g := awsGenerator{} - // subnetName := "simplesubnet-network-subnet" // deploymentID + "-" + nodeName (with _ changed to -) - - // nodeParams := nodeParams{ - // deploymentID: deploymentID, - // nodeName: "Network_Subnet", - // infrastructure: &infrastructure, - // } - - // srv1.PopulateKV(t, map[string][]byte{ - // path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/instances/Network_VPC/0/attributes/vpc_id"): []byte("vpc_id"), - // }) - - // err := g.generateSubnet(ctx, nodeParams, "0", make(map[string]string)) - // require.NoError(t, err, "Unexpected error attempting to generate sub-network for %s", deploymentID) - - // instancesMap := infrastructure.Resource["aws_subnet"].(map[string]interface{}) - // require.Len(t, instancesMap, 1) - // require.Contains(t, instancesMap, subnetName) - - // subnet, ok := instancesMap[subnetName].(*Subnet) - // require.True(t, ok, "%s is not a Subnet", subnetName) - // assert.Equal(t, "10.0.0.0/24", subnet.CidrBlock) - // assert.Equal(t, "vpc_id", subnet.VPCId) + t.Parallel() + deploymentID := loadTestYaml(t) + ctx := context.Background() + infrastructure := commons.Infrastructure{} + g := awsGenerator{} + subnetName := "subnet-network-subnet" // deploymentID + "-" + nodeName (with _ changed to -) + + nodeParams := nodeParams{ + deploymentID: deploymentID, + nodeName: "Network_Subnet", + infrastructure: &infrastructure, + } + + err := g.generateSubnet(ctx, nodeParams, "0", make(map[string]string)) + require.NoError(t, err, "Unexpected error attempting to generate sub-network for %s", deploymentID) + + instancesMap := infrastructure.Resource["aws_subnet"].(map[string]interface{}) + require.Len(t, instancesMap, 1) + require.Contains(t, instancesMap, subnetName) + + subnet, ok := instancesMap[subnetName].(*Subnet) + require.True(t, ok, "%s is not a Subnet", subnetName) + assert.Equal(t, "10.0.0.0/24", subnet.CidrBlock) + assert.Equal(t, "vpc_id", subnet.VPCId) } From fbefb3f683da48158d30f7ff013071162b8ac687 Mon Sep 17 00:00:00 2001 From: hilderic Date: Wed, 22 Apr 2020 16:00:37 +0200 Subject: [PATCH 20/23] test+fix aws_instance --- prov/terraform/aws/aws_instance.go | 42 ++++++++-------- prov/terraform/aws/aws_instance_test.go | 48 ++++++++++--------- prov/terraform/aws/consul_test.go | 4 +- prov/terraform/aws/resources.go | 13 +++-- ...et.yaml => simpleAWSInstanceWitthVPC.yaml} | 17 ++----- prov/terraform/aws/vpc.go | 1 + 6 files changed, 64 insertions(+), 61 deletions(-) rename prov/terraform/aws/testdata/{simpleAWSInstanceWithVPCandSubnet.yaml => simpleAWSInstanceWitthVPC.yaml} (72%) diff --git a/prov/terraform/aws/aws_instance.go b/prov/terraform/aws/aws_instance.go index 7e75aa1f9..05b0bc928 100644 --- a/prov/terraform/aws/aws_instance.go +++ b/prov/terraform/aws/aws_instance.go @@ -161,26 +161,31 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi i := 0 for _, networkReq := range networkRequirements { networkInterface := &NetworkInterface{} - // TODO : Check if subnet and security group are defined in relationship - // networkInterface.SubnetID, err = deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "subnet_id") - // if err != nil { - // return err - // } - - // If not, use default one - defaultSubnetID, err := deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "default_subnet_id") - if err != nil { - return err - } - networkInterface.SubnetID = defaultSubnetID + networkInterface.SecurityGroups = make([]string, 1) + + if networkReq.Relationship == "tosca.relationships.Network" { + defaultSubnetID, err := deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "default_subnet_id") + if err != nil { + return err + } + networkInterface.SubnetID = defaultSubnetID + defaultSecurityGroup, err := deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "default_security_group") + if err != nil { + return err + } + networkInterface.SecurityGroups = append(networkInterface.SecurityGroups, defaultSecurityGroup) - // With default Security Group - defaultSecurityGroup, err := deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "default_security_group") - if err != nil { - return err + } else if networkReq.Relationship == "yorc.relationships.aws.Network" { + networkInterface.SubnetID, err = deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "subnet_id") + if err != nil { + return err + } + securityGroup, err := deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "subnet_id") + if err != nil { + return err + } + networkInterface.SecurityGroups = append(networkInterface.SecurityGroups, securityGroup) } - networkInterface.SecurityGroups = make([]string, 1) - networkInterface.SecurityGroups = append(networkInterface.SecurityGroups, defaultSecurityGroup) name := strings.ToLower("network-inteface-" + strconv.Itoa(i)) name = strings.Replace(strings.ToLower(name), "_", "-", -1) @@ -205,7 +210,6 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi } // No security groups on instance level when defining customs ENI - // (So the instance will have be in the default security groups on creation) instance.SecurityGroups = nil } diff --git a/prov/terraform/aws/aws_instance_test.go b/prov/terraform/aws/aws_instance_test.go index 8b0b1978e..a15f6ac52 100644 --- a/prov/terraform/aws/aws_instance_test.go +++ b/prov/terraform/aws/aws_instance_test.go @@ -375,33 +375,37 @@ func testSimpleAWSInstanceWithPersistentDisk(t *testing.T, cfg config.Configurat } -func simpleAWSInstanceWithVPCandSubnet(t *testing.T, cfg config.Configuration, srv *testutil.TestServer) { - // TODO - // t.Parallel() +func testSimpleAWSInstanceWitthVPC(t *testing.T, cfg config.Configuration, srv *testutil.TestServer) { + t.Parallel() - // deploymentID := loadTestYaml(t) - // g := awsGenerator{} - // infrastructure := commons.Infrastructure{} - // ctx := context.Background() - // env := make([]string, 0) - // outputs := make(map[string]string) + deploymentID := loadTestYaml(t) + g := awsGenerator{} + infrastructure := commons.Infrastructure{} + ctx := context.Background() + env := make([]string, 0) + outputs := make(map[string]string) - // srv.PopulateKV(t, map[string][]byte{ - // path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/nodes/Network_Subnet/type"): []byte("yorc.nodes.aws.Subnet"), - // path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/instances/Network_Subnet/0/attributes/subnet_id"): []byte("subnet_id"), - // }) + srv.PopulateKV(t, map[string][]byte{ + path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/nodes/Network/type"): []byte("yorc.nodes.aws.VPC"), + path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/instances/Network/0/attributes/default_subnet_id"): []byte("default_subnet_id"), + path.Join(consulutil.DeploymentKVPrefix, deploymentID+"/topology/instances/Network/0/attributes/default_security_group"): []byte("default_security_group"), + }) - // err := g.generateAWSInstance(ctx, cfg, deploymentID, "AWS_Compute", "0", &infrastructure, outputs, &env) - // require.Nil(t, err) - // require.Len(t, infrastructure.Resource["aws_instance"], 1) + err := g.generateAWSInstance(ctx, cfg, deploymentID, "AWS_Compute", "0", &infrastructure, outputs, &env) + require.Nil(t, err) + require.Len(t, infrastructure.Resource["aws_instance"], 1) - // instancesMap := infrastructure.Resource["aws_instance"].(map[string]interface{}) - // require.Contains(t, instancesMap, "AWS_Compute-0") + instancesMap := infrastructure.Resource["aws_instance"].(map[string]interface{}) + require.Contains(t, instancesMap, "AWS_Compute-0") - // compute, ok := instancesMap["AWS_Compute-0"].(*ComputeInstance) - // require.True(t, ok, "%s is not a ComputeInstance", "AWS_Compute-0") + compute, ok := instancesMap["AWS_Compute-0"].(*ComputeInstance) + require.True(t, ok, "%s is not a ComputeInstance", "AWS_Compute-0") - // assert.Equal(t, "${aws_network_interface.network-inteface-0.id}", compute.NetworkInterface["network_interface_id"], "Subnetwork is not retrieved") - // assert.Equal(t, "0", compute.NetworkInterface["device_index"], "Subnetwork is not retrieved") + assert.Equal(t, "${aws_network_interface.network-inteface-0.id}", compute.NetworkInterface["network_interface_id"], "Subnetwork is not retrieved") + assert.Equal(t, "0", compute.NetworkInterface["device_index"], "Subnetwork is not retrieved") + instancesMap = infrastructure.Resource["aws_network_interface"].(map[string]interface{}) + require.Contains(t, instancesMap, "network-inteface-0") + eni, ok := instancesMap["network-inteface-0"].(*NetworkInterface) + assert.Equal(t, "default_subnet_id", eni.SubnetID) } diff --git a/prov/terraform/aws/consul_test.go b/prov/terraform/aws/consul_test.go index ea1093882..8c69de00a 100644 --- a/prov/terraform/aws/consul_test.go +++ b/prov/terraform/aws/consul_test.go @@ -63,8 +63,8 @@ func TestRunConsulAWSPackageTests(t *testing.T) { t.Run("simpleAWSInstanceWithMalformedEIP", func(t *testing.T) { testSimpleAWSInstanceWithMalformedEIP(t, cfg) }) - t.Run("simpleAWSInstanceWithVPCandSubnet", func(t *testing.T) { - simpleAWSInstanceWithVPCandSubnet(t, cfg, srv) + t.Run("simpleAWSInstanceWitthVPC", func(t *testing.T) { + testSimpleAWSInstanceWitthVPC(t, cfg, srv) }) t.Run("generateTerraformInfraForAWSNode", func(t *testing.T) { testGenerateTerraformInfraForAWSNode(t, cfg, locationMgr) diff --git a/prov/terraform/aws/resources.go b/prov/terraform/aws/resources.go index 2bf6abce1..4e1bcd7c6 100644 --- a/prov/terraform/aws/resources.go +++ b/prov/terraform/aws/resources.go @@ -110,7 +110,7 @@ type NetworkInterface struct { Attachment map[string]string `json:"attachment,omitempty"` } -// todo +// SecurityRule provide a egress/ingress resource. type SecurityRule struct { FromPort string `json:"from_port,omitempty"` ToPort string `json:"to_port,omitempty"` @@ -118,7 +118,8 @@ type SecurityRule struct { CidrBlock []string `json:"cidr_blocks,omitempty"` } -// todo +// SecurityGroups provides a security group resource +// see : https://www.terraform.io/docs/providers/aws/r/security_group.html type SecurityGroups struct { VPCId string `json:"vpc_id,omitempty"` Egress SecurityRule `json:"egress,omitempty"` @@ -126,20 +127,24 @@ type SecurityGroups struct { Name string `json:"name,omitempty"` } -// todo +// RouteTable provides a resource to create a VPC routing table. +// see : https://www.terraform.io/docs/providers/aws/r/route_table.html type RouteTable struct { VPCId string `json:"vpc_id,omitempty"` Route map[string]string `json:"route,omitempty"` DependsOn []string `json:"depends_on,omitempty"` } +// DefaultRouteTable provides a resource to manage a Default VPC Routing Table +// see : https://www.terraform.io/docs/providers/aws/r/default_route_table.html type DefaultRouteTable struct { DefaultRouteTableID string `json:"default_route_table_id,omitempty"` Route map[string]string `json:"route,omitempty"` DependsOn []string `json:"depends_on,omitempty"` } -// todo +// InternetGateway provides a resource to create a VPC Internet Gateway +// see : https://www.terraform.io/docs/providers/aws/r/internet_gateway.html type InternetGateway struct { VPCId string `json:"vpc_id,omitempty"` } diff --git a/prov/terraform/aws/testdata/simpleAWSInstanceWithVPCandSubnet.yaml b/prov/terraform/aws/testdata/simpleAWSInstanceWitthVPC.yaml similarity index 72% rename from prov/terraform/aws/testdata/simpleAWSInstanceWithVPCandSubnet.yaml rename to prov/terraform/aws/testdata/simpleAWSInstanceWitthVPC.yaml index a698ee006..8c1fa9fbc 100644 --- a/prov/terraform/aws/testdata/simpleAWSInstanceWithVPCandSubnet.yaml +++ b/prov/terraform/aws/testdata/simpleAWSInstanceWitthVPC.yaml @@ -14,7 +14,7 @@ imports: topology_template: node_templates: - Network_VPC: + Network: type: yorc.nodes.aws.VPC properties: cidr_block: 10.0.0.0/16 @@ -30,7 +30,7 @@ topology_template: requirements: - AWS_Compute_network: type_requirement: network - node: Network_Subnet + node: Network capability: tosca.capabilities.Connectivity relationship: tosca.relationships.Network capabilities: @@ -45,15 +45,4 @@ topology_template: protocol: tcp network_name: PRIVATE initiator: source - credentials: {user: centos} - Network_Subnet: - type: yorc.nodes.aws.Subnet - properties: - cidr_block: 10.10.0.0/24 - availability_zone: "us-east-2c" - requirements: - - VPC_Requirements: - type_requirement: dependency - node: Network_VPC - capability: tosca.capabilities.Node - relationship: tosca.relationships.DependsOn \ No newline at end of file + credentials: {user: centos} \ No newline at end of file diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go index ed04fa265..930c98b4e 100644 --- a/prov/terraform/aws/vpc.go +++ b/prov/terraform/aws/vpc.go @@ -401,6 +401,7 @@ func (g *awsGenerator) generateVPCSecurityGroup(ctx context.Context, nodeParams securityGroup.Egress = SecurityRule{FromPort: fromPort, ToPort: toPort, Protocol: protocol} + securityGroup.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", vpcName) commons.AddResource(nodeParams.infrastructure, "aws_security_group", securityGroup.Name, securityGroup) return nil From ac366bad38d9b5d81947f286121bc3f142d16469 Mon Sep 17 00:00:00 2001 From: hilderic Date: Fri, 24 Apr 2020 10:20:06 +0200 Subject: [PATCH 21/23] clean up --- data/tosca/yorc-aws-types.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/data/tosca/yorc-aws-types.yml b/data/tosca/yorc-aws-types.yml index 1f1129c39..c9d12e6ef 100644 --- a/data/tosca/yorc-aws-types.yml +++ b/data/tosca/yorc-aws-types.yml @@ -72,17 +72,6 @@ data_types: type: yorc.datatypes.aws.SecurityRuleType required: true -relationship_types: - yorc.relationships.aws.Network: - derived_from: tosca.relationships.Network - properties: - subnet_id: - type: string - required: true - security_group: - type: string - required: true - node_types: yorc.nodes.aws.Compute: derived_from: yorc.nodes.Compute From f4cf22b8845c867c8514974765c32028e9a29550 Mon Sep 17 00:00:00 2001 From: Hilderic Date: Wed, 9 Sep 2020 15:57:06 +0200 Subject: [PATCH 22/23] VPC+Subnet relationship fix --- prov/terraform/aws/aws_instance.go | 65 ++++++++++++++++++++++++++---- prov/terraform/aws/generator.go | 5 --- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/prov/terraform/aws/aws_instance.go b/prov/terraform/aws/aws_instance.go index 05b0bc928..323f9cc13 100644 --- a/prov/terraform/aws/aws_instance.go +++ b/prov/terraform/aws/aws_instance.go @@ -148,11 +148,13 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi if subnetID != nil { instance.SubnetID = subnetID.RawString() } - hasNetwork, _, err := deployments.HasAnyRequirementFromNodeType(ctx, deploymentID, nodeName, "network", "yorc.nodes.aws.VPC") + + // VPC + hasVPCNetwork, _, err := deployments.HasAnyRequirementFromNodeType(ctx, deploymentID, nodeName, "network", "yorc.nodes.aws.VPC") if err != nil { return err } - if hasNetwork { + if hasVPCNetwork { networkRequirements, err := deployments.GetRequirementsByTypeForNode(ctx, deploymentID, nodeName, "network") if err != nil { return err @@ -190,27 +192,76 @@ func (g *awsGenerator) generateAWSInstance(ctx context.Context, cfg config.Confi name := strings.ToLower("network-inteface-" + strconv.Itoa(i)) name = strings.Replace(strings.ToLower(name), "_", "-", -1) - // First interface will be considered as default one + // First interface will be considered the network interface of the Compute Instance if i == 0 { instance.NetworkInterface = map[string]string{ "network_interface_id": "${aws_network_interface." + name + ".id}", "device_index": strconv.Itoa(i), } } else { - // Others interfaces are considered attachment + // Others interfaces are considered as attachment to the Compute Instance networkInterface.Attachment = map[string]string{ "instance": "${aws_instance." + instance.Tags.Name + ".id}", "device_index": strconv.Itoa(i), } } + // No security groups on instance level when defining customs ENI, it will use a default one created by the VPC + instance.SecurityGroups = nil commons.AddResource(infrastructure, "aws_network_interface", name, networkInterface) - i++ } + } + + // Subnet + hasSubnetNetwork, _, err := deployments.HasAnyRequirementFromNodeType(ctx, deploymentID, nodeName, "network", "yorc.nodes.aws.Subnet") + if err != nil { + return err + } + if hasSubnetNetwork && !hasVPCNetwork { + networkRequirements, err := deployments.GetRequirementsByTypeForNode(ctx, deploymentID, nodeName, "network") + if err != nil { + return err + } + + i := 0 + for _, networkReq := range networkRequirements { + networkInterface := &NetworkInterface{} + networkInterface.SecurityGroups = make([]string, 1) + + if networkReq.Relationship == "tosca.relationships.Network" { + subnetID, err := deployments.LookupInstanceAttributeValue(ctx, deploymentID, networkReq.RequirementAssignment.Node, instanceName, "subnet_id") + if err != nil { + return err + } + networkInterface.SubnetID = subnetID - // No security groups on instance level when defining customs ENI - instance.SecurityGroups = nil + networkInterface.SecurityGroups = instance.SecurityGroups + + } + + name := strings.ToLower("network-inteface-" + strconv.Itoa(i)) + name = strings.Replace(strings.ToLower(name), "_", "-", -1) + + // First interface will be considered the network interface of the Compute Instance + if i == 0 { + instance.NetworkInterface = map[string]string{ + "network_interface_id": "${aws_network_interface." + name + ".id}", + "device_index": strconv.Itoa(i), + } + } else { + // Others interfaces are considered as attachment to the Compute Instance + networkInterface.Attachment = map[string]string{ + "instance": "${aws_instance." + instance.Tags.Name + ".id}", + "device_index": strconv.Itoa(i), + } + } + + // No security groups on instance level when defining customs ENI, it will use a default one created by the VPC + instance.SecurityGroups = nil + commons.AddResource(infrastructure, "aws_network_interface", name, networkInterface) + i++ + } } // Add the AWS instance diff --git a/prov/terraform/aws/generator.go b/prov/terraform/aws/generator.go index 35cf02096..d7718027c 100644 --- a/prov/terraform/aws/generator.go +++ b/prov/terraform/aws/generator.go @@ -125,11 +125,6 @@ func (g *awsGenerator) generateInstances(ctx context.Context, cfg config.Configu switch nodeType { case "yorc.nodes.aws.Compute": - *instances, err = deployments.GetNodeInstancesIds(ctx, nodeParams.deploymentID, nodeParams.nodeName) - if err != nil { - return err - } - err = g.generateAWSInstance(ctx, cfg, nodeParams.deploymentID, nodeParams.nodeName, instanceName, nodeParams.infrastructure, outputs, cmdEnv) if err != nil { return err From c0641d82439f71e1ee203b7fbe26bddcf112f6f4 Mon Sep 17 00:00:00 2001 From: Hilderic Date: Fri, 11 Sep 2020 14:03:07 +0200 Subject: [PATCH 23/23] Removing apify, only SSH connection allowed --- prov/terraform/aws/vpc.go | 18 +++--------------- prov/terraform/aws/vpc_test.go | 9 +-------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/prov/terraform/aws/vpc.go b/prov/terraform/aws/vpc.go index 930c98b4e..61749b0e3 100644 --- a/prov/terraform/aws/vpc.go +++ b/prov/terraform/aws/vpc.go @@ -17,8 +17,6 @@ package aws import ( "context" "fmt" - "io/ioutil" - "net/http" "path" "strconv" "strings" @@ -327,20 +325,10 @@ func (g *awsGenerator) generateVPCSecurityGroups(ctx context.Context, nodeParams } func (g *awsGenerator) generateDefaultSecurityGroup(ctx context.Context, nodeParams nodeParams, vpcName string, nodeKey string, outputs map[string]string) error { - url := "https://api.ipify.org" - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - ip, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - + // Only allow connection SSH connection (port 22 and TCP) securityGroup := &SecurityGroups{} - securityGroup.Egress = SecurityRule{"0", "0", "-1", []string{string(ip) + "/32"}} - securityGroup.Ingress = SecurityRule{"0", "0", "-1", []string{string(ip) + "/32"}} + securityGroup.Egress = SecurityRule{"22", "22", "TCP", []string{"0.0.0.0/0"}} + securityGroup.Ingress = SecurityRule{"22", "22", "TCP", []string{"0.0.0.0/0"}} securityGroup.Name = "Default " + vpcName securityGroup.VPCId = fmt.Sprintf("${aws_vpc.%s.id}", vpcName) diff --git a/prov/terraform/aws/vpc_test.go b/prov/terraform/aws/vpc_test.go index 6256aa2a0..776874812 100644 --- a/prov/terraform/aws/vpc_test.go +++ b/prov/terraform/aws/vpc_test.go @@ -17,8 +17,6 @@ package aws import ( "context" "fmt" - "io/ioutil" - "net/http" "strings" "testing" @@ -84,13 +82,8 @@ func testVPC(t *testing.T, cfg config.Configuration) { securityGroup, ok := instancesMap[defaultSecurityGroupName].(*SecurityGroups) require.True(t, ok, "%s is not a Subnet", defaultSubnetName) - url := "https://api.ipify.org" - resp, err := http.Get(url) - require.NoError(t, err, "Error GET with api.ipify", deploymentID) - defer resp.Body.Close() - ip, err := ioutil.ReadAll(resp.Body) require.NoError(t, err, "Can't read IP on ipify response", deploymentID) - assert.Equal(t, SecurityRule{"0", "0", "-1", []string{string(ip) + "/32"}}, securityGroup.Egress) + assert.Equal(t, SecurityRule{"22", "22", "TCP", []string{"0.0.0.0/0"}}, securityGroup.Egress) } func testVPCWithNestedSubnetAndSG(t *testing.T, srv1 *testutil.TestServer, cfg config.Configuration) {