diff --git a/deploy.sh b/deploy.sh index dcd597b9..cbcfa824 100755 --- a/deploy.sh +++ b/deploy.sh @@ -50,7 +50,7 @@ function template_protection() --profile "$CURRENT_PROFILE_NAME" } -crossaccount_cicd_roles () { +crossaccount_cicd_roles () { pflag=false rflag=false dflag=false @@ -132,7 +132,7 @@ crossaccount_cicd_roles () { exit } -devops_account () { +devops_account () { pflag=false rflag=false dflag=false @@ -257,7 +257,7 @@ devops_account () { template_protection "$STACK_NAME" "$REGION" "$DEVOPS_AWS_PROFILE" rm -Rf "$DIRNAME"/output - declare -a REPOSITORIES=("sdlf-cicd" "sdlf-foundations" "sdlf-team" "sdlf-pipeline" "sdlf-dataset" "sdlf-datalakeLibrary" "sdlf-stageA" "sdlf-stageB" "sdlf-main") + declare -a REPOSITORIES=("sdlf-cicd" "sdlf-foundations" "sdlf-team" "sdlf-pipeline" "sdlf-dataset" "sdlf-datalakeLibrary" "sdlf-stageA" "sdlf-stageB" "sdlf-main" "sdlf-stage-lambda" "sdlf-stage-glue") if "$MONITORING" then REPOSITORIES+=("sdlf-monitoring") @@ -268,8 +268,23 @@ devops_account () { then GITLAB_URL=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --with-decryption --name /SDLF/GitLab/Url --query "Parameter.Value" --output text) GITLAB_ACCESSTOKEN=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --with-decryption --name /SDLF/GitLab/AccessToken --query "Parameter.Value" --output text) - GITLAB_REPOSITORY_URL="https://aws:$GITLAB_ACCESSTOKEN@${GITLAB_URL#https://}sdlf/$REPOSITORY.git" + GITLAB_NAMESPACE_ID=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --with-decryption --name /SDLF/GitLab/NamespaceId --query "Parameter.Value" --output text) + GITLAB_GROUP_NAME=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --name /SDLF/GitLab/SdlfGitLabGroup --query "Parameter.Value" --output text) + + GITLAB_HOST_NAME=gitlab.ssh.covestro.com + + echo "Creating $REPOSITORY repository in GitLab ..." + curl --insecure --request POST --header "PRIVATE-TOKEN: $GITLAB_ACCESSTOKEN" \ + --header "Content-Type: application/json" \ + --data "{\"name\": \"$REPOSITORY\", \"description\": \"$REPOSITORY\", \"path\": \"$REPOSITORY\", \"namespace_id\": \"$GITLAB_NAMESPACE_ID\", \"initialize_with_readme\": false}" \ + --url "${GITLAB_URL}api/v4/projects/" + + + GITLAB_REPOSITORY_URL="https://aws:$GITLAB_ACCESSTOKEN@${GITLAB_URL#https://}${GITLAB_GROUP_NAME}/$REPOSITORY.git" + GITLAB_SSH_URI=git@${GITLAB_HOST_NAME}:${GITLAB_GROUP_NAME}/$REPOSITORY.git + + echo "Origin for repo is $GITLAB_SSH_URI" if [ "$REPOSITORY" = "sdlf-main" ] then mkdir sdlf-main @@ -278,8 +293,9 @@ devops_account () { pushd "$REPOSITORY" || exit if [ ! -d .git ] # if .git exists, deploy.sh has likely been run before - do not try to push the base repositories then - git init - git remote add origin "$GITLAB_REPOSITORY_URL" || exit 1 + git init --initial-branch=main + git remote rename origin old-origin + git remote add origin "$GITLAB_SSH_URI" || exit 1 git add . git commit -m "initial commit" git push origin main || exit 1 @@ -295,7 +311,7 @@ devops_account () { done aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" s3api put-object --bucket "$ARTIFACTS_BUCKET" --key sam-translate.py --body "$DIRNAME"/sdlf-cicd/sam-translate.py - curl -L -O --output-dir "$DIRNAME"/sdlf-cicd/ https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip + curl -L -O --insecure --output-dir "$DIRNAME"/sdlf-cicd/ https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" s3api put-object --bucket "$ARTIFACTS_BUCKET" --key aws-sam-cli-linux-x86_64.zip --body "$DIRNAME"/sdlf-cicd/aws-sam-cli-linux-x86_64.zip rm "$DIRNAME"/sdlf-cicd/aws-sam-cli-linux-x86_64.zip diff --git a/sdlf-cicd/lambda/domain-cicd/src/lambda_function.py b/sdlf-cicd/lambda/domain-cicd/src/lambda_function.py index eb436d18..ee85e6f2 100644 --- a/sdlf-cicd/lambda/domain-cicd/src/lambda_function.py +++ b/sdlf-cicd/lambda/domain-cicd/src/lambda_function.py @@ -2,8 +2,10 @@ import logging import os import zipfile +import ssl from io import BytesIO from tempfile import mkdtemp +from urllib.request import HTTPError, Request, URLError, urlopen import boto3 from botocore.client import Config @@ -163,6 +165,34 @@ def delete_domain_team_role_stack(cloudformation, team): def create_team_repository_cicd_stack(domain, team_name, template_body_url, cloudformation_role): + gitlab_url = ssm.get_parameter(Name="/SDLF/GitLab/Url", WithDecryption=True)["Parameter"]["Value"] + gitlab_accesstoken = ssm.get_parameter(Name="/SDLF/GitLab/AccessToken", WithDecryption=True)["Parameter"]["Value"] + repository = f"sdlf-main-{domain}-{team_name}" + namespace_id = ssm.get_parameter(Name="/SDLF/GitLab/NamespaceId", WithDecryption=True)["Parameter"]["Value"] + url = f"{gitlab_url}api/v4/projects/" + headers = { + "Content-Type": "application/json", + "PRIVATE-TOKEN": gitlab_accesstoken + } + data = { + "name": repository, + "description": repository, + "path": repository, + "namespace_id": namespace_id, + "initialize_with_readme": "false" + } + json_data = json.dumps(data).encode('utf-8') + req = Request(url, data=json_data, headers=headers, method='POST') + unverified_context = ssl._create_unverified_context() + try: + with urlopen(req, context=unverified_context) as response: + response_body = response.read().decode('utf-8') + logger.info(response_body) + except HTTPError as e: + logger.warn(f"HTTP error occurred: {e.code} {e.reason}. Most likely the repository {repository} already exists") + except URLError as e: + logger.error(f"URL error occurred: {e.reason}") + response = {} cloudformation_waiter_type = None stack_name = f"sdlf-cicd-teams-{domain}-{team_name}-repository" @@ -289,11 +319,21 @@ def create_team_pipeline_cicd_stack( "ParameterValue": f"/SDLF/{git_platform}/StageA{git_platform}", "UsePreviousValue": False, }, + { + "ParameterKey": "pStageLambdaRepository", + "ParameterValue": f"/SDLF/{git_platform}/StageLambda{git_platform}", + "UsePreviousValue": False, + }, { "ParameterKey": "pStageBRepository", "ParameterValue": f"/SDLF/{git_platform}/StageB{git_platform}", "UsePreviousValue": False, }, + { + "ParameterKey": "pStageGlueRepository", + "ParameterValue": f"/SDLF/{git_platform}/StageGlue{git_platform}", + "UsePreviousValue": False, + }, { "ParameterKey": "pDatasetRepository", "ParameterValue": f"/SDLF/{git_platform}/Dataset{git_platform}", @@ -361,11 +401,21 @@ def create_team_pipeline_cicd_stack( "ParameterValue": f"/SDLF/{git_platform}/StageA{git_platform}", "UsePreviousValue": False, }, + { + "ParameterKey": "pStageLambdaRepository", + "ParameterValue": f"/SDLF/{git_platform}/StageLambda{git_platform}", + "UsePreviousValue": False, + }, { "ParameterKey": "pStageBRepository", "ParameterValue": f"/SDLF/{git_platform}/StageB{git_platform}", "UsePreviousValue": False, }, + { + "ParameterKey": "pStageGlueRepository", + "ParameterValue": f"/SDLF/{git_platform}/StageGlue{git_platform}", + "UsePreviousValue": False, + }, { "ParameterKey": "pDatasetRepository", "ParameterValue": f"/SDLF/{git_platform}/Dataset{git_platform}", @@ -668,4 +718,4 @@ def lambda_handler(event, context): raise codepipeline.put_job_success_result(jobId=event["CodePipeline.job"]["id"]) - return "Success" + return "Success" \ No newline at end of file diff --git a/sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml b/sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml index 4283514b..99b902bd 100644 --- a/sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml +++ b/sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml @@ -90,10 +90,6 @@ Resources: - ec2:DeleteNetworkInterface # W11 condition applied Resource: - "*" - Condition: - ArnEqualsIfExists: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -101,13 +97,10 @@ Resources: Action: - ec2:CreateNetworkInterfacePermission Resource: - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*" Condition: StringEquals: "ec2:AuthorizedService": codebuild.amazonaws.com - ArnEquals: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" - PolicyName: sdlf-cicd-build-stages-cfn-modules PolicyDocument: @@ -127,7 +120,7 @@ Resources: - codecommit:GetUploadArchiveStatus - codecommit:CancelUploadArchive Resource: - - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pStagesRepositoriesPrefix}* + - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:*:${pStagesRepositoriesPrefix}* rBuildCloudformationModuleStage: Type: AWS::CodeBuild::Project @@ -139,8 +132,13 @@ Resources: EncryptionKey: !Ref pKMSKey VpcConfig: !If - RunInVpc - - SecurityGroupIds: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-security-groups] - Subnets: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets] + - SecurityGroupIds: + !Split [ + ",", + !ImportValue sdlf-cicd-prerequisites-vpc-security-groups, + ] + Subnets: + !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets] VpcId: "{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" Environment: @@ -167,9 +165,8 @@ Resources: && unzip -q aws-sam-cli-linux-x86_64.zip -d sam-installation ./sam-installation/install \ && sam --version - - |- - pip3 install cfn-lint==0.87.7 - pip3 install cloudformation-cli + - pip3 install cfn-lint==0.87.7 + - pip3 install cloudformation-cli - aws s3api get-object --bucket "$ARTIFACTS_BUCKET" --key sam-translate.py sam-translate.py build: commands: @@ -246,8 +243,13 @@ Resources: EncryptionKey: !Ref pKMSKey VpcConfig: !If - RunInVpc - - SecurityGroupIds: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-security-groups] - Subnets: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets] + - SecurityGroupIds: + !Split [ + ",", + !ImportValue sdlf-cicd-prerequisites-vpc-security-groups, + ] + Subnets: + !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets] VpcId: "{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" Environment: diff --git a/sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml b/sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml index ff4100c3..72412d8b 100644 --- a/sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml +++ b/sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml @@ -87,10 +87,6 @@ Resources: - ec2:DeleteNetworkInterface # W11 condition applied Resource: - "*" - Condition: - ArnEqualsIfExists: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -98,13 +94,10 @@ Resources: Action: - ec2:CreateNetworkInterfacePermission Resource: - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*" Condition: StringEquals: "ec2:AuthorizedService": codebuild.amazonaws.com - ArnEquals: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" rGlueJobPackage: diff --git a/sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml b/sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml index 6fc1b989..b566f945 100644 --- a/sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml +++ b/sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml @@ -83,10 +83,6 @@ Resources: - ec2:DeleteNetworkInterface # W11 condition applied Resource: - "*" - Condition: - ArnEqualsIfExists: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -94,13 +90,10 @@ Resources: Action: - ec2:CreateNetworkInterfacePermission Resource: - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*" Condition: StringEquals: "ec2:AuthorizedService": codebuild.amazonaws.com - ArnEquals: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" rBuildLambdaLayersPackage: diff --git a/sdlf-cicd/template-cicd-domain-roles.yaml b/sdlf-cicd/template-cicd-domain-roles.yaml index c20ee0f3..ed572e5a 100644 --- a/sdlf-cicd/template-cicd-domain-roles.yaml +++ b/sdlf-cicd/template-cicd-domain-roles.yaml @@ -427,12 +427,6 @@ Resources: - lambda:CreateFunction - lambda:UpdateFunctionConfiguration Resource: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:sdlf-* - Condition: !If - - RunInVpc - - StringEquals: - "lambda:VpcIds": - - "{{resolve:ssm:/SDLF/VPC/VpcId}}" - - !Ref "AWS::NoValue" - Effect: Allow Action: - lambda:AddPermission diff --git a/sdlf-cicd/template-cicd-domain-team-role.yaml b/sdlf-cicd/template-cicd-domain-team-role.yaml index 6d9b905e..fd4e3436 100644 --- a/sdlf-cicd/template-cicd-domain-team-role.yaml +++ b/sdlf-cicd/template-cicd-domain-team-role.yaml @@ -293,12 +293,6 @@ Resources: - lambda:CreateFunction - lambda:UpdateFunctionConfiguration Resource: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:sdlf-${pTeamName}-* - Condition: !If - - RunInVpc - - StringEquals: - "lambda:VpcIds": - - "{{resolve:ssm:/SDLF/VPC/VpcId}}" - - !Ref "AWS::NoValue" - Effect: Allow Action: - lambda:AddPermission diff --git a/sdlf-cicd/template-cicd-sdlf-pipelines.yaml b/sdlf-cicd/template-cicd-sdlf-pipelines.yaml index e7329b13..877d253f 100644 --- a/sdlf-cicd/template-cicd-sdlf-pipelines.yaml +++ b/sdlf-cicd/template-cicd-sdlf-pipelines.yaml @@ -340,6 +340,7 @@ Resources: - ssm:GetParametersByPath Resource: - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Misc/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/GitLab/* - Effect: Allow Action: - s3:PutObject @@ -663,10 +664,6 @@ Resources: - ec2:DeleteNetworkInterface # W11 condition applied Resource: - "*" - Condition: - ArnEqualsIfExists: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -674,13 +671,10 @@ Resources: Action: - ec2:CreateNetworkInterfacePermission Resource: - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*" Condition: StringEquals: "ec2:AuthorizedService": codebuild.amazonaws.com - ArnEquals: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" rCloudFormationPackageCodeBuildProject: @@ -791,10 +785,6 @@ Resources: - ec2:DeleteNetworkInterface # W11 condition applied Resource: - "*" - Condition: - ArnEqualsIfExists: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -802,13 +792,10 @@ Resources: Action: - ec2:CreateNetworkInterfacePermission Resource: - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*" Condition: StringEquals: "ec2:AuthorizedService": codebuild.amazonaws.com - ArnEquals: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" diff --git a/sdlf-cicd/template-cicd-sdlf-repositories.gitlab.yaml b/sdlf-cicd/template-cicd-sdlf-repositories.gitlab.yaml index a5b299d6..7b9c7e2e 100644 --- a/sdlf-cicd/template-cicd-sdlf-repositories.gitlab.yaml +++ b/sdlf-cicd/template-cicd-sdlf-repositories.gitlab.yaml @@ -8,7 +8,7 @@ Parameters: Default: /SDLF/KMS/CICDKeyId pSdlfGitLabGroup: Type: String - Default: sdlf + Default: covestro-analytics-platform/datamesh pCicdRepository: Type: String Default: sdlf-cicd @@ -27,9 +27,15 @@ Parameters: pStageARepository: Type: String Default: sdlf-stageA + pStageLambdaRepository: + Type: String + Default: sdlf-stage-lambda pStageBRepository: Type: String Default: sdlf-stageB + pStageGlueRepository: + Type: String + Default: sdlf-stage-glue pDatalakeLibraryRepository: Type: String Default: sdlf-datalakeLibrary @@ -58,117 +64,6 @@ Resources: # Name: SDLF # Path: !Ref pSdlfGitLabGroup - rCicdGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Properties: - Name: !Ref pCicdRepository -# Path: !Ref pSdlfGitLabGroup - - rFoundationsGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Properties: - Name: !Ref pFoundationsRepository -# Path: !Ref pSdlfGitLabGroup - - rTeamGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Properties: - Name: !Ref pTeamRepository -# Path: !Ref pSdlfGitLabGroup - - rPipelineGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Properties: - Name: !Ref pPipelineRepository -# Path: !Ref pSdlfGitLabGroup - - rDatasetGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Properties: - Name: !Ref pDatasetRepository -# Path: !Ref pSdlfGitLabGroup - - rStageAGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Properties: - Name: !Ref pStageARepository -# Path: !Ref pSdlfGitLabGroup - - rStageBGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Properties: - Name: !Ref pStageBRepository -# Path: !Ref pSdlfGitLabGroup - - rDatalakeLibraryGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Properties: - Name: !Ref pDatalakeLibraryRepository -# Path: !Ref pSdlfGitLabGroup - - rMainGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Properties: - Name: !Ref pMainRepository -# Path: !Ref pSdlfGitLabGroup - - rMonitoringGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Condition: EnableMonitoring - Properties: - Name: !Ref pMonitoringRepository -# Path: !Ref pSdlfGitLabGroup - rSdlfGitLabGroupSsm: Type: AWS::SSM::Parameter Properties: @@ -225,6 +120,14 @@ Resources: Value: !Ref pStageARepository # !GetAtt rStageAGitLab.Name Description: Name of the StageA repository + rStageLambdaGitLabSsm: + Type: AWS::SSM::Parameter + Properties: + Name: /SDLF/GitLab/StageLambdaGitLab + Type: String + Value: !Ref pStageLambdaRepository # !GetAtt rStageLambdaGitLab.Name + Description: Name of the Stage-Lambda repository + rStageBGitLabSsm: Type: AWS::SSM::Parameter Properties: @@ -233,6 +136,14 @@ Resources: Value: !Ref pStageBRepository # !GetAtt rStageBGitLab.Name Description: Name of the StageB repository + rStageGlueGitLabSsm: + Type: AWS::SSM::Parameter + Properties: + Name: /SDLF/GitLab/StageGlueGitLab + Type: String + Value: !Ref pStageGlueRepository # !GetAtt rStageGlueGitLab.Name + Description: Name of the Stage-Glue repository + rDatalakeLibraryGitLabSsm: Type: AWS::SSM::Parameter Properties: diff --git a/sdlf-cicd/template-cicd-team-pipeline.yaml b/sdlf-cicd/template-cicd-team-pipeline.yaml index 283fee9a..14fc8af1 100644 --- a/sdlf-cicd/template-cicd-team-pipeline.yaml +++ b/sdlf-cicd/template-cicd-team-pipeline.yaml @@ -71,9 +71,15 @@ Parameters: pStageARepository: Type: AWS::SSM::Parameter::Value Default: /SDLF/CodeCommit/StageACodeCommit + pStageLambdaRepository: + Type: AWS::SSM::Parameter::Value + Default: /SDLF/CodeCommit/StageLambdaCodeCommit pStageBRepository: Type: AWS::SSM::Parameter::Value Default: /SDLF/CodeCommit/StageBCodeCommit + pStageGlueRepository: + Type: AWS::SSM::Parameter::Value + Default: /SDLF/CodeCommit/StageGlueCodeCommit pDatasetRepository: Type: AWS::SSM::Parameter::Value Default: /SDLF/CodeCommit/DatasetCodeCommit @@ -158,7 +164,9 @@ Resources: - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pDatalakeLibraryRepository}" - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pPipelineRepository}" - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageARepository}" + - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageLambdaRepository}" - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageBRepository}" + - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageGlueRepository}" - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pDatasetRepository}" "codestar-connections:FullRepositoryId": - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/{{resolve:ssm:/SDLF/${pGitPlatform}/${pTeamName}/Main${pGitPlatform}}}" @@ -166,7 +174,9 @@ Resources: - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pDatalakeLibraryRepository}" - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pPipelineRepository}" - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageARepository}" + - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageLambdaRepository}" - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageBRepository}" + - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageGlueRepository}" - !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pDatasetRepository}" - !Ref "AWS::NoValue" - Effect: Allow @@ -380,6 +390,20 @@ Resources: BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch] OutputArtifactFormat: CODE_ZIP RunOrder: 1 + - Name: SourceStageLambda + ActionTypeId: + Category: Source + Owner: AWS + Provider: CodeStarSourceConnection + Version: "1" + OutputArtifacts: + - Name: SourceStageLambdaArtifact + Configuration: + ConnectionArn: !Sub "{{resolve:ssm:/SDLF/${pGitPlatform}/CodeConnection}}" + FullRepositoryId: !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageLambdaRepository}" + BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch] + OutputArtifactFormat: CODE_ZIP + RunOrder: 1 - Name: SourceStageB ActionTypeId: Category: Source @@ -394,6 +418,20 @@ Resources: BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch] OutputArtifactFormat: CODE_ZIP RunOrder: 1 + - Name: SourceStageGlue + ActionTypeId: + Category: Source + Owner: AWS + Provider: CodeStarSourceConnection + Version: "1" + OutputArtifacts: + - Name: SourceStageGlueArtifact + Configuration: + ConnectionArn: !Sub "{{resolve:ssm:/SDLF/${pGitPlatform}/CodeConnection}}" + FullRepositoryId: !Sub "{{resolve:ssm:/SDLF/GitLab/SdlfGitLabGroup}}/${pStageGlueRepository}" + BranchName: !FindInMap [pCodeCommitBranch, !Ref pEnvironment, branch] + OutputArtifactFormat: CODE_ZIP + RunOrder: 1 - Name: SourceDataset ActionTypeId: Category: Source @@ -579,6 +617,26 @@ Resources: {"name":"TEAM_NAME", "value":"${pSdlfModuleTeam}", "type":"PLAINTEXT"}, {"name":"MODULE_NAME", "value":"stageA", "type":"PLAINTEXT"}] RunOrder: 1 + - + Name: BuildStageLambda + InputArtifacts: + - Name: SourceCicdArtifact + - Name: SourceStageLambdaArtifact + ActionTypeId: + Category: Build + Owner: AWS + Version: "1" + Provider: CodeBuild + Configuration: + PrimarySource: SourceStageLambdaArtifact + ProjectName: !Ref pBuildCloudformationModuleStage + EnvironmentVariables: !Sub >- + [{"name":"ENVIRONMENT", "value":"${pEnvironment}", "type":"PLAINTEXT"}, + {"name":"DOMAIN_NAME", "value":"${pSdlfModuleDomain}", "type":"PLAINTEXT"}, + {"name":"DOMAIN_ACCOUNT_ID", "value":"${pChildAccountId}", "type":"PLAINTEXT"}, + {"name":"TEAM_NAME", "value":"${pSdlfModuleTeam}", "type":"PLAINTEXT"}, + {"name":"MODULE_NAME", "value":"stageLambda", "type":"PLAINTEXT"}] + RunOrder: 1 - Name: BuildStageB InputArtifacts: @@ -599,6 +657,26 @@ Resources: {"name":"TEAM_NAME", "value":"${pSdlfModuleTeam}", "type":"PLAINTEXT"}, {"name":"MODULE_NAME", "value":"stageB", "type":"PLAINTEXT"}] RunOrder: 1 + - + Name: BuildStageGlue + InputArtifacts: + - Name: SourceCicdArtifact + - Name: SourceStageGlueArtifact + ActionTypeId: + Category: Build + Owner: AWS + Version: "1" + Provider: CodeBuild + Configuration: + PrimarySource: SourceStageGlueArtifact + ProjectName: !Ref pBuildCloudformationModuleStage + EnvironmentVariables: !Sub >- + [{"name":"ENVIRONMENT", "value":"${pEnvironment}", "type":"PLAINTEXT"}, + {"name":"DOMAIN_NAME", "value":"${pSdlfModuleDomain}", "type":"PLAINTEXT"}, + {"name":"DOMAIN_ACCOUNT_ID", "value":"${pChildAccountId}", "type":"PLAINTEXT"}, + {"name":"TEAM_NAME", "value":"${pSdlfModuleTeam}", "type":"PLAINTEXT"}, + {"name":"MODULE_NAME", "value":"stageGlue", "type":"PLAINTEXT"}] + RunOrder: 1 - Name: BuildDataset InputArtifacts: @@ -773,4 +851,4 @@ Resources: Targets: - Arn: !Sub arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:${rTeamCodePipeline} RoleArn: !GetAtt rTeamMainCodeCommitTriggerRole.Arn - Id: sdlf-cicd-team + Id: sdlf-cicd-team \ No newline at end of file diff --git a/sdlf-cicd/template-cicd-team-repository.yaml b/sdlf-cicd/template-cicd-team-repository.yaml index cbac9d6b..95569972 100644 --- a/sdlf-cicd/template-cicd-team-repository.yaml +++ b/sdlf-cicd/template-cicd-team-repository.yaml @@ -23,7 +23,6 @@ Parameters: Conditions: CodeCommitNoGitLab: !Equals [!Ref pGitPlatform, "CodeCommit"] - GitLabNoCodeCommit: !Equals [!Ref pGitPlatform, "GitLab"] Resources: rTeamMainCodeCommit: @@ -42,18 +41,6 @@ Resources: RepositoryName: !Sub ${pMainRepositoriesPrefix}${pDomain}-${pTeamName} KmsKeyId: !Ref pKMSKey - rTeamMainGitLab: - Type: GitLab::Projects::Project - Metadata: - cfn-lint: - config: - ignore_checks: - - E3001 - Condition: GitLabNoCodeCommit - Properties: - Name: !Sub ${pMainRepositoriesPrefix}${pDomain}-${pTeamName} -# Path: !Ref pSdlfGitLabGroup - rTeamMainCodeCommitSsm: Type: AWS::SSM::Parameter Properties: diff --git a/sdlf-monitoring/template.yaml b/sdlf-monitoring/template.yaml index 199515dd..f6763141 100644 --- a/sdlf-monitoring/template.yaml +++ b/sdlf-monitoring/template.yaml @@ -433,10 +433,6 @@ Resources: - ec2:CreateNetworkInterface # W12 exception - ec2:DeleteNetworkInterface # W12 exception Resource: "*" - Condition: - ArnEqualsIfExists: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -444,7 +440,7 @@ Resources: Action: - ec2:DescribeVpcAttribute Resource: - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" - !If - RunInVpc @@ -452,13 +448,10 @@ Resources: Action: - ec2:CreateNetworkInterfacePermission Resource: - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*" Condition: StringEquals: "ec2:AuthorizedService": firehose.amazonaws.com - ArnEquals: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" - !Ref "AWS::NoValue" - Effect: Allow Action: diff --git a/sdlf-team/template.yaml b/sdlf-team/template.yaml index a3609e1c..fc76cb43 100644 --- a/sdlf-team/template.yaml +++ b/sdlf-team/template.yaml @@ -548,9 +548,6 @@ Resources: Resource: - "*" Condition: - ArnEqualsIfExists: - "ec2:Vpc": - - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}" "ForAllValues:StringEqualsIfExists": "aws:TagKeys": - aws-glue-service-resource