Gitlab with curl - synching with main head

deploy.sh

@@ -50,7 +50,7 @@ function template_protection()
         --profile "$CURRENT_PROFILE_NAME"
 }
 
-crossaccount_cicd_roles () { 
+crossaccount_cicd_roles () {
     pflag=false
     rflag=false
     dflag=false
@@ -132,7 +132,7 @@ crossaccount_cicd_roles () {
     exit
 }
 
-devops_account () { 
+devops_account () {
     pflag=false
     rflag=false
     dflag=false
@@ -257,7 +257,7 @@ devops_account () {
     template_protection "$STACK_NAME" "$REGION" "$DEVOPS_AWS_PROFILE"
     rm -Rf "$DIRNAME"/output
 
-    declare -a REPOSITORIES=("sdlf-cicd" "sdlf-foundations" "sdlf-team" "sdlf-pipeline" "sdlf-dataset" "sdlf-datalakeLibrary" "sdlf-stageA" "sdlf-stageB" "sdlf-main")
+    declare -a REPOSITORIES=("sdlf-cicd" "sdlf-foundations" "sdlf-team" "sdlf-pipeline" "sdlf-dataset" "sdlf-datalakeLibrary" "sdlf-stageA" "sdlf-stageB" "sdlf-main" "sdlf-stage-lambda" "sdlf-stage-glue")
     if "$MONITORING"
     then
         REPOSITORIES+=("sdlf-monitoring")
@@ -268,8 +268,23 @@ devops_account () {
         then
             GITLAB_URL=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --with-decryption --name /SDLF/GitLab/Url --query "Parameter.Value" --output text)
             GITLAB_ACCESSTOKEN=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --with-decryption --name /SDLF/GitLab/AccessToken --query "Parameter.Value" --output text)
-            GITLAB_REPOSITORY_URL="https://aws:$GITLAB_ACCESSTOKEN@${GITLAB_URL#https://}sdlf/$REPOSITORY.git"
 
+            GITLAB_NAMESPACE_ID=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --with-decryption --name /SDLF/GitLab/NamespaceId --query "Parameter.Value" --output text)
+            GITLAB_GROUP_NAME=$(aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" ssm get-parameter --name /SDLF/GitLab/SdlfGitLabGroup --query "Parameter.Value" --output text)
+
+            GITLAB_HOST_NAME=gitlab.ssh.covestro.com
+
+            echo "Creating $REPOSITORY repository in GitLab ..."
+            curl --insecure --request POST --header "PRIVATE-TOKEN: $GITLAB_ACCESSTOKEN" \
+                --header "Content-Type: application/json" \
+                --data "{\"name\": \"$REPOSITORY\", \"description\": \"$REPOSITORY\", \"path\": \"$REPOSITORY\", \"namespace_id\": \"$GITLAB_NAMESPACE_ID\", \"initialize_with_readme\": false}" \
+                --url "${GITLAB_URL}api/v4/projects/"
+
+
+            GITLAB_REPOSITORY_URL="https://aws:$GITLAB_ACCESSTOKEN@${GITLAB_URL#https://}${GITLAB_GROUP_NAME}/$REPOSITORY.git"
+            GITLAB_SSH_URI=git@${GITLAB_HOST_NAME}:${GITLAB_GROUP_NAME}/$REPOSITORY.git
+
+            echo "Origin for repo is $GITLAB_SSH_URI"
             if [ "$REPOSITORY" = "sdlf-main" ]
             then
                 mkdir sdlf-main
@@ -278,8 +293,9 @@ devops_account () {
             pushd "$REPOSITORY" || exit
             if [ ! -d .git ] # if .git exists, deploy.sh has likely been run before - do not try to push the base repositories
             then
-                git init
-                git remote add origin "$GITLAB_REPOSITORY_URL" || exit 1
+                git init --initial-branch=main
+                git remote rename origin old-origin
+                git remote add origin "$GITLAB_SSH_URI" || exit 1
                 git add .
                 git commit -m "initial commit"
                 git push origin main || exit 1
@@ -295,7 +311,7 @@ devops_account () {
     done
 
     aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" s3api put-object --bucket "$ARTIFACTS_BUCKET" --key sam-translate.py --body "$DIRNAME"/sdlf-cicd/sam-translate.py
-    curl -L -O --output-dir "$DIRNAME"/sdlf-cicd/ https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip
+    curl -L -O --insecure --output-dir "$DIRNAME"/sdlf-cicd/ https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip
     aws --region "$REGION" --profile "$DEVOPS_AWS_PROFILE" s3api put-object --bucket "$ARTIFACTS_BUCKET" --key aws-sam-cli-linux-x86_64.zip --body "$DIRNAME"/sdlf-cicd/aws-sam-cli-linux-x86_64.zip
     rm "$DIRNAME"/sdlf-cicd/aws-sam-cli-linux-x86_64.zip
 

sdlf-cicd/lambda/domain-cicd/src/lambda_function.py

@@ -2,8 +2,10 @@
 import logging
 import os
 import zipfile
+import ssl
 from io import BytesIO
 from tempfile import mkdtemp
+from urllib.request import HTTPError, Request, URLError, urlopen
 
 import boto3
 from botocore.client import Config
@@ -163,6 +165,34 @@ def delete_domain_team_role_stack(cloudformation, team):
 
 
 def create_team_repository_cicd_stack(domain, team_name, template_body_url, cloudformation_role):
+    gitlab_url = ssm.get_parameter(Name="/SDLF/GitLab/Url", WithDecryption=True)["Parameter"]["Value"]
+    gitlab_accesstoken = ssm.get_parameter(Name="/SDLF/GitLab/AccessToken", WithDecryption=True)["Parameter"]["Value"]
+    repository = f"sdlf-main-{domain}-{team_name}"
+    namespace_id = ssm.get_parameter(Name="/SDLF/GitLab/NamespaceId", WithDecryption=True)["Parameter"]["Value"]
+    url = f"{gitlab_url}api/v4/projects/"
+    headers = {
+        "Content-Type": "application/json",
+        "PRIVATE-TOKEN": gitlab_accesstoken
+    }
+    data = {
+        "name": repository,
+        "description": repository,
+        "path": repository,
+        "namespace_id": namespace_id,
+        "initialize_with_readme": "false"
+    }
+    json_data = json.dumps(data).encode('utf-8')
+    req = Request(url, data=json_data, headers=headers, method='POST')
+    unverified_context = ssl._create_unverified_context()
+    try:
+        with urlopen(req, context=unverified_context) as response:
+            response_body = response.read().decode('utf-8')
+            logger.info(response_body)
+    except HTTPError as e:
+        logger.warn(f"HTTP error occurred: {e.code} {e.reason}. Most likely the repository {repository} already exists")
+    except URLError as e:
+        logger.error(f"URL error occurred: {e.reason}")
+
     response = {}
     cloudformation_waiter_type = None
     stack_name = f"sdlf-cicd-teams-{domain}-{team_name}-repository"
@@ -289,11 +319,21 @@ def create_team_pipeline_cicd_stack(
                     "ParameterValue": f"/SDLF/{git_platform}/StageA{git_platform}",
                     "UsePreviousValue": False,
                 },
+                {
+                    "ParameterKey": "pStageLambdaRepository",
+                    "ParameterValue": f"/SDLF/{git_platform}/StageLambda{git_platform}",
+                    "UsePreviousValue": False,
+                },
                 {
                     "ParameterKey": "pStageBRepository",
                     "ParameterValue": f"/SDLF/{git_platform}/StageB{git_platform}",
                     "UsePreviousValue": False,
                 },
+                {
+                    "ParameterKey": "pStageGlueRepository",
+                    "ParameterValue": f"/SDLF/{git_platform}/StageGlue{git_platform}",
+                    "UsePreviousValue": False,
+                },
                 {
                     "ParameterKey": "pDatasetRepository",
                     "ParameterValue": f"/SDLF/{git_platform}/Dataset{git_platform}",
@@ -361,11 +401,21 @@ def create_team_pipeline_cicd_stack(
                         "ParameterValue": f"/SDLF/{git_platform}/StageA{git_platform}",
                         "UsePreviousValue": False,
                     },
+                    {
+                        "ParameterKey": "pStageLambdaRepository",
+                        "ParameterValue": f"/SDLF/{git_platform}/StageLambda{git_platform}",
+                        "UsePreviousValue": False,
+                    },                    
                     {
                         "ParameterKey": "pStageBRepository",
                         "ParameterValue": f"/SDLF/{git_platform}/StageB{git_platform}",
                         "UsePreviousValue": False,
                     },
+                    {
+                        "ParameterKey": "pStageGlueRepository",
+                        "ParameterValue": f"/SDLF/{git_platform}/StageGlue{git_platform}",
+                        "UsePreviousValue": False,
+                    },                    
                     {
                         "ParameterKey": "pDatasetRepository",
                         "ParameterValue": f"/SDLF/{git_platform}/Dataset{git_platform}",
@@ -668,4 +718,4 @@ def lambda_handler(event, context):
         raise
 
     codepipeline.put_job_success_result(jobId=event["CodePipeline.job"]["id"])
-    return "Success"
+    return "Success"

sdlf-cicd/nested-stacks/template-cicd-cfn-module.yaml

@@ -90,24 +90,17 @@ Resources:
                     - ec2:DeleteNetworkInterface # W11 condition applied
                   Resource:
                     - "*"
-                  Condition:
-                    ArnEqualsIfExists:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
               - !If
                 - RunInVpc
                 - Effect: Allow
                   Action:
                     - ec2:CreateNetworkInterfacePermission
                   Resource:
-                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*"
                   Condition:
                     StringEquals:
                       "ec2:AuthorizedService": codebuild.amazonaws.com
-                    ArnEquals:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
         - PolicyName: sdlf-cicd-build-stages-cfn-modules
           PolicyDocument:
@@ -127,7 +120,7 @@ Resources:
                   - codecommit:GetUploadArchiveStatus
                   - codecommit:CancelUploadArchive
                 Resource:
-                  - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${pStagesRepositoriesPrefix}*
+                  - !Sub arn:${AWS::Partition}:codecommit:${AWS::Region}:*:${pStagesRepositoriesPrefix}*
 
   rBuildCloudformationModuleStage:
     Type: AWS::CodeBuild::Project
@@ -139,8 +132,13 @@ Resources:
       EncryptionKey: !Ref pKMSKey
       VpcConfig: !If
         - RunInVpc
-        - SecurityGroupIds: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-security-groups]
-          Subnets: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets]
+        - SecurityGroupIds:
+            !Split [
+              ",",
+              !ImportValue sdlf-cicd-prerequisites-vpc-security-groups,
+            ]
+          Subnets:
+            !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets]
           VpcId: "{{resolve:ssm:/SDLF/VPC/VpcId}}"
         - !Ref "AWS::NoValue"
       Environment:
@@ -167,9 +165,8 @@ Resources:
                     && unzip -q aws-sam-cli-linux-x86_64.zip -d sam-installation
                     ./sam-installation/install \
                     && sam --version
-                - |-
-                    pip3 install cfn-lint==0.87.7
-                    pip3 install cloudformation-cli
+                - pip3 install cfn-lint==0.87.7
+                - pip3 install cloudformation-cli
                 - aws s3api get-object --bucket "$ARTIFACTS_BUCKET" --key sam-translate.py sam-translate.py
             build:
               commands:
@@ -246,8 +243,13 @@ Resources:
       EncryptionKey: !Ref pKMSKey
       VpcConfig: !If
         - RunInVpc
-        - SecurityGroupIds: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-security-groups]
-          Subnets: !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets]
+        - SecurityGroupIds:
+            !Split [
+              ",",
+              !ImportValue sdlf-cicd-prerequisites-vpc-security-groups,
+            ]
+          Subnets:
+            !Split [",", !ImportValue sdlf-cicd-prerequisites-vpc-subnets]
           VpcId: "{{resolve:ssm:/SDLF/VPC/VpcId}}"
         - !Ref "AWS::NoValue"
       Environment:

sdlf-cicd/nested-stacks/template-cicd-glue-job.yaml

@@ -87,24 +87,17 @@ Resources:
                     - ec2:DeleteNetworkInterface # W11 condition applied
                   Resource:
                     - "*"
-                  Condition:
-                    ArnEqualsIfExists:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
               - !If
                 - RunInVpc
                 - Effect: Allow
                   Action:
                     - ec2:CreateNetworkInterfacePermission
                   Resource:
-                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*"
                   Condition:
                     StringEquals:
                       "ec2:AuthorizedService": codebuild.amazonaws.com
-                    ArnEquals:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
 
   rGlueJobPackage:

sdlf-cicd/nested-stacks/template-cicd-lambda-layer.yaml

@@ -83,24 +83,17 @@ Resources:
                     - ec2:DeleteNetworkInterface # W11 condition applied
                   Resource:
                     - "*"
-                  Condition:
-                    ArnEqualsIfExists:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
               - !If
                 - RunInVpc
                 - Effect: Allow
                   Action:
                     - ec2:CreateNetworkInterfacePermission
                   Resource:
-                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*"
                   Condition:
                     StringEquals:
                       "ec2:AuthorizedService": codebuild.amazonaws.com
-                    ArnEquals:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
 
   rBuildLambdaLayersPackage:

sdlf-cicd/template-cicd-domain-roles.yaml

@@ -427,12 +427,6 @@ Resources:
                   - lambda:CreateFunction
                   - lambda:UpdateFunctionConfiguration
                 Resource: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:sdlf-*
-                Condition: !If
-                  - RunInVpc
-                  - StringEquals:
-                      "lambda:VpcIds":
-                        - "{{resolve:ssm:/SDLF/VPC/VpcId}}"
-                  - !Ref "AWS::NoValue"
               - Effect: Allow
                 Action:
                   - lambda:AddPermission

sdlf-cicd/template-cicd-domain-team-role.yaml

@@ -293,12 +293,6 @@ Resources:
               - lambda:CreateFunction
               - lambda:UpdateFunctionConfiguration
             Resource: !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:sdlf-${pTeamName}-*
-            Condition: !If
-              - RunInVpc
-              - StringEquals:
-                  "lambda:VpcIds":
-                    - "{{resolve:ssm:/SDLF/VPC/VpcId}}"
-              - !Ref "AWS::NoValue"
           - Effect: Allow
             Action:
               - lambda:AddPermission

sdlf-cicd/template-cicd-sdlf-pipelines.yaml

@@ -340,6 +340,7 @@ Resources:
                   - ssm:GetParametersByPath
                 Resource:
                   - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/Misc/*
+                  - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SDLF/GitLab/*
               - Effect: Allow
                 Action:
                   - s3:PutObject
@@ -663,24 +664,17 @@ Resources:
                     - ec2:DeleteNetworkInterface # W11 condition applied
                   Resource:
                     - "*"
-                  Condition:
-                    ArnEqualsIfExists:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
               - !If
                 - RunInVpc
                 - Effect: Allow
                   Action:
                     - ec2:CreateNetworkInterfacePermission
                   Resource:
-                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*"
                   Condition:
                     StringEquals:
                       "ec2:AuthorizedService": codebuild.amazonaws.com
-                    ArnEquals:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
 
   rCloudFormationPackageCodeBuildProject:
@@ -791,24 +785,17 @@ Resources:
                     - ec2:DeleteNetworkInterface # W11 condition applied
                   Resource:
                     - "*"
-                  Condition:
-                    ArnEqualsIfExists:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"
               - !If
                 - RunInVpc
                 - Effect: Allow
                   Action:
                     - ec2:CreateNetworkInterfacePermission
                   Resource:
-                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                    - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*"
                   Condition:
                     StringEquals:
                       "ec2:AuthorizedService": codebuild.amazonaws.com
-                    ArnEquals:
-                      "ec2:Vpc":
-                        - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:vpc/{{resolve:ssm:/SDLF/VPC/VpcId}}"
                 - !Ref "AWS::NoValue"