Get production service to work

Author: BenGalewsky

automate/deploy_mdf_flow.py

@@ -67,10 +67,10 @@
     ]))
 
 mdf_flow.save_flow(flow_info_file)
-print("scope = ", mdf_flow.get_scope_for_runAs_role('SubmittingUser')['scopes'][0]['id'])
+print("scope = ", mdf_flow.get_scope_id_for_runAs_role('SubmittingUser')['scopes'][0]['id'])
 
 print("MDF Flow deployed", mdf_flow)
-submitting_user_scope_id = mdf_flow.get_scope_for_runAs_role('SubmittingUser')['scopes'][0]['id']
+submitting_user_scope_id = mdf_flow.get_scope_id_for_runAs_role('SubmittingUser')['scopes'][0]['id']
 
 connect_scope_def = {
     "scope": {

aws/globus_automate_flow.py

@@ -157,6 +157,8 @@ def read_flow(self, path):
             self.flow_id = flow_info['flow_id']
             self.flow_scope = flow_info['flow_scope']
 
-    def get_scope_for_runAs_role(self, rolename):
-        print("--->RunAsScopes ", self.runAsScopes[rolename])
+    def get_scope_id_for_runAs_role(self, rolename):
         return self.globus_auth.scope_id_from_uri(self.runAsScopes[rolename])
+
+    def get_scope_uri_for_runAs_role(self, rolename):
+        return self.runAsScopes[rolename]

infra/mdf/modules/lambdas/main.tf

@@ -1,6 +1,17 @@
 
+#### Create lambda functions for MDF Connect
+#### These functions are deployed as container images
+#### The container images are built and pushed to ECR by the GitHub Action
+#### We explicitly create the log groups so we can tag them and set the retention period
+locals {
+  auth_function_name = "${var.namespace}-auth-${var.env}"
+  submit_function_name = "${var.namespace}-submit-${var.env}"
+  status_function_name = "${var.namespace}-status-${var.env}"
+  submissions_function_name = "${var.namespace}-submissions-${var.env}"
+}
+
 resource "aws_lambda_function" "mdf-connect-auth" {
-  function_name = "${var.namespace}-auth-${var.env}"
+  function_name = local.auth_function_name
   description   = "GlobusAuth Authorizer for MDF Connect"
   image_uri     = "${var.ecr_repos["auth"]}:${var.env}"
   package_type  = "Image"
@@ -10,11 +21,19 @@ resource "aws_lambda_function" "mdf-connect-auth" {
   environment {
       variables = var.env_vars
   }
+  depends_on = [aws_cloudwatch_log_group.auth_log_group]
+  tags = var.resource_tags
 }
 
 
+resource "aws_cloudwatch_log_group" "auth_log_group" {
+  name              = "/aws/lambda/${local.auth_function_name}"
+  retention_in_days = 5
+  tags = var.resource_tags
+}
+
 resource "aws_lambda_function" "mdf-connect-submit" {
-  function_name = "${var.namespace}-submit-${var.env}"
+  function_name = local.submit_function_name
   description   = "Submit Datasets via MDF Connect"
   image_uri     = "${var.ecr_repos["submit"]}:${var.env}"
   package_type  = "Image"
@@ -24,10 +43,18 @@ resource "aws_lambda_function" "mdf-connect-submit" {
   environment {
       variables = var.env_vars
   }
+  depends_on = [aws_cloudwatch_log_group.submit_log_group]
+  tags = var.resource_tags
+}
+
+resource "aws_cloudwatch_log_group" "submit_log_group" {
+  name              = "/aws/lambda/${local.submit_function_name}"
+  retention_in_days = 5
+  tags = var.resource_tags
 }
 
 resource "aws_lambda_function" "mdf-connect-status" {
-  function_name = "${var.namespace}-status-${var.env}"
+  function_name = local.status_function_name
   description   = "Retrieve submit status via MDF Connect"
   image_uri     = "${var.ecr_repos["status"]}:${var.env}"
   package_type  = "Image"
@@ -37,11 +64,18 @@ resource "aws_lambda_function" "mdf-connect-status" {
   environment {
       variables = var.env_vars
   }
+  depends_on = [aws_cloudwatch_log_group.status_log_group]
+  tags = var.resource_tags
 }
 
+resource "aws_cloudwatch_log_group" "status_log_group" {
+  name              = "/aws/lambda/${local.status_function_name}"
+  retention_in_days = 5
+  tags = var.resource_tags
+}
 
 resource "aws_lambda_function" "mdf-connect-submissions" {
-  function_name = "${var.namespace}-submissions-${var.env}"
+  function_name = local.submissions_function_name
   description   = "Retrieve history of submissions via MDF Connect"
 
   image_uri     = "${var.ecr_repos["submissions"]}:${var.env}"
@@ -53,5 +87,12 @@ resource "aws_lambda_function" "mdf-connect-submissions" {
   environment {
       variables = var.env_vars
   }
+  depends_on = [aws_cloudwatch_log_group.submissions_log_group]
+  tags = var.resource_tags
 }
 
+resource "aws_cloudwatch_log_group" "submissions_log_group" {
+  name              = "/aws/lambda/${local.submissions_function_name}"
+  retention_in_days = 5
+  tags = var.resource_tags
+}

infra/mdf/modules/lambdas/variables.tf

@@ -21,4 +21,9 @@ variable "env_vars" {
 variable "ecr_repos" {
   description = "Map of lambda function to ECR repo holding the images."
   type = map(string)
-}
+}
+
+variable "resource_tags" {
+  description = "Tags to apply to all resources."
+  type = map(string)
+}

infra/mdf/modules/permissions/main.tf

@@ -16,9 +16,23 @@ resource "aws_iam_role" "lambda_execution_role" {
 }
 
 
-resource "aws_iam_role_policy_attachment" "AWSLambdaBasicExecutionRole" {
+# role
+data "aws_iam_policy_document" "lambda_logging_role_policy" {
+  statement {
+    actions = [
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+    resources = [
+      "arn:aws:logs:*:*:*"
+    ]
+  }
+}
+
+
+resource "aws_iam_role_policy" "lambda_logging_role" {
   role = aws_iam_role.lambda_execution_role.id
-  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+  policy = data.aws_iam_policy_document.lambda_logging_role_policy.json
 }
 
 resource "aws_iam_policy" "allow_mdf_secrets_access_policy" {

infra/mdf/prod/main.tf

@@ -36,6 +36,7 @@ module "lambdas" {
   namespace                 = var.namespace
   lambda_execution_role_arn = module.permissions.submit_lambda_invoke_arn
   ecr_repos                 = var.ecr_repos
+  resource_tags             = var.resource_tags
 }
 
 module "dynamodb" {

infra/mdf/prod/variables.tf

@@ -30,7 +30,7 @@ variable "env_vars" {
         MANAGE_FLOWS_SCOPE="https://auth.globus.org/scopes/eec9b274-0c81-4334-bdc2-54e90e689b9a/manage_flows"
         MONITOR_BY_GROUP="urn:globus:groups:id:5fc63928-3752-11e8-9c6f-0e00fd09bf20"
         PORTAL_URL="https://acdc.alcf.anl.gov/mdf/detail/"
-        RUN_AS_SCOPE="aa5f9e85-5305-4f9e-9679-95c158e5aa47"
+        RUN_AS_SCOPE="c096e223-59e8-42fe-85ea-956208b0f878"
         SEARCH_INDEX_UUID="1a57bbe5-5272-477f-9d31-343b8258b7a5"
         TEST_DATA_DESTINATION="globus://f10a69a9-338c-4e5b-baa1-0dc92359ab47/mdf_testing/"
         TEST_SEARCH_INDEX_UUID="5acded0c-a534-45af-84be-dcf042e36412"
@@ -47,4 +47,13 @@ variable "ecr_repos" {
     "status" = "557062710055.dkr.ecr.us-east-1.amazonaws.com/mdf-lambdas/status"
     "auth" = "557062710055.dkr.ecr.us-east-1.amazonaws.com/mdf-lambdas/auth"
   }
+}
+
+variable "resource_tags" {
+    type = map(string)
+    default = {
+        "Owner" = "MDF"
+        "Environment" = "Production"
+        "Project" = "MDF Connect"
+    }
 }

scripts/get_mdf_token.py

@@ -9,5 +9,7 @@
     print("Usage: python get_mdf_token.py <MDF Application URL>")
     sys.exit(1)
 
+mdfcc = MDFConnectClient(service_instance=sys.argv[1])
+mdfcc.logout()
 mdfcc = MDFConnectClient(service_instance=sys.argv[1])
 print(mdfcc._MDFConnectClient__authorizer.access_token)

scripts/get_submissions.py

@@ -0,0 +1,4 @@
+from mdf_connect_client import MDFConnectClient
+mdfcc = MDFConnectClient(service_instance="dev")
+
+print(mdfcc.check_all_submissions(newer_than_date=(2023, 10, 1)))

scripts/get_token.py

@@ -0,0 +1,31 @@
+import json
+import os
+import sys
+
+import globus_sdk
+
+if "API_CLIENT_ID" in os.environ:
+    globus_secrets = {
+        "API_CLIENT_ID": os.environ["API_CLIENT_ID"],
+        "API_CLIENT_SECRET": os.environ["API_CLIENT_SECRET"],
+        "smtp_hostname": os.environ["SMTP_HOSTNAME"],
+        "smtp_user": os.environ["SMTP_USER"],
+        "smtp_pass": os.environ["SMTP_PASS"]
+    }
+else:
+    if len(sys.argv) == 2:
+        secret_file = f"../automate/.mdfsecrets.{sys.argv[1]}"
+    else:
+        secret_file = "../automate/.mdfsecretsCCC"
+
+    with open(secret_file, 'r') as f:
+        globus_secrets = json.load(f)
+
+client = globus_sdk.ConfidentialAppAuthClient(globus_secrets['API_CLIENT_ID'], globus_secrets['API_CLIENT_SECRET'])
+scopes = "urn:globus:auth:scope:groups.api.globus.org:all openid email profile " "urn:globus:auth:scope:transfer.api.globus.org:all"
+token_response = client.oauth2_client_credentials_tokens(requested_scopes=scopes)
+print(token_response)
+tokens = token_response.by_resource_server["groups.api.globus.org"]
+GROUP_TOKEN = tokens["access_token"]
+print("Token ",GROUP_TOKEN)
+print(tokens)

scripts/globus_auth_busybox.py

@@ -0,0 +1,16 @@
+import globus_sdk
+CLIENT_ID = 'c296acd8-f454-4ad7-b582-51a48685e775'
+CLIENT_SECRET = 'IgkdmTz8cE+7hS/uo8JPmXE7xtsIJ1xg+2AC2AlHF5g='
+client = globus_sdk.ConfidentialAppAuthClient(CLIENT_ID, CLIENT_SECRET)
+scopes = "urn:globus:auth:scope:groups.api.globus.org:all openid email profile " "urn:globus:auth:scope:transfer.api.globus.org:all"
+token_response = client.oauth2_client_credentials_tokens(requested_scopes=scopes)
+tokens = token_response.by_resource_server["groups.api.globus.org"]
+GROUP_TOKEN = tokens["access_token"]
+print("Token ",GROUP_TOKEN)
+a = globus_sdk.AccessTokenAuthorizer(GROUP_TOKEN)
+groups_client = globus_sdk.GroupsClient(authorizer=a)
+batch = globus_sdk.BatchMembershipActions()
+batch.join("c296acd8-f454-4ad7-b582-51a48685e775")
+groups_client.batch_membership_action("cc192dca-3751-11e8-90c1-0a7c735d220a", batch)
+print(groups_client.get_my_groups())
+

scripts/join_mdf_group.py

@@ -0,0 +1 @@
+../../data-schemas

scripts/schemas

@@ -0,0 +1,24 @@
+import json
+import re
+import sys
+from decimal import Decimal
+
+import boto3
+from boto3.dynamodb.conditions import Key
+from aws.dynamo_manager import DynamoManager
+
+dynamo_manager = DynamoManager()
+v = dynamo_manager.get_current_version('_test_v1')
+print(v)
+sys.exit(0)
+
+response = dest_table.query(
+        KeyConditionExpression=
+            Key('source_name').eq("_test_v1"),
+            ScanIndexForward=False
+    )
+
+versions = {str(x['version']): x for x in response['Items']}
+print(sorted(versions.keys(), key=lambda x:[int(i) if i.isdigit() else i for i in x.split('.')]))
+
+

scripts/status_versions.py

@@ -0,0 +1,66 @@
+import json
+import re
+from decimal import Decimal
+from time import sleep
+
+import boto3
+from boto3.dynamodb.conditions import Key
+from botocore.exceptions import ClientError
+
+RETRY_EXCEPTIONS = ('ProvisionedThroughputExceededException',
+                    'ThrottlingException')
+
+dynamodb = boto3.resource('dynamodb')
+table = dynamodb.Table('prod-status-alpha-1')
+dest_table = dynamodb.Table('dev-status-0.4')
+print(table)
+scan_kwargs = {}
+
+done = False
+start_key = None
+i = 0
+version_re = re.compile("(.+)_(v[0-9].*$)")
+while not done:
+    if i > 10:
+        break
+    i = i + 1
+    if start_key:
+        scan_kwargs['ExclusiveStartKey'] = start_key
+    response = table.scan(**scan_kwargs)
+    items = response.get('Items', [])
+    for item in items:
+        match = version_re.match(item['source_id'])
+        if not match:
+            print("--------->", item['source_id'])
+        else:
+            source_name = match.group(1)
+            version = match.group(2).replace("-", ".")
+            if "." not in version:
+                version = version + ".0"
+
+            # Remove the leading v
+            version = version[1:]
+            print(item['source_id'], f"[{version}] ({source_name})")
+            new_rec = item.copy()
+            new_rec['version'] = version
+            original_submission = json.loads(item['original_submission'])
+            new_rec['source_id'] = original_submission.get('source_name', source_name)
+
+            retries = 0
+            success = False
+            while not success:
+                try:
+                    dest_table.put_item(Item=new_rec)
+                    retries = 0
+                    success = True
+                except ClientError as err:
+                    if err.response['Error']['Code'] not in RETRY_EXCEPTIONS:
+                        raise
+                    print('WHOA, too fast, slow it down retries={}'.format(retries))
+                    sleep(2 ** retries)
+                    retries += 1  # TODO max limit
+
+    start_key = response.get('LastEvaluatedKey', None)
+    done = start_key is None
+
+