setup final workflow step; separated log streams for task

.env.task.conf

@@ -3,3 +3,5 @@ WORKING_DIRECTORY=/scratch-path
 
 FQDN=localhost
 PYTHONPATH=/home/bison/bison
+
+timestamp=$(date +"%Y-%m-%d_%H-%M-%S")

aws/lambda/bison_s8_calc_stats.py

@@ -0,0 +1,130 @@
+"""Lambda to start an EC2 instance to create/analyze a PAM from specimen records."""
+import boto3
+from botocore.exceptions import ClientError, NoCredentialsError
+import botocore.session as bc
+from botocore.client import Config
+from datetime import datetime
+
+print("*** Loading function bison_s8_calc_stats")
+PROJECT = "bison"
+
+# .............................................................................
+# Dataload filename postfixes
+# .............................................................................
+dt = datetime.now()
+yr = dt.year
+mo = dt.month
+bison_datestr = f"{yr}_{mo:02d}_01"
+gbif_datestr = f"{yr}-{mo:02d}-01"
+
+# .............................................................................
+# AWS constants
+# .............................................................................
+REGION = "us-east-1"
+AWS_ACCOUNT = "321942852011"
+AWS_METADATA_URL = "http://169.254.169.254/latest/"
+WORKFLOW_ROLE_NAME = f"{PROJECT}_redshift_lambda_role"
+WORKFLOW_ROLE_ARN = f"arn:aws:iam::{PROJECT}:role/service-role/{WORKFLOW_ROLE_NAME}"
+WORKFLOW_USER = f"project.{PROJECT}"
+
+# S3 locations
+S3_BUCKET = f"{PROJECT}-{AWS_ACCOUNT}-{REGION}"
+S3_IN_DIR = "input"
+S3_OUT_DIR = "output"
+S3_LOG_DIR = "log"
+S3_SUMMARY_DIR = "summary"
+
+# EC2 launch template/version
+EC2_SPOT_TEMPLATE = "bison_spot_task_template"
+TASK = "calc_stats"
+
+# .............................................................................
+# Initialize Botocore session
+# .............................................................................
+timeout = 300
+
+session = boto3.session.Session()
+bc_session = bc.get_session()
+session = boto3.Session(botocore_session=bc_session, region_name=REGION)
+# Initialize Redshift client
+config = Config(connect_timeout=timeout, read_timeout=timeout)
+ec2_client = session.client("ec2", config=config)
+
+
+# --------------------------------------------------------------------------------------
+def lambda_handler(event, context):
+    """Start an EC2 instance to resolve RIIS records, then write the results to S3.
+
+    Args:
+        event: AWS event triggering this function.
+        context: AWS context of the event.
+
+    Returns:
+        instance_id (number): ID of the EC2 instance started.
+
+    Raises:
+        Exception: on requested template does not exist.
+        NoCredentialsError: on failure to get credentials to run EC2 task.
+        ClientError: on failure to run EC2 task.
+        Exception: on no instances started.
+        Exception: on unknown error.
+    """
+    # -------------------------------------
+    # Start instance to run task
+    ver_num = None
+    print("*** ---------------------------------------")
+    print("*** Find template version")
+    response = ec2_client.describe_launch_template_versions(
+        LaunchTemplateName=EC2_SPOT_TEMPLATE
+    )
+    versions = response["LaunchTemplateVersions"]
+    for ver in versions:
+        if ver["VersionDescription"] == TASK:
+            ver_num = ver["VersionNumber"]
+            break
+    if ver_num is None:
+        raise Exception(
+            f"!!! Template {EC2_SPOT_TEMPLATE} version {TASK} does not exist")
+    print(f"*** Found template {EC2_SPOT_TEMPLATE} version {ver_num} for {TASK}.")
+
+    print("*** ---------------------------------------")
+    print("*** Launch EC2 instance with task template version")
+    instance_name = f"bison_{TASK}"
+
+    try:
+        response = ec2_client.run_instances(
+            MinCount=1, MaxCount=1,
+            LaunchTemplate={
+                "LaunchTemplateName": EC2_SPOT_TEMPLATE, "Version": f"{ver_num}"
+            },
+            TagSpecifications=[
+                {
+                    "ResourceType": "instance",
+                    "Tags": [
+                        {"Key": "Name", "Value": instance_name},
+                        {"Key": "TemplateName", "Value": EC2_SPOT_TEMPLATE}
+                    ]
+                }
+            ]
+        )
+    except NoCredentialsError:
+        print("!!! Failed to authenticate for run_instances")
+        raise
+    except ClientError:
+        print(
+            f"!!! Failed to run instance for template {EC2_SPOT_TEMPLATE}, "
+            f"version {ver_num}/{TASK}")
+        raise
+
+    try:
+        instance = response["Instances"][0]
+    except KeyError:
+        raise Exception(f"!!! No instances returned in {response}")
+
+    instance_id = instance["InstanceId"]
+    print(f"*** Started instance {instance_id}. ")
+
+    return {
+        "statusCode": 200,
+        "body": f"Executed bison_s1_annotate_riis lambda starting EC2 {instance_id}"
+    }

aws/userdata/calc_stats.userdata.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+sudo apt-get -y update
+sudo apt-get -y install docker.io
+sudo apt-get -y install docker-compose-v2
+git clone https://github.com/lifemapper/bison.git
+cd bison
+sudo docker compose -f compose.calc_stats.yml up
+sudo shutdown -h now

compose.calc_stats.yml

@@ -0,0 +1,25 @@
+version: "3.9"
+services:
+
+  bison_task:
+    build:
+      context: ./.
+      target: task
+    environment:
+      - TASK_APP=bison.task.calc_stats
+    env_file:
+      ./.env.task.conf
+    logging:
+      driver: awslogs
+      options:
+        awslogs-region: us-east-1
+        awslogs-group: bison_task_calc_stats
+        awslogs-stream: $timestamp
+    volumes:
+      # bind mount from host machine
+      - "/home/ubuntu/aws_data:/volumes/bison_data"
+      # volumes
+      - "scratch-path:/scratch-path"
+
+volumes:
+  scratch-path: