cleanup

Dockerfile

@@ -1,6 +1,7 @@
 # syntax=docker/dockerfile:1
 # ........................................................
 # Backend base image
+# ........................................................
 #FROM python:3.12.3-alpine3.20 as base
 FROM python:3.12-slim-bookworm as base
 
@@ -40,6 +41,7 @@ COPY --chown=bison:bison ./data ./data
 
 # ........................................................
 # Task image from base
+# ........................................................
 FROM base as task
 # Install dev dependencies for debugging
 RUN venv/bin/pip install debugpy
@@ -56,6 +58,7 @@ CMD venv/bin/python -m ${TASK_APP}
 
 # ........................................................
 # Development flask image from base
+# ........................................................
 FROM base as dev-flask
 # Install dev dependencies for debugging
 RUN venv/bin/pip install debugpy
@@ -71,6 +74,7 @@ CMD venv/bin/python -m debugpy --listen 0.0.0.0:${DEBUG_PORT} -m ${FLASK_MANAGE}
 
 # ........................................................
 # Production flask image from base
+# ........................................................
 FROM base as flask
 
 COPY --chown=bison:bison ./flask_app ./flask_app
@@ -79,6 +83,7 @@ CMD venv/bin/python -m gunicorn -w 4 --bind 0.0.0.0:5000 ${FLASK_APP}
 
 # ........................................................
 # Frontend base image (for development)
+# ........................................................
 FROM node:16.10.0-buster as base-front-end
 
 LABEL maintainer="Specify Collections Consortium <github.com/specify>"
@@ -97,6 +102,7 @@ COPY --chown=node:node bison/frontend/js_src .
 
 # ........................................................
 # Frontend image (for production) from base-front-end
+# ........................................................
 FROM base-front-end as front-end
 
 RUN npm run build

aws/lambda/bison_s8_calc_stats.py

@@ -27,16 +27,10 @@
 WORKFLOW_ROLE_ARN = f"arn:aws:iam::{PROJECT}:role/service-role/{WORKFLOW_ROLE_NAME}"
 WORKFLOW_USER = f"project.{PROJECT}"
 
-# S3 locations
-S3_BUCKET = f"{PROJECT}-{AWS_ACCOUNT}-{REGION}"
-S3_IN_DIR = "input"
-S3_OUT_DIR = "output"
-S3_LOG_DIR = "log"
-S3_SUMMARY_DIR = "summary"
-
 # EC2 launch template/version
-EC2_SPOT_TEMPLATE = "bison_spot_task_template"
+EC2_SPOT_TEMPLATE = f"{PROJECT}_spot_task_template"
 TASK = "calc_stats"
+EC2_INSTANCE_NAME = f"{PROJECT}_{TASK}"
 
 # .............................................................................
 # Initialize Botocore session
@@ -46,7 +40,7 @@
 session = boto3.session.Session()
 bc_session = bc.get_session()
 session = boto3.Session(botocore_session=bc_session, region_name=REGION)
-# Initialize Redshift client
+# Initialize EC2 client
 config = Config(connect_timeout=timeout, read_timeout=timeout)
 ec2_client = session.client("ec2", config=config)
 
@@ -89,7 +83,6 @@ def lambda_handler(event, context):
 
     print("*** ---------------------------------------")
     print("*** Launch EC2 instance with task template version")
-    instance_name = f"bison_{TASK}"
 
     try:
         response = ec2_client.run_instances(
@@ -101,7 +94,7 @@ def lambda_handler(event, context):
                 {
                     "ResourceType": "instance",
                     "Tags": [
-                        {"Key": "Name", "Value": instance_name},
+                        {"Key": "Name", "Value": EC2_INSTANCE_NAME},
                         {"Key": "TemplateName", "Value": EC2_SPOT_TEMPLATE}
                     ]
                 }

aws/redshift/queries.sql

@@ -21,6 +21,7 @@ SELECT current_user;
 GRANT CREATE
     ON DATABASE dev
     TO IAMR:bison_redshift_lambda_role
+
 SELECT has_database_privilege('IAMR:bison_redshift_lambda_role', 'dev', 'create');
 
 GRANT ALL

bison/task/calc_stats.py

@@ -133,15 +133,16 @@ def download_dataframe(s3, table_type, datestr, bucket, bucket_dir):
 # Main
 # --------------------------------------------------------------------------------------
 if __name__ == "__main__":
-    """Main script creates a SPECIES_DATASET_MATRIX from county/species list."""
+    """Main script creates county_x_species_matrix from a county/species list."""
     overwrite = True
     datestr = get_current_datadate_str()
     s3 = S3(region=REGION)
     logger = None
 
     dim_region = ANALYSIS_DIM.COUNTY["code"]
     dim_species = ANALYSIS_DIM.species_code()
-    stacked_data_table_type = SUMMARY.get_table_type("list", dim_region, dim_species)
+    stacked_data_table_type = SUMMARY.get_table_type(
+        "list", dim_region, dim_species)
     # Species are always columns (for PAM)
     mtx_table_type = SUMMARY.get_table_type("matrix", dim_region, dim_species)