Merge pull request #773 from populationgenomics/dev

Release

README.md

@@ -36,7 +36,7 @@ It comprises three key components:
 
 As of Jan 15, 2024 this schema should reflect the data structure on the tables:
 
-![Database Structure](resources/schemav7.7.png.png)
+![Database Structure](resources/schemav7.7.png)
 
 You can also find this at [DbDiagram](https://dbdiagram.io/d/Metamist-Schema-v7-7-6600c875ae072629ced6a1fc).
 

db/python/layers/participant.py

@@ -765,13 +765,14 @@ async def get_seqr_individual_template(
             set_headers.update(set(row.keys()))
 
         rows = [{h: r.get(h) for h in set_headers if h in r} for r in rows]
+        headers = []  # get ordered headers if we have data for it
+        for h in SeqrMetadataKeys.get_ordered_headers():
+            if header := lheader_to_json.get(h.value.lower()) in set_headers:
+                headers.append(header)
 
         return {
             'rows': rows,
-            # get ordered headers if we have data for it
-            'headers': [
-                h for h in SeqrMetadataKeys.get_ordered_headers() if h in set_headers
-            ],
+            'headers': headers,
             'header_map': json_header_map,
         }
 

db/python/layers/seqr.py

@@ -11,6 +11,7 @@
 import aiohttp
 import slack_sdk
 import slack_sdk.errors
+from backoff import expo, on_exception
 from cloudpathlib import AnyPath
 
 from cpg_utils.cloud import get_google_identity_token
@@ -498,6 +499,7 @@ async def update_es_index(
         messages.extend(await asyncio.gather(*requests))
         return messages
 
+    @on_exception(expo, aiohttp.ClientResponseError, max_tries=3)
     async def update_saved_variants(
         self,
         session: aiohttp.ClientSession,

etl/load/main.py

@@ -129,7 +129,6 @@ def process_rows(
     else:
         status = ParsingStatus.FAILED
         parsing_result = f'Error: {err_msg} when parsing record with id: {request_id}'
-        # TODO: should we log to error log table as well?
 
     if delivery_attempt == 1:
         # log only at the first attempt,
@@ -167,6 +166,10 @@ def process_rows(
             # publish to notification pubsub
             msg_title = 'Metamist ETL Load Failed'
             try:
+                # limit result to max 100 characters, to avoid spamming slack
+                # sometimes the details can be huge, the whole stacktrace
+                log_details['result'] = str(parsing_result)[:100]
+                log_record['details'] = json.dumps(log_details)
                 pubsub_client = _get_pubsub_client()
                 pubsub_client.publish(
                     NOTIFICATION_PUBSUB_TOPIC,

etl/notification/main.py

@@ -51,8 +51,16 @@ def etl_notify(request: flask.Request):
             'message': f'Missing or empty message: {jbody_str}',
         }, 400
 
-    # TODO: format message to slack message
-    message_blocks = format_slack(message)
+    # format message to slack message
+    try:
+        message_blocks = format_slack(message)
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logging.error(f'Failed to format message: {e}, message: {message}')
+        return {
+            'success': False,
+            'message': f'Failed to format message: {e}',
+        }, 500
+
     success = None
     try:
         client = WebClient(token=SLACK_BOT_TOKEN)

metamist_infrastructure/README.md

@@ -5,7 +5,10 @@ We want to package it up in a way that cpg-infrastructure deployment can consume
 using [entry points](https://amir.rachum.com/python-entry-points/). Hence, this python package
 contains all files required to deploy:
 
-- Metamist ETL framework (cloud function, bigquery, pubsub), with currently no consumer
+- Metamist ETL framework (cloud function, bigquery, pubsub), with currently no consumer:
+
+![Database Structure](../resources/metamist-etl.drawio.png)
+
 
 ## Installation
 

requirements.txt

@@ -1,5 +1,6 @@
 # requirements are for running the server
 # fix boto3/core version to prevent pip downloading all the versions caused by cloudpathlib
+backoff>=2.2.1
 boto3==1.28.56
 botocore==1.31.56
 cpg-utils>=5.0.5