Release 6.9.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 6.8.0
+current_version = 6.9.0
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>[A-z0-9-]+)

api/graphql/schema.py

@@ -843,6 +843,24 @@ async def my_projects(self, info: Info) -> list[GraphQLProject]:
         )
         return [GraphQLProject.from_internal(p) for p in projects]
 
+    @strawberry.field
+    async def analysis_runner(
+        self,
+        info: Info,
+        ar_guid: str,
+    ) -> GraphQLAnalysisRunner:
+        if not ar_guid:
+            raise ValueError('Must provide ar_guid')
+        connection = info.context['connection']
+        alayer = AnalysisRunnerLayer(connection)
+        filter_ = AnalysisRunnerFilter(ar_guid=GenericFilter(eq=ar_guid))
+        analysis_runners = await alayer.query(filter_)
+        if len(analysis_runners) != 1:
+            raise ValueError(
+                f'Expected exactly one analysis runner expected, found {len(analysis_runners)}'
+            )
+        return GraphQLAnalysisRunner.from_internal(analysis_runners[0])
+
 
 schema = strawberry.Schema(
     query=Query, mutation=None, extensions=[QueryDepthLimiter(max_depth=10)]

api/routes/analysis_runner.py

@@ -25,13 +25,13 @@ async def create_analysis_runner_log(  # pylint: disable=too-many-arguments
     description: str,
     driver_image: str,
     config_path: str,
-    cwd: str | None,
     environment: str,
-    hail_version: str | None,
     batch_url: str,
     submitting_user: str,
     meta: dict[str, str],
     output_path: str,
+    hail_version: str | None = None,
+    cwd: str | None = None,
     connection: Connection = get_project_write_connection,
 ) -> str:
     """Create a new analysis runner log"""

api/routes/billing.py

@@ -12,7 +12,7 @@
 from db.python.layers.billing import BillingLayer
 from models.enums import BillingSource
 from models.models import (
-    BillingBatchCostRecord,
+    AnalysisCostRecord,
     BillingColumn,
     BillingCostBudgetRecord,
     BillingTotalCostQueryModel,
@@ -276,7 +276,7 @@ async def get_namespaces(
 
 @router.get(
     '/cost-by-ar-guid/{ar_guid}',
-    response_model=list[BillingBatchCostRecord],
+    response_model=list[AnalysisCostRecord],
     operation_id='costByArGuid',
 )
 @alru_cache(maxsize=10, ttl=BILLING_CACHE_RESPONSE_TTL)
@@ -294,7 +294,7 @@ async def get_cost_by_ar_guid(
 
 @router.get(
     '/cost-by-batch-id/{batch_id}',
-    response_model=list[BillingBatchCostRecord],
+    response_model=list[AnalysisCostRecord],
     operation_id='costByBatchId',
 )
 @alru_cache(maxsize=10, ttl=BILLING_CACHE_RESPONSE_TTL)

api/server.py

@@ -19,7 +19,7 @@
 from db.python.utils import get_logger
 
 # This tag is automatically updated by bump2version
-_VERSION = '6.8.0'
+_VERSION = '6.9.0'
 
 logger = get_logger()
 

api/utils/db.py

@@ -1,3 +1,4 @@
+import json
 import logging
 from os import getenv
 
@@ -29,6 +30,19 @@ def get_ar_guid(request: Request) -> str | None:
     return request.headers.get('sm-ar-guid')
 
 
+def get_extra_audit_log_values(request: Request) -> dict | None:
+    """Get a JSON encoded dictionary from the 'sm-extra-values' header if it exists"""
+    headers = request.headers.get('sm-extra-values')
+    if not headers:
+        return None
+
+    try:
+        return json.loads(headers)
+    except json.JSONDecodeError:
+        logging.error(f'Could not parse sm-extra-values: {headers}')
+        return None
+
+
 def get_on_behalf_of(request: Request) -> str | None:
     """
     Get sm-on-behalf-of if there are requests that were performed on behalf of
@@ -69,12 +83,16 @@ async def dependable_get_write_project_connection(
     request: Request,
     author: str = Depends(authenticate),
     ar_guid: str = Depends(get_ar_guid),
+    extra_values: dict | None = Depends(get_extra_audit_log_values),
     on_behalf_of: str | None = Depends(get_on_behalf_of),
 ) -> Connection:
     """FastAPI handler for getting connection WITH project"""
     meta = {'path': request.url.path}
     if request.client:
         meta['ip'] = request.client.host
+    if extra_values:
+        meta.update(extra_values)
+
     return await ProjectPermissionsTable.get_project_connection(
         project_name=project,
         author=author,
@@ -89,27 +107,37 @@ async def dependable_get_readonly_project_connection(
     project: str,
     author: str = Depends(authenticate),
     ar_guid: str = Depends(get_ar_guid),
+    extra_values: dict | None = Depends(get_extra_audit_log_values),
 ) -> Connection:
     """FastAPI handler for getting connection WITH project"""
+    meta = {}
+    if extra_values:
+        meta.update(extra_values)
+
     return await ProjectPermissionsTable.get_project_connection(
         project_name=project,
         author=author,
         readonly=True,
         on_behalf_of=None,
         ar_guid=ar_guid,
+        meta=meta,
     )
 
 
 async def dependable_get_connection(
     request: Request,
     author: str = Depends(authenticate),
     ar_guid: str = Depends(get_ar_guid),
+    extra_values: dict | None = Depends(get_extra_audit_log_values),
 ):
     """FastAPI handler for getting connection withOUT project"""
     meta = {'path': request.url.path}
     if request.client:
         meta['ip'] = request.client.host
 
+    if extra_values:
+        meta.update(extra_values)
+
     return await SMConnections.get_connection_no_project(
         author, ar_guid=ar_guid, meta=meta
     )

db/project.xml

@@ -1178,22 +1178,32 @@
 		<sql> ALTER TABLE analysis_runner ADD SYSTEM VERSIONING; </sql>
 	</changeSet>
 	<changeSet id="2024-03-15_analysis-runner-table-migration" author="michael.franklin">
+
+	<!-- make the configPath nullable -->
+
+	<sql>
+	SET @@system_versioning_alter_history = 1;
+	ALTER TABLE analysis_runner MODIFY COLUMN config_path VARCHAR(255) NULL;
+	ALTER TABLE analysis_runner MODIFY COLUMN audit_log_id INT NULL;
+	</sql>
+
 	<!-- Migrate all the old analysis_runner records into the new table (don't delete existing ones for now) -->
+
 		<sql> INSERT INTO analysis_runner ( ar_guid, project, timestamp, access_level, repository,
 			`commit`, output_path, script, description, driver_image, config_path, cwd, environment,
 			hail_version, batch_url, submitting_user, meta, audit_log_id ) SELECT
 			COALESCE(JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.ar-guid')), UUID()) as ar_guid,
 			analysis.project as project,
-			STR_TO_DATE(REPLACE(JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.timestamp')), 'T', ' '),
-			'%Y-%m-%d %H:%i:%s.%f') as timestamp, JSON_UNQUOTE(JSON_EXTRACT(analysis.meta,
+			CONVERT(JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.timestamp')), DATETIME) as timestamp,
+			JSON_UNQUOTE(JSON_EXTRACT(analysis.meta,
 			'$.accessLevel')) as access_level, JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.repo'))
 			as repository, JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.commit')) as `commit`,
 			analysis.output as output_path, JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.script')) as
 			script, JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.description')) as description,
 			JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.driverImage')) as driver_image,
 			JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.configPath')) as config_path,
 			JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.cwd')) as cwd,
-			JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.environment')) as environment,
+			COALESCE(JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.environment')), 'gcp') as environment,
 			JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.hailVersion')) as hail_version,
 			JSON_UNQUOTE(JSON_EXTRACT(analysis.meta, '$.batch_url')) as batch_url,
 			COALESCE(audit_log.on_behalf_of, analysis.author) as submitting_user, JSON_REMOVE(

db/python/layers/billing.py

@@ -6,7 +6,7 @@
 from db.python.tables.bq.billing_raw import BillingRawTable
 from models.enums import BillingSource
 from models.models import (
-    BillingBatchCostRecord,
+    AnalysisCostRecord,
     BillingColumn,
     BillingCostBudgetRecord,
     BillingTotalCostQueryModel,
@@ -204,7 +204,7 @@ async def get_running_cost(
     async def get_cost_by_ar_guid(
         self,
         ar_guid: str | None = None,
-    ) -> list[BillingBatchCostRecord]:
+    ) -> list[AnalysisCostRecord]:
         """
         Get Costs by AR GUID
         """
@@ -229,7 +229,7 @@ async def get_cost_by_ar_guid(
     async def get_cost_by_batch_id(
         self,
         batch_id: str | None = None,
-    ) -> list[BillingBatchCostRecord]:
+    ) -> list[AnalysisCostRecord]:
         """
         Get Costs by Batch ID
         """

db/python/tables/bq/billing_daily_extended.py

@@ -7,7 +7,7 @@
     BillingBaseTable,
     time_optimisation_parameter,
 )
-from models.models import BillingBatchCostRecord, BillingColumn
+from models.models import AnalysisCostRecord, BillingColumn
 
 
 class BillingDailyExtendedTable(BillingBaseTable):
@@ -60,7 +60,7 @@ async def get_batch_cost_summary(
         end_time: datetime,
         batch_ids: list[str] | None,
         ar_guid: str | None,
-    ) -> list[BillingBatchCostRecord]:
+    ) -> list[AnalysisCostRecord]:
         """
         Get summary of AR run
         """
@@ -361,9 +361,7 @@ async def get_batch_cost_summary(
         query_job_result = self._execute_query(_query, query_parameters, False)
 
         if query_job_result:
-            return [
-                BillingBatchCostRecord.from_json(dict(row)) for row in query_job_result
-            ]
+            return [AnalysisCostRecord.from_dict(dict(row)) for row in query_job_result]
 
         # return empty list if no record found
         return []

deploy/python/version.txt

@@ -1 +1 @@
-6.8.0
+6.9.0

metamist/graphql/__init__.py

@@ -4,18 +4,23 @@
     - construct queries using the `gql` function (which validates graphql syntax)
     - validate queries with metamist schema (by fetching the schema)
 """
+
 import os
+from json.decoder import JSONDecodeError
 from typing import Any, Dict
 
+import backoff
 from gql import Client
 from gql import gql as gql_constructor
 from gql.transport.aiohttp import AIOHTTPTransport
 from gql.transport.aiohttp import log as aiohttp_logger
+from gql.transport.exceptions import TransportServerError
 from gql.transport.requests import RequestsHTTPTransport
 from gql.transport.requests import log as requests_logger
 
 # this does not import itself, it imports the module
 from graphql import DocumentNode  # type: ignore
+from requests.exceptions import HTTPError
 
 from cpg_utils.cloud import get_google_identity_token
 
@@ -137,8 +142,17 @@ def validate(doc: DocumentNode, client=None, use_local_schema=False):
 
 
 # use older style typing to broaden supported Python versions
+@backoff.on_exception(
+    backoff.expo,
+    exception=(HTTPError, JSONDecodeError, TransportServerError),
+    max_time=10,
+    max_tries=3,
+)
 def query(
-    _query: str | DocumentNode, variables: Dict = None, client: Client = None, log_response: bool = False
+    _query: str | DocumentNode,
+    variables: Dict | None = None,
+    client: Client | None = None,
+    log_response: bool = False,
 ) -> Dict[str, Any]:
     """Query the metamist GraphQL API"""
     if variables is None:
@@ -159,8 +173,17 @@ def query(
     return response
 
 
+@backoff.on_exception(
+    backoff.expo,
+    exception=(HTTPError, JSONDecodeError, TransportServerError),
+    max_time=10,
+    max_tries=3,
+)
 async def query_async(
-    _query: str | DocumentNode, variables: Dict = None, client: Client = None, log_response: bool = False
+    _query: str | DocumentNode,
+    variables: Dict | None = None,
+    client: Client | None = None,
+    log_response: bool = False,
 ) -> Dict[str, Any]:
     """Asynchronously query the Metamist GraphQL API"""
     if variables is None:

models/base.py

@@ -8,6 +8,11 @@
 class SMBase(BaseModel):
     """Base object for all models"""
 
+    @classmethod
+    def from_dict(cls, d: dict):
+        """Create an object from a dictionary"""
+        return cls(**d)
+
 
 def parse_sql_bool(val: str | int | bytes) -> bool | None:
     """Parse a string from a sql bool"""

models/models/__init__.py

@@ -12,7 +12,7 @@
 from models.models.assay import Assay, AssayInternal, AssayUpsert, AssayUpsertInternal
 from models.models.audit_log import AuditLogId, AuditLogInternal
 from models.models.billing import (
-    BillingBatchCostRecord,
+    AnalysisCostRecord,
     BillingColumn,
     BillingCostBudgetRecord,
     BillingCostDetailsRecord,