Billing pre-release (#605)

* Bill 112 (#583)

* START: add billing home page and data page

* FIX: fixed style in the nav. Everythign but search bar

* DONE: menu and billing pages done

* REMOVE: remove redundant code

* User Story 1.3: As a user, I want to sort costs by budget and by the percentage of budget used

* First version of Current Billing Cost page.

* Added dropdown to select grouping by Project, Topic or Dataset.

* [BIL-39] User Story 2.3: As a user, I want to view costs in a time series with daily granularity in the billing dashboard (#588)

* Added gcp-projects API to billing, new Billing CostByTime page

* Updated StackedAreaByDateChart wiht more custom properties.

* Bill 150 (#589)

* API: add api changes to allow the get running cost query to filter using invoice month

* IN PROGRESS: trying to cache the call to the seqr prop map API endpoint. UI cleaned up.

* RM: console.log

* Extended bq looks back time to 300 days as we do not have much loaded in the dev table. (#591)

* REFACOR: move data loading logic into the main page to be used by all charts

* DONE: data table on time view page complete

* Added gcp-projects API to billing, new Billing CostByTime page

* Updated StackedAreaByDateChart with more custom properties.

* First version of Bar and Donut charts.

* Upgrading babel / vulnerability.

* Bil 242 - Hide billing pages when env variables aren't set (#600)

* DONE: menu and billing pages done

* REMOVE: remove redundant code

* API: add api changes to allow the get running cost query to filter using invoice month

* FIX: Dropdown to FieldSelector in BillingInvoiceMonthCost

* FIX: all pages and navigation/links working

* FIX: fixed all of the navigate() issues now all links work

* IN PROGRESS: trying to cache the call to the seqr prop map API endpoint. UI cleaned up.

* LINT: fix typing issues in API

* RM: console.log

* Extended bq looks back time to 300 days as we do not have much loaded in the dev table. (#591)

* Billing cost by time data refactor (#592)

* IN PROGRESS: data table on over time cost page

* DONE: data table on time view page complete

* UPDATE: frontend now checks if the the billing endpoint is returning an OK status or not. Hides billing pages on any failed (not 200) status

---------

Co-authored-by: Milo Hyben <[email protected]>

* Added gcp-projects API to billing, new Billing CostByTime page

* Updated StackedAreaByDateChart with more custom properties.

* First version of Bar and Dount charts.

* Upgrading babel / vulnerability.

* Billing - show 24H table fields only to the current invoice month (#604)

* Fixing last 24H billing calculations.

* Limit Budget % and 24H to only latest month.

* Pick only the latest monthly budget row per gcp_project.

* Added Last 24H UTC day to the table header.

* Removing unused job_config.

* Fixing docker image building issues.

* Getting gcp_project data from gcp_billing view instead of aggregated view. (#606)

* Small fix to enable query cost by ar-guid, added example to API docs. (#611)

---------

Co-authored-by: Sabrina Yan <[email protected]>

api/routes/billing.py

@@ -11,6 +11,8 @@
 )
 from db.python.layers.billing import BillingLayer
 from models.models.billing import (
+    BillingColumn,
+    BillingCostBudgetRecord,
     BillingQueryModel,
     BillingRowRecord,
     BillingTotalCostRecord,
@@ -21,6 +23,22 @@
 router = APIRouter(prefix='/billing', tags=['billing'])
 
 
+@router.get(
+    '/gcp-projects',
+    response_model=list[str],
+    operation_id='getGcpProjects',
+)
+@alru_cache(ttl=BILLING_CACHE_RESPONSE_TTL)
+async def get_gcp_projects(
+    author: str = get_author,
+) -> list[str]:
+    """Get list of all GCP projects in database"""
+    connection = BqConnection(author)
+    billing_layer = BillingLayer(connection)
+    records = await billing_layer.get_gcp_projects()
+    return records
+
+
 @router.get(
     '/topics',
     response_model=list[str],
@@ -95,7 +113,7 @@ async def get_datasets(
 
 
 @router.get(
-    '/sequencing_types',
+    '/sequencing-types',
     response_model=list[str],
     operation_id='getSequencingTypes',
 )
@@ -133,7 +151,7 @@ async def get_stages(
 
 
 @router.get(
-    '/sequencing_groups',
+    '/sequencing-groups',
     response_model=list[str],
     operation_id='getSequencingGroups',
 )
@@ -151,6 +169,25 @@ async def get_sequencing_groups(
     return records
 
 
+@router.get(
+    '/invoice-months',
+    response_model=list[str],
+    operation_id='getInvoiceMonths',
+)
+@alru_cache(ttl=BILLING_CACHE_RESPONSE_TTL)
+async def get_invoice_months(
+    author: str = get_author,
+) -> list[str]:
+    """
+    Get list of all invoice months in database
+    Results are sorted DESC
+    """
+    connection = BqConnection(author)
+    billing_layer = BillingLayer(connection)
+    records = await billing_layer.get_invoice_months()
+    return records
+
+
 @router.post(
     '/query', response_model=list[BillingRowRecord], operation_id='queryBilling'
 )
@@ -198,7 +235,8 @@ async def get_total_cost(
             "fields": ["topic"],
             "start_date": "2023-03-01",
             "end_date": "2023-03-31",
-            "order_by": {"cost": true}
+            "order_by": {"cost": true},
+            "source": "aggregate"
         }
 
     2. Get total cost by day and topic for March 2023, order by day ASC and topic DESC:
@@ -283,9 +321,57 @@ async def get_total_cost(
             "order_by": {"cost": true}
         }
 
+    10. Get total gcp_project for month of March 2023, ordered by cost DESC:
+
+        {
+            "fields": ["gcp_project"],
+            "start_date": "2023-03-01",
+            "end_date": "2023-03-31",
+            "order_by": {"cost": true},
+            "source": "gcp_billing"
+        }
+
+    11. Get total cost by sku for given ar_guid, order by cost DESC:
+
+        {
+            "fields": ["sku"],
+            "start_date": "2023-10-23",
+            "end_date": "2023-10-23",
+            "filters": { "ar_guid": "4e53702e-8b6c-48ea-857f-c5d33b7e72d7"},
+            "order_by": {"cost": true}
+        }
+
     """
 
     connection = BqConnection(author)
     billing_layer = BillingLayer(connection)
     records = await billing_layer.get_total_cost(query)
     return records
+
+
+@router.get(
+    '/running-cost/{field}',
+    response_model=list[BillingCostBudgetRecord],
+    operation_id='getRunningCost',
+)
+@alru_cache(ttl=BILLING_CACHE_RESPONSE_TTL)
+async def get_running_costs(
+    field: BillingColumn,
+    invoice_month: str | None = None,
+    source: str | None = None,
+    author: str = get_author,
+) -> list[BillingCostBudgetRecord]:
+    """
+    Get running cost for specified fields in database
+    e.g. fields = ['gcp_project', 'topic']
+    """
+
+    # TODO replace alru_cache with async-cache?
+    # so we can skip author for caching?
+    # pip install async-cache
+    # @AsyncTTL(time_to_live=BILLING_CACHE_RESPONSE_TTL, maxsize=1024, skip_args=2)
+
+    connection = BqConnection(author)
+    billing_layer = BillingLayer(connection)
+    records = await billing_layer.get_running_cost(field, invoice_month, source)
+    return records

api/settings.py

@@ -37,11 +37,15 @@
 SEQUENCING_GROUP_CHECKSUM_OFFSET = int(os.getenv('SM_SEQUENCINGGROUPCHECKOFFSET', '9'))
 
 # billing settings
-BQ_GCP_BILLING_PROJECT = os.getenv('SM_GCP_BILLING_PROJECT')
+BQ_AGGREG_VIEW = os.getenv('SM_GCP_BQ_AGGREG_VIEW')
+BQ_AGGREG_RAW = os.getenv('SM_GCP_BQ_AGGREG_RAW')
+BQ_AGGREG_EXT_VIEW = os.getenv('SM_GCP_BQ_AGGREG_EXT_VIEW')
+BQ_BUDGET_VIEW = os.getenv('SM_GCP_BQ_BUDGET_VIEW')
+BQ_GCP_BILLING_VIEW = os.getenv('SM_GCP_BQ_BILLING_VIEW')
+
 # This is to optimise BQ queries, DEV table has data only for Mar 2023
-# TODO change to 7 days or similar before merging into DEV
-BQ_DAYS_BACK_OPTIMAL = 210
-BILLING_CACHE_RESPONSE_TTL = 1800  # 30 minutes
+BQ_DAYS_BACK_OPTIMAL = 30   # Look back 30 days for optimal query
+BILLING_CACHE_RESPONSE_TTL = 3600   # 1 Hour
 
 
 def get_default_user() -> str | None:

api/utils/dates.py

@@ -1,4 +1,6 @@
-from datetime import datetime, date
+from datetime import datetime, date, timedelta
+
+INVOICE_DAY_DIFF = 3
 
 
 def parse_date_only_string(d: str | None) -> date | None:
@@ -10,3 +12,43 @@ def parse_date_only_string(d: str | None) -> date | None:
         return datetime.strptime(d, '%Y-%m-%d').date()
     except Exception as excep:
         raise ValueError(f'Date could not be converted: {d}') from excep
+
+
+def get_invoice_month_range(convert_month: date) -> tuple[date, date]:
+    """
+    Get the start and end date of the invoice month for a given date
+    Start and end date are used mostly for optimising BQ queries
+    All our BQ tables/views are partitioned by day
+    """
+    first_day = convert_month.replace(day=1)
+
+    # Grab the first day of invoice month then subtract INVOICE_DAY_DIFF days
+    start_day = first_day + timedelta(days=-INVOICE_DAY_DIFF)
+
+    if convert_month.month == 12:
+        next_month = first_day.replace(month=1, year=convert_month.year + 1)
+    else:
+        next_month = first_day.replace(month=convert_month.month + 1)
+
+    # Grab the last day of invoice month then add INVOICE_DAY_DIFF days
+    last_day = next_month + timedelta(days=-1) + timedelta(days=INVOICE_DAY_DIFF)
+
+    return start_day, last_day
+
+
+def reformat_datetime(
+    in_date: str | None, in_format: str, out_format: str
+) -> str | None:
+    """
+    Reformat datetime as string to another string format
+    This function take string as input and return string as output
+    """
+    if not in_date:
+        return None
+
+    try:
+        result = datetime.strptime(in_date, in_format)
+        return result.strftime(out_format)
+
+    except Exception as excep:
+        raise ValueError(f'Date could not be converted: {in_date}') from excep