Skip to content

Commit

Permalink
Add Billing Page - CostByMonth - summary of topic per invoice month. (#…
Browse files Browse the repository at this point in the history
…696)

* Add Billing Page - CostByMonth - summary of topic per invoice month.

* Adjustments as per PR review.

* Change Start and End date for Cost Across Invoice Months.

* Change Cost Across Invoice Months to select current year as the default.

* Changes as per PR review.

* Upgrading minimum python version to 3.11.

* Pining testcontainers to 3.7.1, new 4.0.0 is not compatible with our tests.

* Fix create_test_subset.py to comply with python 3.11.
  • Loading branch information
milo-hyben authored Mar 7, 2024
1 parent de9b19e commit f67695d
Show file tree
Hide file tree
Showing 15 changed files with 435 additions and 10 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ jobs:
- uses: actions/setup-python@v4
with:
python-version: "3.10"
python-version: "3.11"

- uses: actions/setup-java@v3
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/lint.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:

- uses: actions/setup-python@v4
with:
python-version: "3.10"
python-version: "3.11"

- uses: actions/setup-java@v2
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:

- uses: actions/setup-python@v5
with:
python-version: "3.10"
python-version: "3.11"

- uses: actions/setup-java@v4
with:
Expand Down
2 changes: 1 addition & 1 deletion docs/installation.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ choco install mariadb --version=10.8.3
- Additional dev requirements are listed in `requirements-dev.txt`.
- Packages for the sever-side code are listed in `requirements.txt`.

We *STRONGLY* encourage the use of `pyenv` for managing Python versions. Debugging and the server will run on a minimum python version of 3.10. Refer to the [team-docs](https://github.com/populationgenomics/team-docs/blob/main/python.md) for more instructions on how to set this up.
We *STRONGLY* encourage the use of `pyenv` for managing Python versions. Debugging and the server will run on a minimum python version of 3.11. Refer to the [team-docs](https://github.com/populationgenomics/team-docs/blob/main/python.md) for more instructions on how to set this up.

Use of a virtual environment to contain all requirements is highly recommended:

Expand Down
6 changes: 5 additions & 1 deletion models/models/billing.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,11 @@ def to_filter(self) -> BillingFilter:
# add filters as attributes
for fk, fv in self.filters.items():
# fk is BillColumn, fv is value
setattr(billing_filter, fk.value, GenericBQFilter(eq=fv))
# if fv is a list, then use IN filter
if isinstance(fv, list):
setattr(billing_filter, fk.value, GenericBQFilter(in_=fv))
else:
setattr(billing_filter, fk.value, GenericBQFilter(eq=fv))

return billing_filter

Expand Down
2 changes: 1 addition & 1 deletion mypy.ini
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Global options:

[mypy]
python_version = 3.10
python_version = 3.11
; warn_return_any = True
; warn_unused_configs = True

Expand Down
2 changes: 1 addition & 1 deletion requirements-dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ flake8-bugbear
nest-asyncio
pre-commit
pylint
testcontainers[mariadb]
testcontainers[mariadb]==3.7.1
types-PyMySQL
# some strawberry dependency
strawberry-graphql[debug-server]==0.206.0
Expand Down
4 changes: 3 additions & 1 deletion scripts/create_test_subset.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,9 @@ def main(
logger.info(f'Found {len(all_sids)} sample ids in {project}')

# 3. Randomly select from the remaining sgs
additional_samples.update(random.sample(all_sids - additional_samples, samples_n))
additional_samples.update(
random.sample(list(all_sids - additional_samples), samples_n)
)

# 4. Query all the samples from the selected sgs
logger.info(f'Transfering {len(additional_samples)} samples. Querying metadata.')
Expand Down
9 changes: 9 additions & 0 deletions web/src/Routes.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
BillingCostByAnalysis,
BillingInvoiceMonthCost,
BillingCostByCategory,
BillingCostByMonth,
} from './pages/billing'
import DocumentationArticle from './pages/docs/Documentation'
import SampleView from './pages/sample/SampleView'
Expand Down Expand Up @@ -49,6 +50,14 @@ const Routes: React.FunctionComponent = () => (
/>

<Route path="/billing/" element={<BillingHome />} />
<Route
path="/billing/costByMonth"
element={
<ErrorBoundary>
<BillingCostByMonth />
</ErrorBoundary>
}
/>
<Route path="/billing/invoiceMonthCost" element={<BillingInvoiceMonthCost />} />
<Route
path="/billing/costByTime"
Expand Down
258 changes: 258 additions & 0 deletions web/src/pages/billing/BillingCostByMonth.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,258 @@
import * as React from 'react'
import { useLocation, useNavigate, useSearchParams } from 'react-router-dom'
import { Button, Card, Grid, Input, Message, Table as SUITable } from 'semantic-ui-react'
import {
BillingApi,
BillingColumn,
BillingSource,
BillingTotalCostQueryModel,
BillingTotalCostRecord,
} from '../../sm-api'

import {
getAdjustedDay,
generateInvoiceMonths,
getCurrentInvoiceMonth,
getCurrentInvoiceYearStart,
} from '../../shared/utilities/formatDates'
import { IStackedAreaByDateChartData } from '../../shared/components/Graphs/StackedAreaByDateChart'
import BillingCostByMonthTable from './components/BillingCostByMonthTable'
import LoadingDucks from '../../shared/components/LoadingDucks/LoadingDucks'
import generateUrl from '../../shared/utilities/generateUrl'
import FieldSelector from './components/FieldSelector'

const BillingCostByTime: React.FunctionComponent = () => {
const [searchParams] = useSearchParams()

const [start, setStart] = React.useState<string>(
searchParams.get('start') ?? getCurrentInvoiceYearStart()
)
const [end, setEnd] = React.useState<string>(
searchParams.get('end') ?? getCurrentInvoiceMonth()
)

// Data loading
const [isLoading, setIsLoading] = React.useState<boolean>(true)
const [error, setError] = React.useState<string | undefined>()
const [message, setMessage] = React.useState<string | undefined>()
const [months, setMonths] = React.useState<string[]>([])
const [data, setData] = React.useState<IStackedAreaByDateChartData[]>([])

// use navigate and update url params
const location = useLocation()
const navigate = useNavigate()

const updateNav = (st: string, ed: string) => {
const url = generateUrl(location, {
start: st,
end: ed,
})
navigate(url)
}

const changeDate = (name: string, value: string) => {
let start_update = start
let end_update = end
if (name === 'start') start_update = value
if (name === 'end') end_update = value
setStart(start_update)
setEnd(end_update)
updateNav(start_update, end_update)
}

const convertInvoiceMonth = (invoiceMonth: string, start: Boolean) => {
const year = invoiceMonth.substring(0, 4)
const month = invoiceMonth.substring(4, 6)
if (start) return `${year}-${month}-01`
// get last day of month
const lastDay = new Date(parseInt(year), parseInt(month), 0).getDate()
return `${year}-${month}-${lastDay}`
}

const convertCostCategory = (costCategory: string) => {
if (costCategory === 'Cloud Storage') {
return 'Storage Cost'
}
return 'Compute Cost'
}

const getData = (query: BillingTotalCostQueryModel) => {
setIsLoading(true)
setError(undefined)
setMessage(undefined)
new BillingApi()
.getTotalCost(query)
.then((response) => {
setIsLoading(false)

// calc totals per topic, month and category
const recTotals: { [key: string]: { [key: string]: number } } = {}
const recMonths: string[] = []

response.data.forEach((item: BillingTotalCostRecord) => {
const { day, cost_category, topic, cost } = item
const ccat = convertCostCategory(cost_category)
if (recMonths.indexOf(day) === -1) {
recMonths.push(day)
}
if (!recTotals[topic]) {
recTotals[topic] = {}
}
if (!recTotals[topic][day]) {
recTotals[topic][day] = {}
}
if (!recTotals[topic][day][ccat]) {
recTotals[topic][day][ccat] = 0
}
recTotals[topic][day][ccat] += cost
})

setMonths(recMonths)
setData(recTotals)
})
.catch((er) => setError(er.message))
}

const messageComponent = () => {
if (message) {
return (
<Message negative onDismiss={() => setError(undefined)}>
{message}
</Message>
)
}
if (error) {
return (
<Message negative onDismiss={() => setError(undefined)}>
{error}
<br />
<Button negative onClick={() => setStart(start)}>
Retry
</Button>
</Message>
)
}
if (isLoading) {
return (
<div>
<LoadingDucks />
<p style={{ textAlign: 'center', marginTop: '5px' }}>
<em>This query takes a while...</em>
</p>
</div>
)
}
return null
}

const dataComponent = () => {
if (message || error || isLoading) {
return null
}

if (!message && !error && !isLoading && (!data || data.length === 0)) {
return (
<Card
fluid
style={{ padding: '20px', overflowX: 'scroll' }}
id="billing-container-charts"
>
No Data
</Card>
)
}

return (
<>
<Card
fluid
style={{ padding: '20px', overflowX: 'scroll' }}
id="billing-container-data"
>
<BillingCostByMonthTable
start={start}
end={end}
isLoading={isLoading}
data={data}
months={months}
/>
</Card>
</>
)
}

const onMonthStart = (event: any, data: any) => {
changeDate('start', data.value)
}

const onMonthEnd = (event: any, data: any) => {
changeDate('end', data.value)
}

React.useEffect(() => {
if (Boolean(start) && Boolean(end)) {
// valid selection, retrieve data
getData({
fields: [BillingColumn.Topic, BillingColumn.CostCategory],
start_date: getAdjustedDay(convertInvoiceMonth(start, true), -2),
end_date: getAdjustedDay(convertInvoiceMonth(end, false), 3),
order_by: { day: false },
source: BillingSource.Aggregate,
time_periods: 'invoice_month',
filters: {
invoice_month: generateInvoiceMonths(start, end),
},
})
} else {
// invalid selection,
setIsLoading(false)
setError(undefined)

if (start === undefined || start === null || start === '') {
setMessage('Please select Start date')
} else if (end === undefined || end === null || end === '') {
setMessage('Please select End date')
}
}
}, [start, end])

return (
<>
<Card fluid style={{ padding: '20px' }} id="billing-container">
<h1
style={{
fontSize: 40,
}}
>
Cost Across Invoice Months (Topic only)
</h1>

<Grid columns="equal" stackable doubling>
<Grid.Column className="field-selector-label">
<FieldSelector
label="Start"
fieldName={BillingColumn.InvoiceMonth}
onClickFunction={onMonthStart}
selected={start}
/>
</Grid.Column>

<Grid.Column className="field-selector-label">
<FieldSelector
label="Finish"
fieldName={BillingColumn.InvoiceMonth}
onClickFunction={onMonthEnd}
selected={end}
/>
</Grid.Column>
</Grid>
</Card>

{messageComponent()}

{dataComponent()}
</>
)
}

export default BillingCostByTime
2 changes: 1 addition & 1 deletion web/src/pages/billing/BillingInvoiceMonthCost.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ const BillingCurrentCost = () => {

return (
<>
<h1>Billing By Invoice Month</h1>
<h1>Cost By Invoice Month</h1>

<Grid columns="equal" stackable doubling>
<Grid.Column>
Expand Down
Loading

0 comments on commit f67695d

Please sign in to comment.