From 0ff05ef5b26e93c63492a37d8fdcb8c5335a6ed9 Mon Sep 17 00:00:00 2001 From: Jim O'Donnell Date: Wed, 6 Mar 2024 13:24:51 +0000 Subject: [PATCH] feat: map dosing and observation variables (#351) * Map data to model variables Show a table of dose amount data (and units), with select menus to map these to model inputs. Show a table of observation variables (and units), with select menus to map those to model outputs. * Add a mapped qname to biomarkers Store the mapped variable qname on biomarker types. Read it from OBSERVATION_VARIABLE in a dataset. * Support ES2015 * Allow for multiple variable mappings Map each Administration ID to a dosing compartment. Map each Observation ID to a model output. Add new columns to the CSV data, with mappings and optional units. * Preview the final dataset Add a final step to the upload, which will preview the final CSV before saving it. * Save modified dataset to backend - load or create a dataset when we start an upload. - save the dataset when we finish an upload. - modify the `/datasets/:dataset_id:/csv` endpoint to accept a JSON string. - update the dataset API to allow filtering by project ID. * Allow for a single unit column - When there's a single unit column, use that column for both dosing and observations. - Split the CSV data into dosing rows and observation rows. - Add administration route to the Map Dosing screen. - Allow for dimensionless observation units. - Filter mapped observation variables for compatibility with the observation unit value. --- frontend-v2/src/app/backendApi.ts | 30 +-- frontend-v2/src/features/data/LoadData.tsx | 13 +- .../src/features/data/LoadDataStepper.tsx | 90 +++++++- frontend-v2/src/features/data/LoadDataTab.tsx | 14 +- frontend-v2/src/features/data/MapDosing.tsx | 203 ++++++++++++++++ frontend-v2/src/features/data/MapHeaders.tsx | 7 +- .../src/features/data/MapObservations.tsx | 217 +++++++++++++++++- frontend-v2/src/features/data/PreviewData.tsx | 52 +++++ .../src/features/data/normaliseDataHeaders.ts | 1 + frontend-v2/tsconfig.json | 2 +- pkpdapp/pkpdapp/api/serializers/dataset.py | 6 +- pkpdapp/pkpdapp/api/views/dataset.py | 21 +- .../0006_biomarkertype_mapped_qname.py | 27 +++ pkpdapp/pkpdapp/models/biomarker_type.py | 5 + pkpdapp/pkpdapp/models/dataset.py | 9 +- pkpdapp/pkpdapp/utils/data_parser.py | 8 + pkpdapp/schema.yml | 25 +- 17 files changed, 665 insertions(+), 65 deletions(-) create mode 100644 frontend-v2/src/features/data/MapDosing.tsx create mode 100644 frontend-v2/src/features/data/PreviewData.tsx create mode 100644 pkpdapp/pkpdapp/migrations/0006_biomarkertype_mapped_qname.py diff --git a/frontend-v2/src/app/backendApi.ts b/frontend-v2/src/app/backendApi.ts index 52befdd3..6fd8a602 100644 --- a/frontend-v2/src/app/backendApi.ts +++ b/frontend-v2/src/app/backendApi.ts @@ -236,7 +236,10 @@ const injectedRtkApi = api.injectEndpoints({ }), }), datasetList: build.query({ - query: () => ({ url: `/api/dataset/` }), + query: (queryArg) => ({ + url: `/api/dataset/`, + params: { project_id: queryArg.projectId }, + }), }), datasetCreate: build.mutation< DatasetCreateApiResponse, @@ -1158,7 +1161,10 @@ export type CompoundDestroyApiArg = { id: number; }; export type DatasetListApiResponse = /** status 200 */ DatasetRead[]; -export type DatasetListApiArg = void; +export type DatasetListApiArg = { + /** Filter results by project ID */ + projectId?: number; +}; export type DatasetCreateApiResponse = /** status 201 */ DatasetRead; export type DatasetCreateApiArg = { dataset: Dataset; @@ -1671,6 +1677,7 @@ export type BiomarkerType = { display?: boolean; color?: number; axis?: boolean; + mapped_qname?: string; stored_unit: number; dataset: number; display_unit: number; @@ -1689,6 +1696,7 @@ export type BiomarkerTypeRead = { display?: boolean; color?: number; axis?: boolean; + mapped_qname?: string; stored_unit: number; dataset: number; display_unit: number; @@ -1701,6 +1709,7 @@ export type PatchedBiomarkerType = { display?: boolean; color?: number; axis?: boolean; + mapped_qname?: string; stored_unit?: number; dataset?: number; display_unit?: number; @@ -1719,6 +1728,7 @@ export type PatchedBiomarkerTypeRead = { display?: boolean; color?: number; axis?: boolean; + mapped_qname?: string; stored_unit?: number; dataset?: number; display_unit?: number; @@ -2127,9 +2137,7 @@ export type Inference = { time_elapsed?: number; number_of_function_evals?: number; task_id?: string | null; - metadata?: { - [key: string]: any; - }; + metadata?: any; project: number; algorithm?: number; initialization_inference?: number | null; @@ -2149,9 +2157,7 @@ export type InferenceRead = { time_elapsed?: number; number_of_function_evals?: number; task_id?: string | null; - metadata?: { - [key: string]: any; - }; + metadata?: any; project: number; algorithm?: number; initialization_inference?: number | null; @@ -2170,9 +2176,7 @@ export type PatchedInference = { time_elapsed?: number; number_of_function_evals?: number; task_id?: string | null; - metadata?: { - [key: string]: any; - }; + metadata?: any; project?: number; algorithm?: number; initialization_inference?: number | null; @@ -2192,9 +2196,7 @@ export type PatchedInferenceRead = { time_elapsed?: number; number_of_function_evals?: number; task_id?: string | null; - metadata?: { - [key: string]: any; - }; + metadata?: any; project?: number; algorithm?: number; initialization_inference?: number | null; diff --git a/frontend-v2/src/features/data/LoadData.tsx b/frontend-v2/src/features/data/LoadData.tsx index b13921ef..8865e076 100644 --- a/frontend-v2/src/features/data/LoadData.tsx +++ b/frontend-v2/src/features/data/LoadData.tsx @@ -1,9 +1,9 @@ -import { Alert, Box, Stack, Table, TableBody, TableCell, TableContainer, TableHead, TableRow } from '@mui/material'; -import Papa, { ParseError, ParseMeta } from 'papaparse' -import React, {useCallback, useState} from 'react' +import { Alert, Box, Stack } from '@mui/material'; +import Papa from 'papaparse' +import { FC, useCallback, useState} from 'react' import {useDropzone} from 'react-dropzone' import MapHeaders from './MapHeaders'; -import { manditoryHeaders, normaliseHeader, normalisedHeaders } from './normaliseDataHeaders'; +import { manditoryHeaders, normaliseHeader } from './normaliseDataHeaders'; import { StepperState } from './LoadDataStepper'; export type Row = {[key: string]: string}; @@ -48,7 +48,7 @@ const validateNormalisedFields = (fields: Field[]) => { return errors; } -const LoadData: React.FC = ({state, firstTime}) => { +const LoadData: FC = ({state, firstTime}) => { const [errors, setErrors] = useState(firstTime ? [] : validateNormalisedFields(state.normalisedFields)); const [showData, setShowData] = useState(state.data.length > 0 && state.fields.length > 0); @@ -62,7 +62,6 @@ const LoadData: React.FC = ({state, firstTime}) => { // Parse the CSV data const rawCsv = reader.result as string; const csvData = Papa.parse(rawCsv.trim(), { header: true }); - const data = csvData.data as Data; const fields = csvData.meta.fields || []; const normalisedFields = fields.map(normaliseHeader); const errors = csvData.errors.map((e) => e.message).concat(validateNormalisedFields(normalisedFields)); @@ -77,7 +76,7 @@ const LoadData: React.FC = ({state, firstTime}) => { reader.readAsText(file) }) - }, []) + }, [state]) const {getRootProps, getInputProps} = useDropzone({onDrop}) const setNormalisedFields = (fields: Field[]) => { diff --git a/frontend-v2/src/features/data/LoadDataStepper.tsx b/frontend-v2/src/features/data/LoadDataStepper.tsx index 0485ec2d..28a1c618 100644 --- a/frontend-v2/src/features/data/LoadDataStepper.tsx +++ b/frontend-v2/src/features/data/LoadDataStepper.tsx @@ -1,4 +1,6 @@ -import * as React from 'react'; +import { FC, useEffect } from 'react'; +import { useSelector } from "react-redux"; +import Papa from 'papaparse' import Box from '@mui/material/Box'; import Stepper from '@mui/material/Stepper'; import Step from '@mui/material/Step'; @@ -8,9 +10,18 @@ import Typography from '@mui/material/Typography'; import LoadData from './LoadData'; import { useState } from 'react'; import MapObservations from './MapObservations'; +import MapDosing from './MapDosing'; +import PreviewData from './PreviewData'; +import { RootState } from "../../app/store"; +import { + DatasetRead, + useDatasetListQuery, + useDatasetCreateMutation, + useDatasetCsvUpdateMutation, +} from '../../app/backendApi'; -const stepLabels = ['Upload Data', 'Map Observations']; -const stepComponents = [LoadData, MapObservations]; +const stepLabels = ['Upload Data', 'Map Dosing', 'Map Observations', 'Preview Dataset']; +const stepComponents = [LoadData, MapDosing, MapObservations, PreviewData]; type Row = {[key: string]: string}; type Data = Row[]; @@ -27,17 +38,30 @@ export type StepperState = { setData: (data: Data) => void; amountUnit?: string; setAmountUnit: (amountUnit: string) => void; - observationUnits?: {[key: string]: string}; - setObservationUnits: (observationUnits: {[key: string]: string}) => void; } -const LoadDataStepper: React.FC = () => { +const LoadDataStepper: FC = () => { + const [dataset, setDataset] = useState(null); const [data, setData] = useState([]); const [fields, setFields] = useState([]); const [normalisedFields, setNormalisedFields] = useState([]); const [timeUnit, setTimeUnit] = useState(undefined); const [amountUnit, setAmountUnit] = useState(undefined); - const [observationUnits, setObservationUnits] = useState<{[key: string]: string}>({}); + const selectedProject = useSelector( + (state: RootState) => state.main.selectedProject, + ); + const selectedProjectOrZero = selectedProject || 0; + const { data: datasets = [], isLoading: isDatasetLoading } = useDatasetListQuery( + { projectId: selectedProjectOrZero }, + { skip: !selectedProject }, + ); + const [ + createDataset + ] = useDatasetCreateMutation(); + const [ + updateDataset + ] = useDatasetCsvUpdateMutation(); + const state = { fields, @@ -49,13 +73,50 @@ const LoadDataStepper: React.FC = () => { timeUnit, setTimeUnit, amountUnit, - setAmountUnit, - observationUnits, - setObservationUnits, + setAmountUnit }; const [stepState, setStepState] = useState({ activeStep: 0, maxStep: 0 }); const StepComponent = stepComponents[stepState.activeStep]; + const isFinished = stepState.activeStep === stepLabels.length; + + useEffect(function onDataLoad() { + async function addDataset() { + let [dataset] = datasets; + if (!dataset) { + const response = await createDataset({ + dataset: { + name: 'New Dataset', + project: selectedProjectOrZero, + } + }); + if ('data' in response && response.data) { + dataset = response.data; + } + } + console.log({dataset}); + setDataset(dataset); + } + if (!isDatasetLoading) { + addDataset(); + } + }, [datasets, createDataset, isDatasetLoading]); + + useEffect(function onFinished() { + if (isFinished && dataset?.id) { + try { + const csv = Papa.unparse(data); + updateDataset({ + id: dataset.id, + datasetCsv: { + csv + } + }) + } catch (e) { + console.error(e); + } + } + }, [isFinished, updateDataset, dataset?.id, data]) const handleNext = () => { setStepState((prevActiveStep) => ({ @@ -77,10 +138,15 @@ const LoadDataStepper: React.FC = () => { ))} - {stepState.activeStep === stepLabels.length ? 'The process is completed' : } + + {isFinished ? + 'The process is completed' : + + } + - diff --git a/frontend-v2/src/features/data/LoadDataTab.tsx b/frontend-v2/src/features/data/LoadDataTab.tsx index 394f3147..864c534d 100644 --- a/frontend-v2/src/features/data/LoadDataTab.tsx +++ b/frontend-v2/src/features/data/LoadDataTab.tsx @@ -1,17 +1,7 @@ -import * as React from 'react'; +import { useState } from 'react'; import Button from '@mui/material/Button'; -import Avatar from '@mui/material/Avatar'; -import List from '@mui/material/List'; -import ListItem from '@mui/material/ListItem'; -import ListItemAvatar from '@mui/material/ListItemAvatar'; -import ListItemButton from '@mui/material/ListItemButton'; -import ListItemText from '@mui/material/ListItemText'; import DialogTitle from '@mui/material/DialogTitle'; import Dialog from '@mui/material/Dialog'; -import PersonIcon from '@mui/icons-material/Person'; -import AddIcon from '@mui/icons-material/Add'; -import Typography from '@mui/material/Typography'; -import { blue } from '@mui/material/colors'; import LoadDataStepper from './LoadDataStepper'; import { DialogContent } from '@mui/material'; @@ -38,7 +28,7 @@ function LoadDataDialog(props: LoadDataDialogProps) { } export default function LoadDataTab() { - const [open, setOpen] = React.useState(false); + const [open, setOpen] = useState(false); const handleClickOpen = () => { setOpen(true); diff --git a/frontend-v2/src/features/data/MapDosing.tsx b/frontend-v2/src/features/data/MapDosing.tsx new file mode 100644 index 00000000..cb087ac5 --- /dev/null +++ b/frontend-v2/src/features/data/MapDosing.tsx @@ -0,0 +1,203 @@ +import { FC } from 'react'; +import { + Box, + Select, + FormControl, + MenuItem, + InputLabel, + Table, + TableHead, + TableRow, + TableCell, + TableBody, + Typography, + SelectChangeEvent +} from "@mui/material"; +import { StepperState } from "./LoadDataStepper"; +import { useSelector } from "react-redux"; +import { RootState } from "../../app/store"; +import { + VariableRead, + useCombinedModelListQuery, + useProjectRetrieveQuery, + useUnitListQuery, + useVariableListQuery +} from "../../app/backendApi"; + +interface IMapDosing { + state: StepperState; + firstTime: boolean; +} + +const MapDosing: FC = ({ state, firstTime }: IMapDosing) => { + const projectId = useSelector( + (state: RootState) => state.main.selectedProject, + ); + const projectIdOrZero = projectId || 0; + const { data: project } = + useProjectRetrieveQuery({ id: projectId || 0 }, { skip: !projectId }); + const { data: models = [] } = + useCombinedModelListQuery( + { projectId: projectIdOrZero }, + { skip: !projectId }, + ); + const { data: units } = useUnitListQuery( + { compoundId: project?.compound }, + { skip: !project || !project.compound }, + ); + const [model] = models; + const { data: variables } = useVariableListQuery( + { dosedPkModelId: model?.id || 0 }, + { skip: !model?.id }, + ); + + const amountField = state.fields.find( + (field, i) => state.normalisedFields[i] === 'Amount' + ); + const dosingRows = amountField ? state.data.filter(row => row[amountField] && row[amountField] !== '.') : []; + const amountUnitField = state.fields.find( + (field, i) => ['Amount Unit', 'Unit'].includes(state.normalisedFields[i]) + ); + const administrationIdField = state.fields.find( + (field, i) => state.normalisedFields[i] === 'Administration ID' + ); + const administrationIds = administrationIdField ? + dosingRows.map(row => row[administrationIdField]) : + []; + const uniqueAdministrationIds = [...new Set(administrationIds)]; + const routeField = state.fields.find(field => field.toLowerCase() === 'route'); + + const isAmount = (variable: VariableRead) => { + const amountUnits = units?.find( + (unit) => unit.symbol === "pmol/kg", + )?.compatible_units; + const variableUnit = units?.find((unit) => unit.id === variable.unit); + return variableUnit?.symbol !== "" && + amountUnits?.find( + (unit) => parseInt(unit.id) === variable.unit, + ) !== undefined; + } + const modelAmounts = variables?.filter(isAmount) || []; + + const handleAmountMappingChange = (id: string) => (event: SelectChangeEvent) => { + const nextData = [...state.data]; + const { value } = event.target; + nextData.filter(row => administrationIdField ? row[administrationIdField] === id : true) + .forEach(row => { + row['Dosing Variable'] = value; + }) + state.setData(nextData); + } + const handleAmountUnitChange = (id: string) => (event: SelectChangeEvent) => { + const nextData = [...state.data]; + const { value } = event.target; + nextData.filter(row => administrationIdField ? row[administrationIdField] === id : true) + .forEach(row => { + row['Amount Unit'] = value; + }) + state.setData(nextData); + } + return ( + <> +

Map dose amounts to dosing compartments in the model.

+ + + + + + + {administrationIdField} + + + + + Amount + + + + + Route + + + + + Unit + + + + + Dosing Compartment + + + + + + {uniqueAdministrationIds.map((adminId, index) => { + const currentRow = dosingRows.find(row => administrationIdField ? row[administrationIdField] === adminId : true); + const selectedVariable = variables?.find(variable => variable.qname === currentRow?.['Dosing Variable']); + const compatibleUnits = units?.find(unit => unit.id === selectedVariable?.unit)?.compatible_units; + const adminUnit = amountUnitField && currentRow && currentRow[amountUnitField]; + const amount = amountField && currentRow?.[amountField]; + const route = routeField && currentRow?.[routeField]; + return ( + + + {adminId} + + + + {amount} + + + + + {route} + + + + {adminUnit ? + adminUnit : + + Units + + + } + + + + Variable + + + + + ) + })} + +
+ + + ) +} + +export default MapDosing; + diff --git a/frontend-v2/src/features/data/MapHeaders.tsx b/frontend-v2/src/features/data/MapHeaders.tsx index fad26d2a..f606822b 100644 --- a/frontend-v2/src/features/data/MapHeaders.tsx +++ b/frontend-v2/src/features/data/MapHeaders.tsx @@ -1,5 +1,6 @@ -import { TableContainer, Table, TableHead, TableRow, TableCell, TableBody, Select, FormControl, MenuItem, InputLabel, Typography } from "@mui/material"; -import LoadData, { Data, Field } from "./LoadData"; +import { FC } from "react"; +import { Table, TableHead, TableRow, TableCell, TableBody, Select, FormControl, MenuItem, InputLabel, Typography } from "@mui/material"; +import { Data, Field } from "./LoadData"; import { normalisedHeaders } from "./normaliseDataHeaders"; interface IMapHeaders { @@ -9,7 +10,7 @@ interface IMapHeaders { setNormalisedFields: (fields: Field[]) => void; } -const MapHeaders: React.FC = ({data, fields, normalisedFields, setNormalisedFields}: IMapHeaders) => { +const MapHeaders: FC = ({data, fields, normalisedFields, setNormalisedFields}: IMapHeaders) => { const normalisedHeadersOptions = normalisedHeaders.map((header) => ({value: header, label: header})); diff --git a/frontend-v2/src/features/data/MapObservations.tsx b/frontend-v2/src/features/data/MapObservations.tsx index cc8726d6..25062c98 100644 --- a/frontend-v2/src/features/data/MapObservations.tsx +++ b/frontend-v2/src/features/data/MapObservations.tsx @@ -1,13 +1,224 @@ -import { Field, Data } from "./LoadData"; +import { FC } from 'react'; +import { Box, Select, FormControl, MenuItem, InputLabel, Table, TableHead, TableRow, TableCell, TableBody, Typography, SelectChangeEvent } from "@mui/material"; import { StepperState } from "./LoadDataStepper"; +import { useSelector } from "react-redux"; +import { RootState } from "../../app/store"; +import { + useCombinedModelListQuery, + useProjectRetrieveQuery, + useUnitListQuery, + useVariableListQuery +} from "../../app/backendApi"; interface IMapObservations { state: StepperState; firstTime: boolean; } -const MapObservations: React.FC = ({state, firstTime}: IMapObservations) => { - return (null) +const MapObservations: FC = ({state}: IMapObservations) => { + const projectId = useSelector( + (state: RootState) => state.main.selectedProject, + ); + const projectIdOrZero = projectId || 0; + const { data: project } = + useProjectRetrieveQuery({ id: projectId || 0 }, { skip: !projectId }); + const { data: models = [] } = + useCombinedModelListQuery( + { projectId: projectIdOrZero }, + { skip: !projectId }, + ); + const [ model ] = models; + const { data: variables } = useVariableListQuery( + { dosedPkModelId: model?.id || 0 }, + { skip: !model?.id }, + ); + const { data: units } = useUnitListQuery( + { compoundId: project?.compound }, + { skip: !project || !project.compound }, + ); + + const observationField = state.fields.find( + (field, i) => state.normalisedFields[i] === 'Observation' + ) || ''; + const observationIdField = state.fields.find( + (field, i) => state.normalisedFields[i] === 'Observation ID' + ); + const observationRows = observationField ? state.data.filter(row => row[observationField] !== '.') : []; + const observationIds = observationIdField ? + observationRows.map(row => row[observationIdField]) : + [observationField]; + const uniqueObservationIds = [...new Set(observationIds)]; + const observationValues = observationField ? + observationRows.map(row => row[observationField]) : + []; + const observationUnitField = state.fields.find( + (field, i) => ['Observation Unit', 'Unit'].includes(state.normalisedFields[i]) + ); + const observationUnits = observationRows.map(row => row[observationUnitField || 'Observation Unit']); + const observationVariables = observationRows.map(row => row['Observation Variable']); + + const filterOutputs = model?.is_library_model + ? ["environment.t", "PDCompartment.C_Drug"] + : []; + const modelOutputs = + variables?.filter( + (variable) => + !variable.constant && !filterOutputs.includes(variable.qname), + ) || []; + + const handleObservationChange = (id: string) => (event: SelectChangeEvent) => { + const nextData = [...state.data]; + const { value } = event.target; + const selectedVariable = variables?.find(variable => variable.qname === value); + const defaultUnit = units?.find(unit => unit.id === selectedVariable?.unit); + nextData.filter(row => observationIdField ? row[observationIdField] === id : true) + .forEach(row => { + row['Observation Variable'] = value; + if (!observationUnitField && defaultUnit) { + row['Observation Unit'] = defaultUnit?.symbol + } + }); + state.setData(nextData); + } + const handleUnitChange = (id: string) => (event: SelectChangeEvent) => { + const nextData = [...state.data]; + const { value } = event.target; + nextData.filter(row => observationIdField ? row[observationIdField] === id : true) + .forEach(row => { + row[observationUnitField || 'Observation Unit'] = value; + }); + state.setData(nextData); + } + return ( + <> +

Map observations to variables in the model.

+ + + + + + + {observationIdField} + + + + + Unit + + + + + Observation + + + + + + {uniqueObservationIds.map((obsId) => { + const currentRow = observationRows.find(row => observationIdField ? row[observationIdField] === obsId : true); + const selectedVariable = variables?.find(variable => variable.qname === currentRow?.['Observation Variable']); + const compatibleUnits = units?.find(unit => unit.id === selectedVariable?.unit)?.compatible_units; + const obsUnit = observationUnitField && currentRow && currentRow[observationUnitField]; + let obsUnitSymbol = obsUnit; + ['%', 'fraction', 'ratio'].forEach(token => { + if (obsUnitSymbol?.toLowerCase().includes(token)) { + obsUnitSymbol = ''; + } + }); + const compatibleVariables = modelOutputs.filter(variable => { + const variableUnit = units?.find(unit => unit.id === variable.unit); + const compatibleSymbols = variableUnit?.compatible_units.map(u => u.symbol); + return compatibleSymbols?.includes(obsUnitSymbol || ''); + }); + return ( + + + {obsId} + + + {obsUnit ? + obsUnit : + + Units + + + } + + + + Variable + + + + + ) + })} + +
+ + + + + + Name + + + + + Observation + + + + + Observation Unit + + + + + Mapping + + + + + + {observationValues.map((observation, index) => { + const obsId = observationIds[index]; + const obsUnit = observationUnits[index]; + const obsVariable = observationVariables[index]; + return ( + + {obsId} + {observation} + {obsUnit} + {obsVariable} + + ) + })} + +
+ + + ) } export default MapObservations; diff --git a/frontend-v2/src/features/data/PreviewData.tsx b/frontend-v2/src/features/data/PreviewData.tsx new file mode 100644 index 00000000..9713a8f5 --- /dev/null +++ b/frontend-v2/src/features/data/PreviewData.tsx @@ -0,0 +1,52 @@ +import { FC } from 'react'; +import { Box, Table, TableHead, TableRow, TableCell, TableBody, Typography } from "@mui/material"; +import { StepperState } from "./LoadDataStepper"; + +interface IPreviewData { + state: StepperState; + firstTime: boolean; +} + +const PreviewData: FC = ({ state, firstTime }: IPreviewData) => { + const { data } = state; + const fields = [ + ...state.fields, + 'Dosing Variable', + 'Observation Variable' + ]; + if (!state.normalisedFields.find(field => field === 'Amount Unit')) { + fields.push('Amount Unit') + } + if (!state.normalisedFields.find(field => field === 'Observation Unit')) { + fields.push('Observation Unit') + } + + return ( + + + + + {fields.map((field, index) => ( + + + {field} + + + ))} + + + + {data.map((row, index) => ( + + {fields.map((field, index) => ( + {row[field]} + ))} + + ))} + +
+ + ) +} + +export default PreviewData; diff --git a/frontend-v2/src/features/data/normaliseDataHeaders.ts b/frontend-v2/src/features/data/normaliseDataHeaders.ts index 19be3006..678bbd34 100644 --- a/frontend-v2/src/features/data/normaliseDataHeaders.ts +++ b/frontend-v2/src/features/data/normaliseDataHeaders.ts @@ -21,6 +21,7 @@ const normalisation = { 'Regressor': ['x', 'regressor'], 'Time': ['time', 't', 'ivar'], 'Time Unit': ['time_unit', 'time_units', 't_units', 'tunit'], + 'Unit': ['unit', 'units'], } export const manditoryHeaders = ['ID', 'Time', 'Observation', 'Administration ID', 'Amount'] diff --git a/frontend-v2/tsconfig.json b/frontend-v2/tsconfig.json index 9d379a3c..2d519b5a 100644 --- a/frontend-v2/tsconfig.json +++ b/frontend-v2/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es5", + "target": "es2015", "lib": ["dom", "dom.iterable", "esnext"], "allowJs": true, "skipLibCheck": true, diff --git a/pkpdapp/pkpdapp/api/serializers/dataset.py b/pkpdapp/pkpdapp/api/serializers/dataset.py index 5dc9baec..03f06ca7 100644 --- a/pkpdapp/pkpdapp/api/serializers/dataset.py +++ b/pkpdapp/pkpdapp/api/serializers/dataset.py @@ -7,7 +7,6 @@ from pkpdapp.utils import DataParser from rest_framework import serializers from drf_spectacular.utils import extend_schema_field -import codecs from pkpdapp.models import ( Dataset, Protocol, ) @@ -38,19 +37,18 @@ def get_protocols(self, dataset): class DatasetCsvSerializer(serializers.ModelSerializer): - csv = serializers.FileField() + csv = serializers.CharField() class Meta: model = Dataset fields = ['csv'] def validate_csv(self, csv): - utf8_file = codecs.EncodedFile(csv.open(), "utf-8") parser = DataParser() # error in columns try: - data = parser.parse_from_stream(utf8_file) + data = parser.parse_from_str(csv) except RuntimeError as err: raise serializers.ValidationError(str(err)) except UnicodeDecodeError as err: diff --git a/pkpdapp/pkpdapp/api/views/dataset.py b/pkpdapp/pkpdapp/api/views/dataset.py index 74b31e9c..384d2ba0 100644 --- a/pkpdapp/pkpdapp/api/views/dataset.py +++ b/pkpdapp/pkpdapp/api/views/dataset.py @@ -4,8 +4,10 @@ # copyright notice and full license details. # from rest_framework import ( - viewsets, decorators, parsers, response, status + viewsets, decorators, response, status ) +from drf_spectacular.utils import extend_schema, OpenApiParameter +from drf_spectacular.types import OpenApiTypes from pkpdapp.api.views import ( ProjectFilter, ) @@ -20,11 +22,24 @@ class DatasetView(viewsets.ModelViewSet): serializer_class = DatasetSerializer filter_backends = [ProjectFilter] + @extend_schema( + parameters=[ + OpenApiParameter( + name='project_id', + description='Filter results by project ID', + required=False, + type=OpenApiTypes.INT, + location=OpenApiParameter.QUERY + ), + ], + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + @decorators.action( detail=True, serializer_class=DatasetCsvSerializer, - methods=['PUT'], - parser_classes=[parsers.MultiPartParser], + methods=['PUT'] ) def csv(self, request, pk): obj = self.get_object() diff --git a/pkpdapp/pkpdapp/migrations/0006_biomarkertype_mapped_qname.py b/pkpdapp/pkpdapp/migrations/0006_biomarkertype_mapped_qname.py new file mode 100644 index 00000000..5085fe68 --- /dev/null +++ b/pkpdapp/pkpdapp/migrations/0006_biomarkertype_mapped_qname.py @@ -0,0 +1,27 @@ +# +# This file is part of PKPDApp (https://github.com/pkpdapp-team/pkpdapp) which +# is released under the BSD 3-clause license. See accompanying LICENSE.md for +# copyright notice and full license details. +# +# Generated by Django 3.2.23 on 2024-02-23 16:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('pkpdapp', '0005_alter_derivedvariable_type'), + ] + + operations = [ + migrations.AddField( + model_name='biomarkertype', + name='mapped_qname', + field=models.CharField( + default='', + help_text='qname of the mapped model variable', + max_length=50 + ), + ), + ] diff --git a/pkpdapp/pkpdapp/models/biomarker_type.py b/pkpdapp/pkpdapp/models/biomarker_type.py index 23f3e331..8b2ccfd9 100644 --- a/pkpdapp/pkpdapp/models/biomarker_type.py +++ b/pkpdapp/pkpdapp/models/biomarker_type.py @@ -71,6 +71,11 @@ class BiomarkerType(models.Model): 'True/False if biomarker type displayed on LHS/RHS axis' ) ) + mapped_qname = models.CharField( + default='', + max_length=50, + help_text='qname of the mapped model variable' + ) def get_project(self): return self.dataset.get_project() diff --git a/pkpdapp/pkpdapp/models/dataset.py b/pkpdapp/pkpdapp/models/dataset.py index 96429278..524b81f6 100644 --- a/pkpdapp/pkpdapp/models/dataset.py +++ b/pkpdapp/pkpdapp/models/dataset.py @@ -60,12 +60,18 @@ def replace_data(self, data: pd.DataFrame): # create biomarker types # assume AMOUNT_UNIT and TIME_UNIT are constant for each bt bts_unique = data_without_dose[ - ['OBSERVATION_NAME', 'OBSERVATION_UNIT', 'TIME_UNIT'] + [ + 'OBSERVATION_NAME', + 'OBSERVATION_UNIT', + 'OBSERVATION_VARIABLE', + 'TIME_UNIT' + ] ].drop_duplicates() biomarker_types = {} for i, row in bts_unique.iterrows(): unit = Unit.objects.get(symbol=row['OBSERVATION_UNIT']) observation_name = row['OBSERVATION_NAME'] + observation_variable = row['OBSERVATION_VARIABLE'] biomarker_types[observation_name] = BiomarkerType.objects.create( name=observation_name, description="", @@ -75,6 +81,7 @@ def replace_data(self, data: pd.DataFrame): display_time_unit=time_unit, dataset=self, color=i, + mapped_qname=observation_variable ) # create subjects diff --git a/pkpdapp/pkpdapp/utils/data_parser.py b/pkpdapp/pkpdapp/utils/data_parser.py index 4f9ff5cf..c41b4320 100644 --- a/pkpdapp/pkpdapp/utils/data_parser.py +++ b/pkpdapp/pkpdapp/utils/data_parser.py @@ -43,6 +43,9 @@ class DataParser: "OBSERVATION_UNIT", "OBSERVATIONUNIT" ], + "OBSERVATION_VARIABLE": [ + "Observbation Variable", "OBSERVATION_VARIABLE" + ], "COMPOUND": [ "Compound", "COMPOUND" ], @@ -67,6 +70,7 @@ class DataParser: "AMOUNT_UNIT", "OBSERVATION_UNIT", "OBSERVATION_NAME", + "OBSERVATION_VARIABLE", "COMPOUND", "ROUTE", "INFUSION_TIME", @@ -157,6 +161,10 @@ def map_unit_names(x): if "OBSERVATION_NAME" not in found_cols: data["OBSERVATION_NAME"] = "observation" + # put in blank observation variable if not present + if "OBSERVATION_VARIABLE" not in found_cols: + data["OBSERVATION_VARIABLE"] = "" + # put in default compound name if not present if "COMPOUND" not in found_cols: data["COMPOUND"] = "unknown compound" diff --git a/pkpdapp/schema.yml b/pkpdapp/schema.yml index 1995becf..b9d68dd9 100644 --- a/pkpdapp/schema.yml +++ b/pkpdapp/schema.yml @@ -693,6 +693,12 @@ paths: /api/dataset/: get: operationId: dataset_list + parameters: + - in: query + name: project_id + schema: + type: integer + description: Filter results by project ID tags: - dataset security: @@ -845,6 +851,12 @@ paths: - dataset requestBody: content: + application/json: + schema: + $ref: '#/components/schemas/DatasetCsv' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/DatasetCsv' multipart/form-data: schema: $ref: '#/components/schemas/DatasetCsv' @@ -3126,6 +3138,10 @@ components: axis: type: boolean description: True/False if biomarker type displayed on LHS/RHS axis + mapped_qname: + type: string + description: qname of the mapped model variable + maxLength: 50 stored_unit: type: integer description: unit for the value stored in :model:`pkpdapp.Biomarker` @@ -3415,7 +3431,6 @@ components: properties: csv: type: string - format: uri required: - csv DerivedVariable: @@ -3605,8 +3620,6 @@ components: description: If executing, this is the celery task id maxLength: 40 metadata: - type: object - additionalProperties: {} description: metadata for inference project: type: integer @@ -3865,6 +3878,10 @@ components: axis: type: boolean description: True/False if biomarker type displayed on LHS/RHS axis + mapped_qname: + type: string + description: qname of the mapped model variable + maxLength: 50 stored_unit: type: integer description: unit for the value stored in :model:`pkpdapp.Biomarker` @@ -4174,8 +4191,6 @@ components: description: If executing, this is the celery task id maxLength: 40 metadata: - type: object - additionalProperties: {} description: metadata for inference project: type: integer