add data upload, map headers and set time unit

Fix type errors for data sub-page tabs

Case insensitive header normalisation

Trim blank lines from CSV

Add Administration ID to the CSV data parser

Allow time_unit as a header

frontend-v2/package.json

@@ -23,6 +23,7 @@
     "postcss": "^8.4.32",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
+    "react-dropzone": "^14.2.3",
     "react-hook-form": "^7.43.9",
     "react-player": "^2.13.0",
     "react-plotly.js": "^2.6.0",

frontend-v2/src/features/data/Data.tsx

@@ -0,0 +1,26 @@
+import { DynamicTabs, TabPanel } from "../../components/DynamicTabs";
+import LoadDataTab from "./LoadDataTab";
+import { SubPageName } from "../main/mainSlice";
+
+const Data: React.FC = () => {
+  const tabKeys = [
+    SubPageName.LOAD_DATA,
+    SubPageName.STRATIFICATION,
+    SubPageName.VISUALISATION
+  ];
+  return (
+    <DynamicTabs tabNames={tabKeys}>
+      <TabPanel>
+        <LoadDataTab  />
+      </TabPanel>
+      <TabPanel>
+        <LoadDataTab  />
+      </TabPanel>
+      <TabPanel>
+        <LoadDataTab  />
+      </TabPanel>
+    </DynamicTabs>
+  );
+}
+
+export default Data;

frontend-v2/src/features/data/LoadData.tsx

@@ -0,0 +1,109 @@
+import { Alert, Box, Stack, Table, TableBody, TableCell, TableContainer, TableHead, TableRow } from '@mui/material';
+import Papa, { ParseError, ParseMeta } from 'papaparse'
+import React, {useCallback, useState} from 'react'
+import {useDropzone} from 'react-dropzone'
+import MapHeaders from './MapHeaders';
+import { manditoryHeaders, normaliseHeader, normalisedHeaders } from './normaliseDataHeaders';
+import { StepperState } from './LoadDataStepper';
+
+export type Row = {[key: string]: string};
+export type Data = Row[];
+export type Field = string;
+
+
+const style = {
+  dropArea: {
+    width: "100%",
+    height: "150px",
+    border: "2px dashed #000",
+    marginBottom: "10px",
+    display: "flex",
+    justifyContent: "center",
+    alignItems: "center",
+    cursor: "pointer",
+    ':hover': {
+      backgroundColor: "#f0f0f0"
+    }
+  },
+  dropAreaContainer: {
+    display: "flex",
+    justifyContent: "center",
+    alignItems: "center",
+  }
+};
+
+interface ILoadDataProps {
+  state: StepperState;
+  firstTime: boolean;
+}
+
+const validateNormalisedFields = (fields: Field[]) => {
+  const errors: string[] = [];
+  // check for mandatory fields
+  for (const field of manditoryHeaders) {
+    if (!fields.includes(field)) {
+      errors.push(`${field} has not been defined`);
+    }
+  }
+  return errors;
+}
+
+const LoadData: React.FC<ILoadDataProps> = ({state, firstTime}) => {
+  const [errors, setErrors] = useState<string[]>(firstTime ? [] : validateNormalisedFields(state.normalisedFields));
+  const [showData, setShowData] = useState<boolean>(state.data.length > 0 && state.fields.length > 0);
+
+  const onDrop = useCallback((acceptedFiles: File[]) => {
+    acceptedFiles.forEach((file) => {
+      const reader = new FileReader()
+
+      reader.onabort = () => setErrors(['file reading was aborted'])
+      reader.onerror = () => setErrors(['file reading has failed'])
+      reader.onload = () => {
+        // Parse the CSV data
+        const rawCsv = reader.result as string;
+        const csvData = Papa.parse(rawCsv.trim(), { header: true });
+        const data = csvData.data as Data;
+        const fields = csvData.meta.fields || [];
+        const normalisedFields = fields.map(normaliseHeader);
+        const errors = csvData.errors.map((e) => e.message).concat(validateNormalisedFields(normalisedFields));
+        state.setData(csvData.data as Data);
+        state.setFields(fields);
+        state.setNormalisedFields(normalisedFields)
+        setErrors(errors);
+        if (csvData.data.length > 0 && csvData.meta.fields) {
+          setShowData(true);
+        }
+      }
+      reader.readAsText(file)
+    })
+
+  }, [])
+  const {getRootProps, getInputProps} = useDropzone({onDrop})
+
+  const setNormalisedFields = (fields: Field[]) => {
+    state.setNormalisedFields(fields);
+    setErrors(validateNormalisedFields(fields));
+  }
+
+
+  return (
+    <Stack spacing={2}>
+      <div style={style.dropAreaContainer}>
+        <div {...getRootProps({style: style.dropArea})}>
+          <input {...getInputProps()} />
+          <p>Drag 'n' drop some files here, or click to select files</p>
+        </div>
+      </div>
+      <Box sx={{ width: '100%', maxHeight: "20vh", overflow: 'auto', whiteSpace: 'nowrap'}}>
+        {errors.map((error, index) => (
+          <Alert severity="error" key={index}>{error}</Alert>
+        ))}
+      </Box>
+      <Box component="div" sx={{ maxHeight: "40vh", overflow: 'auto', overflowX: 'auto' }}>
+        {showData && <MapHeaders data={state.data} fields={state.fields} setNormalisedFields={setNormalisedFields} normalisedFields={state.normalisedFields}/>}
+      </Box>
+    </Stack>
+  )
+}
+
+export default LoadData;

frontend-v2/src/features/data/LoadDataStepper.tsx

@@ -0,0 +1,91 @@
+import * as React from 'react';
+import Box from '@mui/material/Box';
+import Stepper from '@mui/material/Stepper';
+import Step from '@mui/material/Step';
+import StepLabel from '@mui/material/StepLabel';
+import Button from '@mui/material/Button';
+import Typography from '@mui/material/Typography';
+import LoadData from './LoadData';
+import { useState } from 'react';
+import MapObservations from './MapObservations';
+
+const stepLabels = ['Upload Data', 'Map Observations'];
+const stepComponents = [LoadData, MapObservations];
+
+type Row = {[key: string]: string};
+type Data = Row[];
+type Field = string;
+
+export type StepperState = {
+  fields: Field[];
+  normalisedFields: Field[];
+  data: Data;
+  timeUnit?: string;
+  setTimeUnit: (timeUnit: string) => void;
+  setFields: (fields: Field[]) => void;
+  setNormalisedFields: (fields: Field[]) => void;
+  setData: (data: Data) => void;
+  amountUnit?: string;
+  setAmountUnit: (amountUnit: string) => void;
+  observationUnits?: {[key: string]: string};
+  setObservationUnits: (observationUnits: {[key: string]: string}) => void;
+}
+
+const LoadDataStepper: React.FC = () => {
+  const [data, setData] = useState<Data>([]);
+  const [fields, setFields] = useState<string[]>([]);
+  const [normalisedFields, setNormalisedFields] = useState<string[]>([]);
+  const [timeUnit, setTimeUnit] = useState<string | undefined>(undefined);
+  const [amountUnit, setAmountUnit] = useState<string | undefined>(undefined);
+  const [observationUnits, setObservationUnits] = useState<{[key: string]: string}>({});
+
+  const state = {
+    fields,
+    normalisedFields,
+    data,
+    setFields,
+    setNormalisedFields,
+    setData,
+    timeUnit,
+    setTimeUnit,
+    amountUnit,
+    setAmountUnit,
+    observationUnits,
+    setObservationUnits,
+  };
+
+  const [stepState, setStepState] = useState({ activeStep: 0, maxStep: 0 });
+  const StepComponent = stepComponents[stepState.activeStep];
+
+  const handleNext = () => {
+    setStepState((prevActiveStep) => ({ 
+      activeStep: prevActiveStep.activeStep + 1, 
+      maxStep: Math.max(prevActiveStep.maxStep, prevActiveStep.activeStep + 1)
+    }));
+  };
+
+  const handleBack = () => {
+    setStepState((prevActiveStep) => ({ ...prevActiveStep, activeStep: prevActiveStep.activeStep - 1 }));
+  };
+
+  return (
+    <Box sx={{ width: '100%' }}>
+      <Stepper activeStep={stepState.activeStep} alternativeLabel>
+        {stepLabels.map((step, index) => (
+          <Step key={index}>
+            <StepLabel>{step}</StepLabel>
+          </Step>
+        ))}
+      </Stepper>
+      <Typography>{stepState.activeStep === stepLabels.length ? 'The process is completed' : <StepComponent state={state} firstTime={stepState.activeStep === stepState.maxStep}/>}</Typography>
+      <Box sx={{ display: 'flex', justifyContent: 'space-between', marginTop: 1 }}>
+        <Button disabled={stepState.activeStep === 0} onClick={handleBack}>Back</Button>
+        <Button variant="contained" color="primary" onClick={handleNext}>
+          {stepState.activeStep === stepLabels.length - 1 ? 'Finish' : 'Next'}
+        </Button>
+      </Box>
+    </Box>
+  );
+}
+
+export default LoadDataStepper;

frontend-v2/src/features/data/LoadDataTab.tsx

@@ -0,0 +1,62 @@
+import * as React from 'react';
+import Button from '@mui/material/Button';
+import Avatar from '@mui/material/Avatar';
+import List from '@mui/material/List';
+import ListItem from '@mui/material/ListItem';
+import ListItemAvatar from '@mui/material/ListItemAvatar';
+import ListItemButton from '@mui/material/ListItemButton';
+import ListItemText from '@mui/material/ListItemText';
+import DialogTitle from '@mui/material/DialogTitle';
+import Dialog from '@mui/material/Dialog';
+import PersonIcon from '@mui/icons-material/Person';
+import AddIcon from '@mui/icons-material/Add';
+import Typography from '@mui/material/Typography';
+import { blue } from '@mui/material/colors';
+import LoadDataStepper from './LoadDataStepper';
+import { DialogContent } from '@mui/material';
+
+export interface LoadDataDialogProps {
+  open: boolean;
+  onClose: () => void;
+}
+
+function LoadDataDialog(props: LoadDataDialogProps) {
+  const { onClose, open } = props;
+
+  const handleClose = () => {
+    onClose();
+  };
+
+  return (
+    <Dialog onClose={handleClose} open={open} maxWidth='lg' fullWidth>
+      <DialogTitle>Upload New Dataset</DialogTitle>
+      <DialogContent>
+        <LoadDataStepper /> 
+      </DialogContent>
+    </Dialog>
+  );
+}
+
+export default function LoadDataTab() {
+  const [open, setOpen] = React.useState(false);
+
+  const handleClickOpen = () => {
+    setOpen(true);
+  };
+
+  const handleClose = () => {
+    setOpen(false);
+  };
+
+  return (
+    <div>
+      <Button variant="outlined" onClick={handleClickOpen}>
+        Upload New Dataset
+      </Button>
+      <LoadDataDialog
+        open={open}
+        onClose={handleClose}
+      />
+    </div>
+  );
+}

frontend-v2/src/features/data/MapHeaders.tsx

@@ -0,0 +1,62 @@
+import { TableContainer, Table, TableHead, TableRow, TableCell, TableBody, Select, FormControl, MenuItem, InputLabel, Typography } from "@mui/material";
+import LoadData, { Data, Field } from "./LoadData";
+import { normalisedHeaders } from "./normaliseDataHeaders";
+
+interface IMapHeaders {
+  data: Data;
+  fields: Field[];
+  normalisedFields: Field[];
+  setNormalisedFields: (fields: Field[]) => void;
+}
+
+const MapHeaders: React.FC<IMapHeaders> = ({data, fields, normalisedFields, setNormalisedFields}: IMapHeaders) => {
+
+  const normalisedHeadersOptions = normalisedHeaders.map((header) => ({value: header, label: header}));
+
+  const handleFieldChange = (index: number) => (event: any) => {
+    const newFields = [...normalisedFields];
+    newFields[index] = event.target.value;
+    setNormalisedFields(newFields);
+  }
+
+  return (
+    <Table>
+      <TableHead>
+        <TableRow>
+          {fields.map((field, index) => (
+            <TableCell key={index}>
+              <Typography variant="h6" component="div" sx={{ flexGrow: 1, marginBottom: 1 }} align="center">
+                {field}
+              </Typography>
+              <FormControl fullWidth>
+                <InputLabel id={`select-${index}-label`}>Column Type</InputLabel>
+                <Select
+                  labelId={`select-${index}-label`}
+                  id={`select-${index}`}
+                  value={normalisedFields[index]}
+                  label="Column Type"
+                  onChange={handleFieldChange(index)}
+                >
+                  {normalisedHeadersOptions.map((option) => (
+                    <MenuItem key={option.value} value={option.value}>{option.label}</MenuItem>
+                  ))}
+                </Select>
+              </FormControl>
+            </TableCell>
+          ))}
+        </TableRow>
+      </TableHead>
+      <TableBody>
+        {data.map((row, index) => (
+          <TableRow key={index}>
+            {fields.map((field, index) => (
+              <TableCell key={index}>{row[field]}</TableCell>
+            ))}
+          </TableRow>
+        ))}
+      </TableBody>
+    </Table>
+  )
+}
+
+export default MapHeaders;

frontend-v2/src/features/data/MapObservations.tsx

@@ -0,0 +1,15 @@
+import { Field, Data } from "./LoadData";
+import { StepperState } from "./LoadDataStepper";
+
+interface IMapObservations {
+  state: StepperState;
+  firstTime: boolean;
+}
+
+const MapObservations: React.FC<IMapObservations> = ({state, firstTime}: IMapObservations) => {
+  return (null)
+}
+
+export default MapObservations;
+
+