continuing full system updates. reworking validation system to operat…

…e by retrieving full coremeasurement row inserted (to get plot/quadrat/personnel info) instead of using user-provided data. Because some of the validations use plot/quadrat, etc., when those validations fail the user-provided data will not provide info about it.
Smithsonian · Feb 27, 2024 · 7288db0 · 7288db0
1 parent 7dfbb8b
commit 7288db0
Show file tree

Hide file tree

Showing 11 changed files with 190 additions and 134 deletions.
diff --git a/.github/workflows/new-file-upload-system_forestgeo-livesite.yml b/.github/workflows/new-file-upload-system_forestgeo-livesite.yml
@@ -40,13 +40,13 @@ jobs:
           echo NODE_ENV=development >> frontend/.env
           echo PORT=3000 >> frontend/.env
 
-#      - name: Cache node modules
-#        uses: actions/cache@v2
-#        with:
-#          path: frontend/node_modules
-#          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-#          restore-keys: |
-#            ${{ runner.os }}-node-
+      - name: Cache node modules
+        uses: actions/cache@v2
+        with:
+          path: frontend/node_modules
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
 
       - name: npm install, build, and test
         run: |

diff --git a/frontend/app/api/singlecmid/route.ts b/frontend/app/api/singlecmid/route.ts
@@ -0,0 +1,40 @@
+import {NextRequest, NextResponse} from "next/server";
+import {CoreMeasurementsResult, getConn, getSchema, runQuery} from "@/components/processors/processormacros";
+import {PoolConnection} from "mysql2/promise";
+import {CoreMeasurementsRDS} from "@/config/sqlmacros";
+import {bitToBoolean} from "@/config/macros";
+
+export async function GET(request: NextRequest) {
+  const schema = getSchema();
+  const cmID = parseInt(request.nextUrl.searchParams.get('cmid')!);
+  let conn: PoolConnection | null = null;
+  try {
+    conn = await getConn();
+    let query = `SELECT * FROM ${schema}.CoreMeasurements WHERE CoreMeasurementID = ? LIMIT 1`;
+    const results = await runQuery(conn, query, [cmID]);
+    let coreMeasurementRows: CoreMeasurementsRDS[] = results.map((row: CoreMeasurementsResult, index: number) => ({
+      // ... mapping fields ...
+      id: index + 1,
+      coreMeasurementID: row.CoreMeasurementID,
+      censusID: row.CensusID,
+      plotID: row.PlotID,
+      quadratID: row.QuadratID,
+      treeID: row.TreeID,
+      stemID: row.StemID,
+      personnelID: row.PersonnelID,
+      isValidated: bitToBoolean(row.IsValidated),
+      measurementDate: row.MeasurementDate,
+      measuredDBH: row.MeasuredDBH,
+      measuredHOM: row.MeasuredHOM,
+      description: row.Description,
+      userDefinedFields: row.UserDefinedFields,
+      // ... other fields as needed
+    }));
+
+    return new NextResponse(JSON.stringify(coreMeasurementRows), {status: 200});
+  } catch(error: any) {
+    throw new Error('SQL query failed: ' + error.message);
+  } finally {
+    if (conn) conn.release();
+  }
+}
diff --git a/frontend/app/api/validations/validationerrordisplay/route.ts b/frontend/app/api/validations/validationerrordisplay/route.ts
@@ -1,8 +1,7 @@
 import {NextResponse} from "next/server";
-import {getConn, getSchema, PersonnelResult, runQuery} from "@/components/processors/processormacros";
+import {getConn, getSchema, runQuery} from "@/components/processors/processormacros";
 import {PoolConnection} from "mysql2/promise";
 import {CMError} from "@/config/macros";
-import {PersonnelRDS} from "@/config/sqlmacros";
 export async function GET() {
   let conn: PoolConnection | null = null;
 

diff --git a/frontend/components/client/clientmacros.tsx b/frontend/components/client/clientmacros.tsx
@@ -5,9 +5,8 @@ import {CensusRDS, PlotRDS, QuadratsRDS} from "@/config/sqlmacros";
 import {Census, Plot, Quadrat} from "@/config/macros";
 import {useCensusLoadDispatch, usePlotsLoadDispatch, useQuadratsLoadDispatch} from "@/app/contexts/coredataprovider";
 import {useCensusListDispatch, usePlotListDispatch, useQuadratListDispatch} from "@/app/contexts/listselectionprovider";
-import {Box, Typography} from "@mui/material";
+import {Box, LinearProgress, LinearProgressProps, Typography} from "@mui/material";
 import React from "react";
-import { LinearProgress, LinearProgressProps } from "@mui/joy";
 
 async function updateQuadratsIDB() {
   const quadratRDSResponse = await fetch(`/api/fetchall/quadrats`, {method: 'GET'});
@@ -107,11 +106,11 @@ export async function loadServerDataIntoIDB(dataType: string) {
 export function LinearProgressWithLabel(props: LinearProgressProps & { value?: number, currentlyrunningmsg?: string }) {
   return (
     <Box sx={{display: 'flex', flex: 1, alignItems: 'center', flexDirection: 'column'}}>
-      <Box sx={{ mr: 1}}>
+      <Box sx={{width: '100%', mr: 1}}>
         {props.value ? (
-          <LinearProgress size={"lg"} determinate {...props} />
+          <LinearProgress variant="determinate" {...props} />
         ) : (
-          <LinearProgress size={"lg"} {...props} />
+          <LinearProgress variant={"indeterminate"} {...props} />
         )}
       </Box>
       <Box sx={{minWidth: 35, display: 'flex', flex: 1, flexDirection: 'column'}}>

diff --git a/frontend/components/sidebar.tsx b/frontend/components/sidebar.tsx
@@ -207,7 +207,7 @@ export default function Sidebar() {
           })}
         />
         <Box sx={{display: 'flex', gap: 1, alignItems: 'left'}}>
-          <Typography level="h1">ForestGEO: <Typography className={"text-teal-400"}>C. & S. America</Typography></Typography>
+          <Typography level="h1">ForestGEO: <Typography className={"text-teal-400"}>Americas</Typography></Typography>
         </Box>
         <Divider orientation={"horizontal"}/>
         <Box
@@ -296,9 +296,10 @@ export default function Sidebar() {
               </Link>
             </Breadcrumbs>
             <Divider orientation={"horizontal"}/>
-            <Button variant={"soft"} sx={{width: 'fit-content'}} onClick={restorePlotCensus} color={"primary"} title={"Restore Plot and Census"}>
+            <Button variant={"soft"} sx={{width: 'fit-content', marginBottom: 1}} onClick={restorePlotCensus} color={"primary"} title={"Restore Plot and Census"}>
               Restore Plot and Census
             </Button>
+            <Divider orientation={"horizontal"} />
             <Modal open={openPlotSelectionModal} onClose={() => {
               setPlot(currentPlot);
               setOpenPlotSelectionModal(false);
@@ -413,7 +414,6 @@ export default function Sidebar() {
               </ModalDialog>
             </Modal>
           </List>
-          <Divider orientation={"horizontal"}/>
         </Box>
         <Divider/>
         <LoginLogout/>

diff --git a/frontend/components/uploadsystem/uploadfire.tsx b/frontend/components/uploadsystem/uploadfire.tsx
@@ -148,7 +148,7 @@ const UploadFire: React.FC<UploadFireProps> = ({
         <Box sx={{display: 'flex', flex: 1, width: '100%', alignItems: 'center', mt: 4}}>
           <Stack direction={"column"}>
             <Typography variant="h6" gutterBottom>{`Total Operations: ${totalOperations}`}</Typography>
-            <LinearProgressWithLabel determinate value={(completedOperations / totalOperations) * 100}
+            <LinearProgressWithLabel variant={"determinate"} value={(completedOperations / totalOperations) * 100}
                                      currentlyrunningmsg={currentlyRunning}/>
           </Stack>
         </Box>

diff --git a/frontend/components/uploadsystem/uploadparent.tsx b/frontend/components/uploadsystem/uploadparent.tsx
@@ -483,7 +483,8 @@ export default function UploadParent() {
       case ReviewStates.VALIDATE_ERRORS_FOUND:
         return <UploadValidationErrorDisplay
           allRowToCMID={allRowToCMID}
-          setReviewState={setReviewState}/>;
+          setReviewState={setReviewState}
+          uploadForm={uploadForm}/>;
       case ReviewStates.UPDATE:
         return <UploadUpdateValidations
           setReviewState={setReviewState}

diff --git a/frontend/components/uploadsystem/uploadvalidation.tsx b/frontend/components/uploadsystem/uploadvalidation.tsx
@@ -36,8 +36,8 @@ const UploadValidation: React.FC<UploadValidationProps> = ({
   const [currentPromptApi, setCurrentPromptApi] = useState<string>('');
   const [tempMinMax, setTempMinMax] = useState<{ min: number | string, max: number | string }>({min: '', max: ''});
   const [validationProgress, setValidationProgress] = useState<Record<string, number>>({});
-  const [currentValidationIndex, setCurrentValidationIndex] = useState(0);
-
+  const [useDefaultValues, setUseDefaultValues] = useState<boolean>(false);
+  const [defaultValuesDialogOpen, setDefaultValuesDialogOpen] = useState<boolean>(true);
 
   const validationAPIs: string[] = [
     'dbhgrowthexceedsmax',
@@ -54,6 +54,12 @@ const UploadValidation: React.FC<UploadValidationProps> = ({
     'treestemsdiffquadrats',
   ];
 
+  const defaultMinMaxValues: Record<string, { min: number, max: number }> = {
+    'screendbhminmax': { min: 1, max: 500 },
+    'screenhomminmax': { min: 1, max: 1.5},
+    // ... [other default values]
+  };
+
   // Define the type for validation messages
   type ValidationMessages = {
     [key: string]: string;
@@ -78,10 +84,20 @@ const UploadValidation: React.FC<UploadValidationProps> = ({
   useEffect(() => {
     const initialProgress = validationAPIs.reduce((acc, api) => ({ ...acc, [api]: 0 }), {});
     setValidationProgress(initialProgress);
-    // Start with the first prompt for DBH
-    promptForInput('screendbhminmax');
   }, []);
 
+  const handleDefaultValuesSelection = (useDefaults: boolean) => {
+    setUseDefaultValues(useDefaults);
+    setDefaultValuesDialogOpen(false);
+
+    if (useDefaults) {
+      setMinMaxValues(defaultMinMaxValues);
+      showNextPrompt(0, false).catch(console.error);
+    } else {
+      promptForInput('screendbhminmax');
+    }
+  };
+
   const promptForInput = (api: string) => {
     setCurrentPromptApi(api);
     setPromptOpen(true);
@@ -100,66 +116,57 @@ const UploadValidation: React.FC<UploadValidationProps> = ({
       promptForInput('screenhomminmax');
     } else if (currentPromptApi === 'screenhomminmax') {
       // Start validations after both prompts are done
-      showNextPrompt(0);
+      showNextPrompt(0, false).catch(console.error);
     }
   };
 
-  const showNextPrompt = (index: number) => {
+  const showNextPrompt = async (index: number, foundError: boolean = false) => {
     if (index >= validationAPIs.length) {
       setIsValidationComplete(true); // All validations are complete
-      checkForErrors();
+      setErrorsFound(foundError);
+      if (foundError) {
+        setReviewState(ReviewStates.VALIDATE_ERRORS_FOUND);
+      }
       return;
     }
 
     const api = validationAPIs[index];
 
-    // Directly perform validations for screendbhminmax and screenhomminmax
-    if (api === 'screendbhminmax' || api === 'screenhomminmax') {
-      performValidation(api).then(result => {
-        setValidationResults(prevResults => ({ ...prevResults, [api]: result }));
-        setValidationProgress(prevProgress => ({ ...prevProgress, [api]: 100 }));
-        showNextPrompt(index + 1);
-      });
-    } else {
-      performValidation(api).then(result => {
-        setValidationResults(prevResults => ({ ...prevResults, [api]: result }));
-        setValidationProgress(prevProgress => ({ ...prevProgress, [api]: 100 }));
-        showNextPrompt(index + 1);
-      });
-    }
+    performValidation(api).then(({ response, hasError }) => {
+      setValidationResults(prevResults => ({...prevResults, [api]: response}));
+      setValidationProgress(prevProgress => ({ ...prevProgress, [api]: 100 }));
+      showNextPrompt(index + 1, foundError || hasError);
+    });
   };
 
-  const performValidation = async (api: string): Promise<ValidationResponse> => {
+  const performValidation = async (api: string): Promise<{ response: ValidationResponse, hasError: boolean }> => {
     let queryParams = `plotID=${currentPlot?.id}&censusID=${currentCensus?.censusID}`;
     if (['screendbhminmax', 'screenhomminmax'].includes(api)) {
-      const values = minMaxValues[api];
-      if (values) {
-        queryParams += `&minValue=${values.min}&maxValue=${values.max}`;
-      } else {
-        console.error(`Validation values for ${api} are not set`);
-        return { failedRows: 0, message: `Validation values for ${api} are not set`, totalRows: 0 };
-      }
+      const values = minMaxValues[api] || defaultMinMaxValues[api]; // Use default if not set
+      queryParams += `&minValue=${values.min}&maxValue=${values.max}`;
     }
     try {
       const response = await fetch(`/api/validations/${api}?${queryParams}`);
       if (!response.ok) {
         throw new Error(`Error executing ${api}`);
       }
       const result = await response.json();
-      setValidationProgress(prevProgress => ({ ...prevProgress, [api]: 100 }));
-      return result;
+      const hasError = result.failedRows > 0;
+      return { response: result, hasError };
     } catch (error: any) {
       console.error(`Error performing validation for ${api}:`, error);
       setApiErrors(prev => [...prev, `Failed to execute ${api}: ${error.message}`]);
       setValidationProgress(prevProgress => ({ ...prevProgress, [api]: -1 }));
-      return { failedRows: 0, message: error.message, totalRows: 0 };
+      return { response: {failedRows: 0, message: error.message, totalRows: 0 }, hasError: false};
     }
   };
 
   const checkForErrors = () => {
     const foundErrors = Object.values(validationResults).some(result => result.failedRows > 0);
+    console.log(`found errors: ${foundErrors}`);
     setErrorsFound(foundErrors);
   };
+
   const renderProgressBars = () => {
     return validationAPIs.map(api => (
       <Box key={api} sx={{mb: 2}}>
@@ -204,9 +211,27 @@ const UploadValidation: React.FC<UploadValidationProps> = ({
     );
   };
 
+  const renderDefaultValuesDialog = () => {
+    return (
+      <Dialog open={defaultValuesDialogOpen} onClose={() => handleDefaultValuesSelection(false)}>
+        <DialogTitle>Validation Parameters</DialogTitle>
+        <DialogContent>
+          <DialogContentText>
+            Would you like to use default values for validation parameters or provide them manually?
+          </DialogContentText>
+        </DialogContent>
+        <DialogActions>
+          <Button onClick={() => handleDefaultValuesSelection(true)}>Use Default Values</Button>
+          <Button onClick={() => handleDefaultValuesSelection(false)}>Manual Input</Button>
+        </DialogActions>
+      </Dialog>
+    );
+  };
+
   return (
     <Box sx={{width: '100%', p: 2, display: 'flex', flex: 1, flexDirection: 'column'}}>
-      {renderPromptModal() /* Render the modal dialog */}
+      {renderDefaultValuesDialog()}
+      {renderPromptModal()}
 
       {!isValidationComplete ? (
         <Box sx={{display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center'}}>
@@ -219,13 +244,13 @@ const UploadValidation: React.FC<UploadValidationProps> = ({
           {apiErrors.length > 0 && (
             <Box sx={{mb: 2}}>
               <Typography color="error">Some validations could not be performed:</Typography>
-              {apiErrors.map((error, index) => (
-                <Typography key={index} color="error">- {error}</Typography>
+              {apiErrors.map((error) => (
+                <Typography key={error} color="error">- {error}</Typography>
               ))}
             </Box>
           )}
-          {Object.entries(validationResults).map(([api, result], index) => (
-            <Box key={index} sx={{mb: 2}}>
+          {Object.entries(validationResults).map(([api, result]) => (
+            <Box key={api} sx={{mb: 2}}>
               <Typography>{api}:</Typography>
               {result.failedRows > 0 ? (
                 <>
@@ -238,17 +263,17 @@ const UploadValidation: React.FC<UploadValidationProps> = ({
               )}
             </Box>
           ))}
-          {errorsFound ? (
-            <Button variant="contained" color="error" sx={{mt: 2, width: 'fit-content'}}
-                    onClick={() => setReviewState(ReviewStates.VALIDATE_ERRORS_FOUND)}>
-              Review Errors
-            </Button>
-          ) : (
-            <Button variant="contained" color="success" sx={{mt: 2, width: 'fit-content'}}
-                    onClick={() => setReviewState(ReviewStates.UPDATE)}>
-              Complete Upload
-            </Button>
-          )}
+          {/*{errorsFound ? (*/}
+          {/*  <Button variant="contained" color="error" sx={{mt: 2, width: 'fit-content'}}*/}
+          {/*          onClick={() => setReviewState(ReviewStates.VALIDATE_ERRORS_FOUND)}>*/}
+          {/*    Review Errors*/}
+          {/*  </Button>*/}
+          {/*) : (*/}
+          {/*  <Button variant="contained" color="success" sx={{mt: 2, width: 'fit-content'}}*/}
+          {/*          onClick={() => setReviewState(ReviewStates.UPDATE)}>*/}
+          {/*    Complete Upload*/}
+          {/*  </Button>*/}
+          {/*)}*/}
         </Box>
       )}
     </Box>