emergency hotfix -- migration and handling system was not correctly a…

…ccounting for many-to-many relationship between quadrats and census, thus losing extensive historical data. system reworked and repaired to 1) incorporate censusquadrat table, 2) clean sql scripts and rework migration system & table structures to properly handle censusquadrat table, 3) rework all relevant API endpoints to correctly reference censusquadrat table, and 4) rework fixeddata PATCH/POST/DELETE to handle interactions with censusquadrat along with processor files

frontend/app/api/cmprevalidation/[dataType]/[[...slugs]]/route.ts

@@ -36,7 +36,7 @@ export async function GET(_request: NextRequest, { params }: { params: { dataTyp
           });
         break;
       case 'personnel':
-        const pQuery = `SELECT 1 FROM ${schema}.${params.dataType} WHERE CensusID IN (SELECT CensusID from ${schema}.census WHERE PlotID = ${plotID} AND PlotCensusNumber = ${plotCensusNumber})`; // Check if the table has any row
+        const pQuery = `SELECT 1 FROM ${schema}.personnel WHERE CensusID IN (SELECT CensusID from ${schema}.census WHERE PlotID = ${plotID} AND PlotCensusNumber = ${plotCensusNumber})`; // Check if the table has any row
         const pResults = await runQuery(connection, pQuery);
         if (connection) connection.release();
         if (pResults.length === 0)
@@ -45,33 +45,39 @@ export async function GET(_request: NextRequest, { params }: { params: { dataTyp
           });
         break;
       case 'quadrats':
-        const query = `SELECT 1 FROM ${schema}.${params.dataType} WHERE PlotID = ${plotID} AND CensusID IN (SELECT CensusID from ${schema}.census WHERE PlotID = ${plotID} AND PlotCensusNumber = ${plotCensusNumber})`; // Check if the table has any row
+        const query = `SELECT 1 FROM ${schema}.quadrats q
+         JOIN ${schema}.censusquadrat cq ON cq.QuadratID = q.QuadratID 
+         JOIN ${schema}.census c ON cq.CensusID = c.CensusID 
+         WHERE q.PlotID = ${plotID} AND c.PlotCensusNumber = ${plotCensusNumber} LIMIT 1`;
         const results = await runQuery(connection, query);
         if (connection) connection.release();
         if (results.length === 0)
           return new NextResponse(null, {
             status: HTTPResponses.PRECONDITION_VALIDATION_FAILURE
           });
         break;
-      case 'subquadrats':
-        const subquadratsQuery = `SELECT 1
-                                FROM ${schema}.${params.dataType} s
-                                JOIN ${schema}.quadrats q ON s.QuadratID = q.QuadratID
-                                WHERE q.PlotID = ${plotID}
-                                  AND q.CensusID IN (SELECT CensusID from ${schema}.census WHERE PlotID = ${plotID} AND PlotCensusNumber = ${plotCensusNumber}) LIMIT 1`;
-        const subquadratsResults = await runQuery(connection, subquadratsQuery);
-        if (connection) connection.release();
-        if (subquadratsResults.length === 0)
-          return new NextResponse(null, {
-            status: HTTPResponses.PRECONDITION_VALIDATION_FAILURE
-          });
-        break;
+      // case 'subquadrats':
+      //   const subquadratsQuery = `SELECT 1
+      //                           FROM ${schema}.${params.dataType} s
+      //                           JOIN ${schema}.quadrats q ON s.QuadratID = q.QuadratID
+      //                           WHERE q.PlotID = ${plotID}
+      //                             AND q.CensusID IN (SELECT CensusID from ${schema}.census WHERE PlotID = ${plotID} AND PlotCensusNumber = ${plotCensusNumber}) LIMIT 1`;
+      //   const subquadratsResults = await runQuery(connection, subquadratsQuery);
+      //   if (connection) connection.release();
+      //   if (subquadratsResults.length === 0)
+      //     return new NextResponse(null, {
+      //       status: HTTPResponses.PRECONDITION_VALIDATION_FAILURE
+      //     });
+      //   break;
       case 'quadratpersonnel':
         // Validation for quadrats table
         const quadratsQuery = `SELECT 1
-                             FROM ${schema}.quadrats
-                             WHERE PlotID = ${plotID}
-                               AND CensusID IN (SELECT CensusID from ${schema}.census WHERE PlotID = ${plotID} AND PlotCensusNumber = ${plotCensusNumber}) LIMIT 1`;
+                             FROM ${schema}.quadrats q
+                             JOIN ${schema}.censusquadrat cq on cq.QuadratID = q.QuadratID
+                             JOIN ${schema}.census c on cq.CensusID = c.CensusID
+                             JOIN ${schema}.personnel p ON p.CensusID = c.CensusID
+                             WHERE q.PlotID = ${plotID}
+                               AND c.PlotCensusNumber = ${plotCensusNumber} LIMIT 1`;
         const quadratsResults = await runQuery(connection, quadratsQuery);
         if (connection) connection.release();
         if (quadratsResults.length === 0)

frontend/app/api/fetchall/[[...slugs]]/route.ts

@@ -17,6 +17,12 @@ const buildQuery = (schema: string, fetchType: string, plotID?: string, plotCens
   } else if (fetchType === 'roles') {
     return `SELECT *
                  FROM ${schema}.${fetchType}`;
+  } else if (fetchType === 'quadrats') {
+    return `
+      SELECT * FROM ${schema}.quadrats q
+        JOIN ${schema}.censusquadrat cq ON cq.QuadratID = q.QuadratID
+        JOIN ${schema}.census c ON cq.CensusID = c.CensusID
+        WHERE q.PlotID = ${plotID} AND c.PlotID = ${plotID} AND c.PlotCensusNumber = ${plotCensusNumber}`;
   } else {
     let query = `SELECT *
                  FROM ${schema}.${fetchType}`;

frontend/app/api/fixeddata/[dataType]/[[...slugs]]/route.ts

@@ -74,8 +74,9 @@ export async function GET(
       case 'quadrats':
         paginatedQuery = `
             SELECT SQL_CALC_FOUND_ROWS q.*
-            FROM ${schema}.${params.dataType} q
-                     JOIN ${schema}.census c ON q.PlotID = c.PlotID AND q.CensusID = c.CensusID
+            FROM ${schema}.quadrats q
+                     JOIN ${schema}.censusquadrat cq ON q.QuadratID = cq.QuadratID
+                     JOIN ${schema}.census c ON cq.CensusID = c.CensusID
             WHERE q.PlotID = ?
               AND c.PlotID = ?
               AND c.PlotCensusNumber = ? LIMIT ?, ?;`;
@@ -112,21 +113,21 @@ export async function GET(
             ORDER BY q.MeasurementDate ASC LIMIT ?, ?;`;
         queryParams.push(plotID, plotID, plotCensusNumber, page * pageSize, pageSize);
         break;
-      case 'subquadrats':
-        if (!quadratID || quadratID === 0) {
-          throw new Error('QuadratID must be provided as part of slug fetch query, referenced fixeddata slug route');
-        }
-        paginatedQuery = `
-            SELECT SQL_CALC_FOUND_ROWS s.*
-            FROM ${schema}.subquadrats s
-                     JOIN ${schema}.quadrats q ON s.QuadratID = q.QuadratID
-                     JOIN ${schema}.census c ON q.CensusID = c.CensusID
-            WHERE q.QuadratID = ?
-              AND q.PlotID = ?
-              AND c.PlotID = ?
-              AND c.PlotCensusNumber = ? LIMIT ?, ?;`;
-        queryParams.push(quadratID, plotID, plotID, plotCensusNumber, page * pageSize, pageSize);
-        break;
+      // case 'subquadrats':
+      //   if (!quadratID || quadratID === 0) {
+      //     throw new Error('QuadratID must be provided as part of slug fetch query, referenced fixeddata slug route');
+      //   }
+      //   paginatedQuery = `
+      //       SELECT SQL_CALC_FOUND_ROWS s.*
+      //       FROM ${schema}.subquadrats s
+      //                JOIN ${schema}.quadrats q ON s.QuadratID = q.QuadratID
+      //                JOIN ${schema}.census c ON q.CensusID = c.CensusID
+      //       WHERE q.QuadratID = ?
+      //         AND q.PlotID = ?
+      //         AND c.PlotID = ?
+      //         AND c.PlotCensusNumber = ? LIMIT ?, ?;`;
+      //   queryParams.push(quadratID, plotID, plotID, plotCensusNumber, page * pageSize, pageSize);
+      //   break;
       case 'census':
         paginatedQuery = `
             SELECT SQL_CALC_FOUND_ROWS *
@@ -221,9 +222,12 @@ export async function GET(
 // required dynamic parameters: dataType (fixed),[ schema, gridID value] -> slugs
 export async function POST(request: NextRequest, { params }: { params: { dataType: string; slugs?: string[] } }) {
   if (!params.slugs) throw new Error('slugs not provided');
-  const [schema, gridID] = params.slugs;
+  const [schema, gridID, plotIDParam, censusIDParam] = params.slugs;
   if (!schema || !gridID) throw new Error('no schema or gridID provided');
 
+  const plotID = plotIDParam ? parseInt(plotIDParam) : undefined;
+  const censusID = censusIDParam ? parseInt(censusIDParam) : undefined;
+
   let conn: PoolConnection | null = null;
   const { newRow } = await request.json();
   let insertIDs: { [key: string]: number } = {};
@@ -254,21 +258,26 @@ export async function POST(request: NextRequest, { params }: { params: { dataTyp
       // Use handleUpsertForSlices and retrieve the insert IDs
       insertIDs = await handleUpsertForSlices(conn, schema, newRowData, queryConfig);
     }
-
     // Handle the case for 'attributes'
     else if (params.dataType === 'attributes') {
       const insertQuery = format('INSERT INTO ?? SET ?', [`${schema}.${params.dataType}`, newRowData]);
       const results = await runQuery(conn, insertQuery);
       insertIDs = { attributes: results.insertId }; // Standardize output with table name as key
     }
-
     // Handle all other cases
     else {
       delete newRowData[demappedGridID];
       if (params.dataType === 'plots') delete newRowData.NumQuadrats;
       const insertQuery = format('INSERT INTO ?? SET ?', [`${schema}.${params.dataType}`, newRowData]);
       const results = await runQuery(conn, insertQuery);
       insertIDs = { [params.dataType]: results.insertId }; // Standardize output with table name as key
+
+      // special handling needed for quadrats --> need to correlate incoming quadrats with current census
+      if (params.dataType === 'quadrats' && censusID) {
+        const cqQuery = format('INSERT INTO ?? SET ?', [`${schema}.censusquadrats`, { CensusID: censusID, QuadratID: insertIDs.quadrats }]);
+        const results = await runQuery(conn, cqQuery);
+        if (results.length === 0) throw new Error('Error inserting to censusquadrats');
+      }
     }
 
     // Commit the transaction and return the standardized response
@@ -387,6 +396,11 @@ export async function DELETE(request: NextRequest, { params }: { params: { dataT
     // Handle deletion for tables
     const deleteRowData = MapperFactory.getMapper<any, any>(params.dataType).demapData([newRow])[0];
     const { [demappedGridID]: gridIDKey } = deleteRowData;
+    // for quadrats, censusquadrat needs to be cleared before quadrat can be deleted
+    if (params.dataType === 'quadrats') {
+      const qDeleteQuery = format(`DELETE FROM ?? WHERE ?? = ?`, [`${schema}.censusquadrat`, demappedGridID, gridIDKey]);
+      await runQuery(conn, qDeleteQuery);
+    }
     const deleteQuery = format(`DELETE FROM ?? WHERE ?? = ?`, [`${schema}.${params.dataType}`, demappedGridID, gridIDKey]);
     await runQuery(conn, deleteQuery);
     await conn.commit();

frontend/app/api/rollover/[dataType]/[[...slugs]]/route.ts

@@ -32,23 +32,11 @@ export async function POST(request: NextRequest, { params }: { params: { dataTyp
     switch (params.dataType) {
       case 'quadrats':
         query = `
-          INSERT INTO ${schema}.quadrats (PlotID, CensusID, QuadratName, StartX, StartY, CoordinateUnits, DimensionX, DimensionY, DimensionUnits, Area, AreaUnits, QuadratShape)
-          SELECT 
-              PlotID, 
-              ?, 
-              QuadratName, 
-              StartX, 
-              StartY, 
-              CoordinateUnits, 
-              DimensionX, 
-              DimensionY, 
-              DimensionUnits, 
-              Area, 
-              AreaUnits, 
-              QuadratShape
-          FROM ${schema}.quadrats
-          WHERE CensusID = ? AND QuadratID IN (${incoming.map(() => '?').join(', ')});`;
-        queryParams = [Number(newCensusID), Number(sourceCensusID), ...incoming];
+          INSERT INTO censusquadrat (CensusID, QuadratID)
+          SELECT ?, q.QuadratID
+          FROM quadrats q
+          WHERE q.QuadratID IN (${incoming.map(() => '?').join(', ')});`;
+        queryParams = [Number(newCensusID), ...incoming];
         await runQuery(conn, query, queryParams);
         break;
       case 'personnel':

frontend/components/datagrids/applications/isolated/isolatedquadratsdatagrid.tsx

@@ -18,7 +18,6 @@ export default function IsolatedQuadratsDataGrid() {
     id: 0,
     quadratID: 0,
     plotID: currentPlot?.plotID,
-    censusID: currentCensus?.dateRanges[0].censusID,
     quadratName: '',
     startX: 0,
     startY: 0,

frontend/components/datagrids/applications/quadratsdatagrid.tsx

@@ -19,7 +19,6 @@ export default function QuadratsDataGrid() {
     id: 0,
     quadratID: 0,
     plotID: 0,
-    censusID: 0,
     quadratName: '',
     startX: 0,
     startY: 0,

frontend/components/datagrids/isolateddatagridcommons.tsx

@@ -507,7 +507,10 @@ export default function IsolatedDataGridCommons(props: Readonly<IsolatedDataGrid
     paginationModel: { page: number }
   ): Promise<GridRowModel> => {
     const gridID = getGridID(gridType);
-    const fetchProcessQuery = createPostPatchQuery(schemaName ?? '', gridType, gridID);
+    const fetchProcessQuery =
+      gridType !== 'quadrats'
+        ? createPostPatchQuery(schemaName ?? '', gridType, gridID)
+        : createPostPatchQuery(schemaName ?? '', gridType, gridID, currentPlot?.plotID, currentCensus?.dateRanges[0].censusID);
 
     try {
       const response = await fetch(fetchProcessQuery, {

frontend/components/processors/processcensus.tsx

@@ -21,13 +21,7 @@ export async function processCensus(props: Readonly<SpecialProcessingProps>): Pr
     const speciesID = await fetchPrimaryKey<SpeciesResult>(schema, 'species', { SpeciesCode: spcode }, connection, 'SpeciesID');
 
     // Fetch quadrat
-    const quadratID = await fetchPrimaryKey<QuadratsResult>(
-      schema,
-      'quadrats',
-      { QuadratName: quadrat, PlotID: plotID, CensusID: censusID },
-      connection,
-      'QuadratID'
-    );
+    const quadratID = await fetchPrimaryKey<QuadratsResult>(schema, 'quadrats', { QuadratName: quadrat, PlotID: plotID }, connection, 'QuadratID');
 
     if (tag) {
       // Handle Tree Upsert

frontend/components/processors/processormacros.tsx

@@ -64,7 +64,6 @@ export const fileMappings: Record<string, FileMapping> = {
     columnMappings: {
       quadrat: 'QuadratName',
       plotID: 'PlotID',
-      censusID: 'CensusID',
       startx: 'StartX',
       starty: 'StartY',
       coordinateunit: 'CoordinateUnits',

frontend/config/datagridhelpers.ts

@@ -43,7 +43,9 @@ export type ProcessPostPatchQueryFunction = (
   // incorporated validation system into this too
   siteSchema: string,
   dataType: string,
-  gridID: string
+  gridID: string,
+  plotID?: number,
+  censusID?: number
 ) => string;
 export type ProcessDeletionQueryFunction = (siteSchema: string, dataType: string, gridID: string, deletionID: number | string) => string;
 
@@ -93,8 +95,14 @@ const columnVisibilityMap: { [key: string]: { [key: string]: boolean } } = {
 export const getColumnVisibilityModel = (gridType: string): { [key: string]: boolean } => {
   return columnVisibilityMap[gridType] || columnVisibilityMap.default;
 };
-export const createPostPatchQuery: ProcessPostPatchQueryFunction = (siteSchema: string, dataType: string, gridID: string) => {
-  return `/api/fixeddata/${dataType}/${siteSchema}/${gridID}`;
+export const createPostPatchQuery: ProcessPostPatchQueryFunction = (
+  siteSchema: string,
+  dataType: string,
+  gridID: string,
+  plotID?: number,
+  censusID?: number
+) => {
+  return `/api/fixeddata/${dataType}/${siteSchema}/${gridID}` + (plotID ? `/${plotID}` : '') + (censusID ? `/${censusID}` : '');
 };
 export const createFetchQuery: FetchQueryFunction = (
   siteSchema: string,

frontend/config/sqlrdsdefinitions/zones.ts

@@ -76,7 +76,6 @@ export type QuadratRDS = {
   id?: number;
   quadratID?: number;
   plotID?: number;
-  censusID?: number;
   quadratName?: string;
   startX?: number;
   startY?: number;