Fix FK violations when deleting annotations (bulk import delete + others)

frontend/src/locale/de.json

@@ -511,6 +511,7 @@
   "ROW": "Zeile",
   "APPLY_TO_ALL_ROWS_NOTE": "+ kennzeichnet, dass die Änderung auf alle Zeilen dieser Spalte angewendet werden kann",
   "BULK_IMPORT_DELETE_TASK": "Massenimport-Aufgabe löschen",
+  "BULK_IMPORT_DELETE_TASK_IN_PROGRESS": "Wird gelöscht...",
   "BULK_IMPORT_DATA_UPLOADED": "Daten hochgeladen",
   "BULK_IMPORT_IMAGE_UPLOADED": "Bild hochgeladen",
   "SPREADSHEET_UPLOADED_TITLE": "Tabellen-Datei hochgeladen: {fileName}",

frontend/src/locale/en.json

@@ -509,6 +509,7 @@
     "ROW": "Row",
     "APPLY_TO_ALL_ROWS_NOTE": "+ denotes change can apply to all rows for this column",
     "BULK_IMPORT_DELETE_TASK": "Delete Import Task",
+    "BULK_IMPORT_DELETE_TASK_IN_PROGRESS": "Deleting...",
     "BULK_IMPORT_DATA_UPLOADED": "Data Uploaded",
     "BULK_IMPORT_IMAGE_UPLOADED": "Image Uploaded",
     "SPREADSHEET_UPLOADED_TITLE": "Spreadsheet Uploaded: {fileName}",

frontend/src/locale/es.json

@@ -511,6 +511,7 @@
   "ROW": "Fila",
   "APPLY_TO_ALL_ROWS_NOTE": "+ denota que el cambio se puede aplicar a todas las filas de esta columna",
   "BULK_IMPORT_DELETE_TASK": "Eliminar tarea de importación masiva",
+  "BULK_IMPORT_DELETE_TASK_IN_PROGRESS": "Eliminando...",
   "BULK_IMPORT_DATA_UPLOADED": "Datos subidos",
   "BULK_IMPORT_IMAGE_UPLOADED": "Imagen subida",
   "SPREADSHEET_UPLOADED_TITLE": "Hoja de cálculo subida: {fileName}",

frontend/src/locale/fr.json

@@ -511,6 +511,7 @@
   "ROW": "Ligne",
   "APPLY_TO_ALL_ROWS_NOTE": "+ indique que la modification peut s'appliquer à toutes les lignes de cette colonne",
   "BULK_IMPORT_DELETE_TASK": "Supprimer la tâche d'importation en masse",
+  "BULK_IMPORT_DELETE_TASK_IN_PROGRESS": "Suppression...",
   "BULK_IMPORT_DATA_UPLOADED": "Données téléchargées",
   "BULK_IMPORT_IMAGE_UPLOADED": "Image téléchargée",
   "SPREADSHEET_UPLOADED_TITLE": "Feuille de calcul téléchargée: {fileName}",

frontend/src/locale/it.json

@@ -511,6 +511,7 @@
   "ROW": "Riga",
   "APPLY_TO_ALL_ROWS_NOTE": "+ indica che la modifica può essere applicata a tutte le righe di questa colonna",
   "BULK_IMPORT_DELETE_TASK": "Elimina attività di importazione",
+  "BULK_IMPORT_DELETE_TASK_IN_PROGRESS": "Eliminazione in corso...",
   "BULK_IMPORT_DATA_UPLOADED": "Dati caricati",
   "BULK_IMPORT_IMAGE_UPLOADED": "Immagine caricata",
   "SPREADSHEET_UPLOADED_TITLE": "Foglio di calcolo caricato: {fileName}",

frontend/src/pages/BulkImport/BulkImportTask.jsx

@@ -35,6 +35,7 @@ const BulkImportTask = observer(() => {
   const [userRoles, setUserRoles] = useState(null);
   const store = useLocalObservable(() => new BulkImportTaskStore());
   const [rowsPerPage, setRowsPerPage] = useState(10);
+  const [isDeleting, setIsDeleting] = useState(false);
 
   const previousLocationID = task?.matchingLocations || [];
 
@@ -76,12 +77,14 @@ const BulkImportTask = observer(() => {
 
   const deleteTask = async () => {
     if (!task?.id) return;
+    if (isDeleting) return;
 
     const confirmed = window.confirm(
       intl.formatMessage({ id: "BULK_IMPORT_DELETE_TASK_CONFIRM" }),
     );
     if (!confirmed) return;
 
+    setIsDeleting(true);
     try {
       const res = await fetch(`/api/v3/bulk-import/${task.id}`, {
         method: "DELETE",
@@ -102,6 +105,7 @@ const BulkImportTask = observer(() => {
           { error: err.message || "" },
         ),
       );
+      setIsDeleting(false);
     }
   };
 
@@ -568,6 +572,7 @@ const BulkImportTask = observer(() => {
         <Col xs="auto">
           <MainButton
             onClick={deleteTask}
+            disabled={isDeleting}
             shadowColor={theme.statusColors.red500}
             color={theme.statusColors.red500}
             noArrow={true}
@@ -579,9 +584,23 @@ const BulkImportTask = observer(() => {
               marginLeft: 0,
               marginTop: "1rem",
               marginBottom: "2rem",
+              opacity: isDeleting ? 0.7 : 1,
+              cursor: isDeleting ? "not-allowed" : "pointer",
             }}
           >
-            <FormattedMessage id="BULK_IMPORT_DELETE_TASK" />
+            {isDeleting ? (
+              <>
+                <Spinner
+                  animation="border"
+                  size="sm"
+                  role="status"
+                  className="me-2"
+                />
+                <FormattedMessage id="BULK_IMPORT_DELETE_TASK_IN_PROGRESS" />
+              </>
+            ) : (
+              <FormattedMessage id="BULK_IMPORT_DELETE_TASK" />
+            )}
           </MainButton>
         </Col>
       </Row>

src/main/java/org/ecocean/api/patch/EncounterPatchValidator.java

@@ -248,7 +248,7 @@ public static JSONObject applyPatch(Encounter enc, JSONObject patch, User user,
                     enc.addAnnotation(triv);
                     myShepherd.getPM().makePersistent(triv);
                 }
-                myShepherd.getPM().deletePersistent(ann);
+                myShepherd.throwAwayAnnotation(ann);
                 value = ann;
             } else if (path.equals("occurrenceId")) {
                 // this may be overkill. *technically* an Encounter should be contained in (at most) ONE Occurrence

src/main/java/org/ecocean/ia/MatchResult.java

@@ -397,6 +397,10 @@ public int numberProspects() {
         return Util.collectionSize(prospects);
     }
 
+    public Set<Annotation> getCandidates() {
+        return candidates;
+    }
+
     public Set<String> prospectScoreTypes() {
         Set<String> types = new HashSet<String>();
 

src/main/java/org/ecocean/servlet/AnnotationEdit.java

@@ -163,7 +163,7 @@ public class AnnotationEdit extends HttpServlet {
                             myShepherd.getPM().deletePersistent(enc);
                             rtn.put("encounterDeleted", true);
                         }
-                        myShepherd.getPM().deletePersistent(annot);
+                        myShepherd.throwAwayAnnotation(annot);
                         myShepherd.getPM().deletePersistent(feat);
                         System.out.println(
                             "INFO: AnnotationEdit.remove deleted [enc,annot,feat]=[" +

src/main/java/org/ecocean/servlet/EncounterRemoveAnnotation.java

@@ -104,7 +104,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
                         enc.addComments("<p data-annot-id=\"" + ann.getId() +
                             "\">Annotation deleted by " + user.getDisplayName() + " on " +
                             Util.prettyTimeStamp() + "</p>");
-                        myShepherd.getPM().deletePersistent(ann);
+                        myShepherd.throwAwayAnnotation(ann);
                         myShepherd.updateDBTransaction();
                         res.put("revertToTrivial", true);
                     }
@@ -121,7 +121,7 @@ else if (!ann.isTrivial()) {
                             "\">Annotation deleted by " + user.getDisplayName() + " on " +
                             Util.prettyTimeStamp() + "</p>");
                         enc.removeAnnotation(ann);
-                        myShepherd.getPM().deletePersistent(ann);
+                        myShepherd.throwAwayAnnotation(ann);
                         myShepherd.commitDBTransaction();
                     }
                     response.setStatus(HttpServletResponse.SC_OK);

src/main/java/org/ecocean/servlet/importer/ImportTask.java

@@ -498,44 +498,36 @@ public static void deleteWithRelated(String id, User user, Shepherd myShepherd)
         if (!user.isAdmin(myShepherd) && !Collaboration.canUserAccessImportTask(itask,
             myShepherd.getContext(), user.getUsername()))
             throw new IOException("user does not have privileges to delete task");
+        long startedAt = System.currentTimeMillis();
         Util.mark("ImportTask.deleteWithRelated(" + id + ") started");
         try {
             List<Encounter> allEncs = new ArrayList<Encounter>(itask.getEncounters());
-            int total = allEncs.size();
-            for (int i = 0; i < allEncs.size(); i++) {
-                Encounter enc = allEncs.get(i);
+            // First pass: detach annotations from encounters and collect them for a single
+            // batched FK-cleanup at the end. The previous per-annotation throwAwayAnnotation
+            // ran ~6 unindexed JDOQL queries per annotation, which becomes catastrophic for
+            // large imports. The batch path collapses that to a constant number of queries.
+            List<Annotation> allAnns = new ArrayList<Annotation>();
+            long phaseStart = System.currentTimeMillis();
+            for (Encounter enc : allEncs) {
                 Occurrence occ = myShepherd.getOccurrence(enc);
                 MarkedIndividual mark = myShepherd.getMarkedIndividualQuiet(enc.getIndividualID());
                 List<Project> projects = myShepherd.getProjectsForEncounter(enc);
                 ArrayList<Annotation> anns = enc.getAnnotations();
-                for (Annotation ann : anns) {
-                    enc.removeAnnotation(ann);
-                    // myShepherd.updateDBTransaction();
-                    List<Task> iaTasks = Task.getTasksFor(ann, myShepherd);
-                    if (iaTasks != null && !iaTasks.isEmpty()) {
-                        for (Task iaTask : iaTasks) {
-                            iaTask.removeObject(ann);
-                            // myShepherd.updateDBTransaction();
-                        }
+                if (anns != null) {
+                    for (Annotation ann : new ArrayList<Annotation>(anns)) {
+                        enc.removeAnnotation(ann);
+                        if (ann != null) allAnns.add(ann);
                     }
-                    myShepherd.throwAwayAnnotation(ann);
-                    // myShepherd.updateDBTransaction();
                 }
-                // handle occurrences
                 if (occ != null) {
                     occ.removeEncounter(enc);
-                    // myShepherd.updateDBTransaction();
                     if (occ.getEncounters().size() == 0) {
                         myShepherd.throwAwayOccurrence(occ);
-                        // myShepherd.updateDBTransaction();
                     }
                 }
-                // handle markedindividual
                 if (mark != null) {
                     mark.removeEncounter(enc);
-                    // myShepherd.updateDBTransaction();
                     if (mark.getEncounters().size() == 0) {
-                        // remove scheduled tasks referencing this individual
                         List<ScheduledIndividualMerge> mergeTasks =
                             myShepherd.getAllIncompleteScheduledIndividualMerges();
                         if (mergeTasks != null) {
@@ -546,44 +538,43 @@ public static void deleteWithRelated(String id, User user, Shepherd myShepherd)
                                 }
                             }
                         }
-                        // check for social unit membership and remove
                         List<SocialUnit> units = myShepherd.getAllSocialUnitsForMarkedIndividual(
                             mark);
                         if (units != null && units.size() > 0) {
                             for (SocialUnit unit : units) {
-                                boolean worked = unit.removeMember(mark, myShepherd);
-                                // if (worked) myShepherd.updateDBTransaction();
+                                unit.removeMember(mark, myShepherd);
                             }
                         }
                         myShepherd.throwAwayMarkedIndividual(mark);
-                        // myShepherd.updateDBTransaction();
                     }
                 }
-                // handle projects
                 if (projects != null && projects.size() > 0) {
                     for (Project project : projects) {
                         project.removeEncounter(enc);
-                        // myShepherd.updateDBTransaction();
                     }
                 }
                 itask.removeEncounter(enc);
                 itask.addLog("Servlet DeleteImportTask removed Encounter: " +
                     enc.getCatalogNumber());
-                // myShepherd.updateDBTransaction();
                 try {
                     myShepherd.throwAwayEncounter(enc);
                 } catch (Exception e) {
                     System.out.println("Exception on throwAwayEncounter!!");
                     e.printStackTrace();
                 }
-                // myShepherd.updateDBTransaction();
             }
+            Util.mark("ImportTask.deleteWithRelated(" + id + ") encounter loop done (" +
+                allEncs.size() + " encs, " + allAnns.size() + " anns)", phaseStart);
+            // Single batched FK cleanup + delete for all annotations across the import.
+            phaseStart = System.currentTimeMillis();
+            myShepherd.throwAwayAnnotations(allAnns);
+            Util.mark("ImportTask.deleteWithRelated(" + id +
+                ") batched annotation cleanup done", phaseStart);
             myShepherd.getPM().deletePersistent(itask);
-            // myShepherd.commitDBTransaction();
         } catch (Exception ex) {
             throw new IOException("general exception on ImportTask delete: " + ex);
         }
-        Util.mark("ImportTask.deleteWithRelated(" + id + ") completed");
+        Util.mark("ImportTask.deleteWithRelated(" + id + ") completed", startedAt);
     }
 
     // this is hobbled together from some complex code in import.jsp