();
diff --git a/src/main/java/org/ecocean/servlet/AnnotationEdit.java b/src/main/java/org/ecocean/servlet/AnnotationEdit.java
index 24e327ca8e..fd9863a930 100644
--- a/src/main/java/org/ecocean/servlet/AnnotationEdit.java
+++ b/src/main/java/org/ecocean/servlet/AnnotationEdit.java
@@ -163,7 +163,7 @@ public class AnnotationEdit extends HttpServlet {
myShepherd.getPM().deletePersistent(enc);
rtn.put("encounterDeleted", true);
}
- myShepherd.getPM().deletePersistent(annot);
+ myShepherd.throwAwayAnnotation(annot);
myShepherd.getPM().deletePersistent(feat);
System.out.println(
"INFO: AnnotationEdit.remove deleted [enc,annot,feat]=[" +
diff --git a/src/main/java/org/ecocean/servlet/EncounterRemoveAnnotation.java b/src/main/java/org/ecocean/servlet/EncounterRemoveAnnotation.java
index 8841fe0e4e..0ee4ccb92d 100644
--- a/src/main/java/org/ecocean/servlet/EncounterRemoveAnnotation.java
+++ b/src/main/java/org/ecocean/servlet/EncounterRemoveAnnotation.java
@@ -104,7 +104,7 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
enc.addComments("Annotation deleted by " + user.getDisplayName() + " on " +
Util.prettyTimeStamp() + "
");
- myShepherd.getPM().deletePersistent(ann);
+ myShepherd.throwAwayAnnotation(ann);
myShepherd.updateDBTransaction();
res.put("revertToTrivial", true);
}
@@ -121,7 +121,7 @@ else if (!ann.isTrivial()) {
"\">Annotation deleted by " + user.getDisplayName() + " on " +
Util.prettyTimeStamp() + "");
enc.removeAnnotation(ann);
- myShepherd.getPM().deletePersistent(ann);
+ myShepherd.throwAwayAnnotation(ann);
myShepherd.commitDBTransaction();
}
response.setStatus(HttpServletResponse.SC_OK);
diff --git a/src/main/java/org/ecocean/servlet/importer/ImportTask.java b/src/main/java/org/ecocean/servlet/importer/ImportTask.java
index 73906e0b10..6fded9d75b 100644
--- a/src/main/java/org/ecocean/servlet/importer/ImportTask.java
+++ b/src/main/java/org/ecocean/servlet/importer/ImportTask.java
@@ -498,44 +498,36 @@ public static void deleteWithRelated(String id, User user, Shepherd myShepherd)
if (!user.isAdmin(myShepherd) && !Collaboration.canUserAccessImportTask(itask,
myShepherd.getContext(), user.getUsername()))
throw new IOException("user does not have privileges to delete task");
+ long startedAt = System.currentTimeMillis();
Util.mark("ImportTask.deleteWithRelated(" + id + ") started");
try {
List allEncs = new ArrayList(itask.getEncounters());
- int total = allEncs.size();
- for (int i = 0; i < allEncs.size(); i++) {
- Encounter enc = allEncs.get(i);
+ // First pass: detach annotations from encounters and collect them for a single
+ // batched FK-cleanup at the end. The previous per-annotation throwAwayAnnotation
+ // ran ~6 unindexed JDOQL queries per annotation, which becomes catastrophic for
+ // large imports. The batch path collapses that to a constant number of queries.
+ List allAnns = new ArrayList();
+ long phaseStart = System.currentTimeMillis();
+ for (Encounter enc : allEncs) {
Occurrence occ = myShepherd.getOccurrence(enc);
MarkedIndividual mark = myShepherd.getMarkedIndividualQuiet(enc.getIndividualID());
List projects = myShepherd.getProjectsForEncounter(enc);
ArrayList anns = enc.getAnnotations();
- for (Annotation ann : anns) {
- enc.removeAnnotation(ann);
- // myShepherd.updateDBTransaction();
- List iaTasks = Task.getTasksFor(ann, myShepherd);
- if (iaTasks != null && !iaTasks.isEmpty()) {
- for (Task iaTask : iaTasks) {
- iaTask.removeObject(ann);
- // myShepherd.updateDBTransaction();
- }
+ if (anns != null) {
+ for (Annotation ann : new ArrayList(anns)) {
+ enc.removeAnnotation(ann);
+ if (ann != null) allAnns.add(ann);
}
- myShepherd.throwAwayAnnotation(ann);
- // myShepherd.updateDBTransaction();
}
- // handle occurrences
if (occ != null) {
occ.removeEncounter(enc);
- // myShepherd.updateDBTransaction();
if (occ.getEncounters().size() == 0) {
myShepherd.throwAwayOccurrence(occ);
- // myShepherd.updateDBTransaction();
}
}
- // handle markedindividual
if (mark != null) {
mark.removeEncounter(enc);
- // myShepherd.updateDBTransaction();
if (mark.getEncounters().size() == 0) {
- // remove scheduled tasks referencing this individual
List mergeTasks =
myShepherd.getAllIncompleteScheduledIndividualMerges();
if (mergeTasks != null) {
@@ -546,44 +538,43 @@ public static void deleteWithRelated(String id, User user, Shepherd myShepherd)
}
}
}
- // check for social unit membership and remove
List units = myShepherd.getAllSocialUnitsForMarkedIndividual(
mark);
if (units != null && units.size() > 0) {
for (SocialUnit unit : units) {
- boolean worked = unit.removeMember(mark, myShepherd);
- // if (worked) myShepherd.updateDBTransaction();
+ unit.removeMember(mark, myShepherd);
}
}
myShepherd.throwAwayMarkedIndividual(mark);
- // myShepherd.updateDBTransaction();
}
}
- // handle projects
if (projects != null && projects.size() > 0) {
for (Project project : projects) {
project.removeEncounter(enc);
- // myShepherd.updateDBTransaction();
}
}
itask.removeEncounter(enc);
itask.addLog("Servlet DeleteImportTask removed Encounter: " +
enc.getCatalogNumber());
- // myShepherd.updateDBTransaction();
try {
myShepherd.throwAwayEncounter(enc);
} catch (Exception e) {
System.out.println("Exception on throwAwayEncounter!!");
e.printStackTrace();
}
- // myShepherd.updateDBTransaction();
}
+ Util.mark("ImportTask.deleteWithRelated(" + id + ") encounter loop done (" +
+ allEncs.size() + " encs, " + allAnns.size() + " anns)", phaseStart);
+ // Single batched FK cleanup + delete for all annotations across the import.
+ phaseStart = System.currentTimeMillis();
+ myShepherd.throwAwayAnnotations(allAnns);
+ Util.mark("ImportTask.deleteWithRelated(" + id +
+ ") batched annotation cleanup done", phaseStart);
myShepherd.getPM().deletePersistent(itask);
- // myShepherd.commitDBTransaction();
} catch (Exception ex) {
throw new IOException("general exception on ImportTask delete: " + ex);
}
- Util.mark("ImportTask.deleteWithRelated(" + id + ") completed");
+ Util.mark("ImportTask.deleteWithRelated(" + id + ") completed", startedAt);
}
// this is hobbled together from some complex code in import.jsp
diff --git a/src/main/java/org/ecocean/shepherd/core/Shepherd.java b/src/main/java/org/ecocean/shepherd/core/Shepherd.java
index 52b5a1fef6..d2481a5553 100644
--- a/src/main/java/org/ecocean/shepherd/core/Shepherd.java
+++ b/src/main/java/org/ecocean/shepherd/core/Shepherd.java
@@ -17,6 +17,7 @@
import org.ecocean.grid.ScanTask;
import org.ecocean.grid.ScanWorkItem;
import org.ecocean.ia.MatchResult;
+import org.ecocean.ia.MatchResultProspect;
import org.ecocean.ia.Task;
import org.ecocean.media.*;
import org.ecocean.movement.Path;
@@ -467,8 +468,129 @@ public void throwAwayMicrosatelliteMarkersAnalysis(MicrosatelliteMarkersAnalysis
pm.deletePersistent(analysis);
}
+ // Caller must detach this Annotation from its owning Encounter (Encounter.removeAnnotation)
+ // before calling — that's a business decision about which encounter the annotation
+ // belongs to. This method handles the FK-constrained dependents that JDO mapping cascades
+ // do not reach: MatchResult, MatchResultProspect, Embedding, and the Task.objectAnnotations
+ // join collection. Without this cleanup, the JDO commit fails with constraints like
+ // MATCHRESULT_FK1 (MatchResult.queryAnnotation), the FKs on MatchResultProspect.annotation,
+ // EMBEDDING.ANNOTATION_ID, and the Task↔Annotation join table.
public void throwAwayAnnotation(Annotation ad) {
- pm.deletePersistent(ad);
+ if (ad == null) return;
+ throwAwayAnnotations(java.util.Collections.singletonList(ad));
+ }
+
+ // Batch version: collects all FK-bound dependents in a single pass per dependent table
+ // (one JDOQL query per table instead of N). Used by ImportTask.deleteWithRelated for
+ // bulk-import deletion which can otherwise be O(N) queries × seq scans of large IA tables.
+ public void throwAwayAnnotations(Collection anns) {
+ if (anns == null || anns.isEmpty()) return;
+ List annIds = new ArrayList();
+ for (Annotation a : anns) {
+ if (a != null && a.getId() != null) annIds.add(a.getId());
+ }
+ if (annIds.isEmpty()) return;
+ cleanUpAnnotationReferencesBatch(annIds);
+ pm.flush(); // ensure dependent deletes hit the DB before the annotation rows go
+ for (Annotation a : anns) {
+ if (a != null) pm.deletePersistent(a);
+ }
+ }
+
+ private void cleanUpAnnotationReferencesBatch(Collection annIds) {
+ // 1. Delete every MatchResultProspect whose annotation OR whose parent MR's
+ // queryAnnotation is in the doomed set. Single query covers both cases.
+ Query qProspects = pm.newQuery(MatchResultProspect.class);
+ qProspects.setFilter(
+ ":ids.contains(annotation.id) || :ids.contains(matchResult.queryAnnotation.id)");
+ try {
+ @SuppressWarnings("unchecked")
+ Collection prospects =
+ (Collection)qProspects.execute(annIds);
+ if (prospects != null) {
+ for (MatchResultProspect p : new ArrayList(prospects)) {
+ pm.deletePersistent(p);
+ }
+ }
+ } finally {
+ qProspects.closeAll();
+ }
+ pm.flush(); // commit prospect deletes before MR FK check
+
+ // 2. Delete MatchResults whose queryAnnotation is in the doomed set.
+ Query qMRs = pm.newQuery(MatchResult.class);
+ qMRs.setFilter(":ids.contains(queryAnnotation.id)");
+ try {
+ @SuppressWarnings("unchecked")
+ Collection mrs = (Collection)qMRs.execute(annIds);
+ if (mrs != null) {
+ for (MatchResult mr : new ArrayList(mrs)) {
+ pm.deletePersistent(mr);
+ }
+ }
+ } finally {
+ qMRs.closeAll();
+ }
+ pm.flush();
+
+ // 3. Remove all doomed annotations from any surviving MatchResult.candidates collection.
+ // Defensive — the field is rarely populated (see MatchResult.java comment) but the
+ // join-table FK would still block deletion if any row referenced these annotations.
+ Query qCand = pm.newQuery(MatchResult.class);
+ qCand.setFilter("candidates.contains(c) && :ids.contains(c.id)");
+ qCand.declareVariables("org.ecocean.Annotation c");
+ try {
+ @SuppressWarnings("unchecked")
+ Collection mrsWithCand = (Collection)qCand.execute(annIds);
+ if (mrsWithCand != null) {
+ for (MatchResult mr : new ArrayList(mrsWithCand)) {
+ if (mr.getCandidates() == null) continue;
+ Iterator it = mr.getCandidates().iterator();
+ while (it.hasNext()) {
+ Annotation cAnn = it.next();
+ if (cAnn != null && annIds.contains(cAnn.getId())) it.remove();
+ }
+ }
+ }
+ } finally {
+ qCand.closeAll();
+ }
+
+ // 4. Delete Embeddings whose annotation is in the doomed set.
+ Query qEmb = pm.newQuery(Embedding.class);
+ qEmb.setFilter(":ids.contains(annotation.id)");
+ try {
+ @SuppressWarnings("unchecked")
+ Collection embs = (Collection)qEmb.execute(annIds);
+ if (embs != null) {
+ for (Embedding e : new ArrayList(embs)) {
+ pm.deletePersistent(e);
+ }
+ }
+ } finally {
+ qEmb.closeAll();
+ }
+
+ // 5. Remove all doomed annotations from any Task.objectAnnotations join collection.
+ Query qTasks = pm.newQuery(Task.class);
+ qTasks.setFilter("objectAnnotations.contains(a) && :ids.contains(a.id)");
+ qTasks.declareVariables("org.ecocean.Annotation a");
+ try {
+ @SuppressWarnings("unchecked")
+ Collection tasks = (Collection)qTasks.execute(annIds);
+ if (tasks != null) {
+ for (Task t : new ArrayList(tasks)) {
+ if (t.getObjectAnnotations() == null) continue;
+ List toRemove = new ArrayList();
+ for (Annotation a : t.getObjectAnnotations()) {
+ if (a != null && annIds.contains(a.getId())) toRemove.add(a);
+ }
+ for (Annotation a : toRemove) t.removeObject(a);
+ }
+ }
+ } finally {
+ qTasks.closeAll();
+ }
}
public void throwAwayOccurrence(Occurrence occ) {
diff --git a/src/main/resources/org/ecocean/ia/package.jdo b/src/main/resources/org/ecocean/ia/package.jdo
index b47b49b153..9f75bf2c43 100755
--- a/src/main/resources/org/ecocean/ia/package.jdo
+++ b/src/main/resources/org/ecocean/ia/package.jdo
@@ -86,11 +86,12 @@ alter table "TASK" alter column "PARAMETERS" type text;
-
+
+
@@ -102,6 +103,7 @@ alter table "TASK" alter column "PARAMETERS" type text;
+
diff --git a/src/main/resources/org/ecocean/package.jdo b/src/main/resources/org/ecocean/package.jdo
index 74519b8018..3e3fd08d94 100755
--- a/src/main/resources/org/ecocean/package.jdo
+++ b/src/main/resources/org/ecocean/package.jdo
@@ -1023,6 +1023,7 @@
+