Fix OpenSearch index staleness when encounters assigned to individuals

src/main/java/org/ecocean/Encounter.java

@@ -5050,6 +5050,9 @@ public org.json.JSONObject processPatch(org.json.JSONArray patchArr, User user,
         this.setDWCDateLastModified();
         this._log(resArr);
         this.setSkipAutoIndexing(false);
+        // Explicitly reindex since postStore fired while skipAutoIndexing was true
+        IndexingManager im = IndexingManagerFactory.getIndexingManager();
+        if (im != null) im.addIndexingQueueEntry(this, false);
         return rtn;
     }
 

src/main/java/org/ecocean/IndexingManager.java

@@ -83,6 +83,39 @@ public void run() {
 
     }
 
+    // GH-1514: queue deep reindex for each MarkedIndividual identified by id,
+    // so sibling encounters pick up refreshed individualNumberEncounters (and
+    // the other individual-derived denormalized fields on the encounter index).
+    // Safe to call with an empty or null set. Callers should invoke this AFTER
+    // the caller's DB transaction has committed, since IndexingManager spins
+    // a background Shepherd that reads the individual by id.
+    //
+    // Opens its own short-lived read-only Shepherd for the id->object resolution
+    // rather than reusing the caller's. Callers in servlets typically close their
+    // Shepherd in a finally block before (or alongside) queueing; reusing it here
+    // would silently no-op because getMarkedIndividualQuiet uses the underlying
+    // closed PersistenceManager. The passed-in Shepherd is used only for its
+    // context string.
+    public static void queueIndividualsByIdForDeepReindex(Shepherd myShepherd,
+        java.util.Collection<String> individualIds) {
+        if ((individualIds == null) || individualIds.isEmpty()) return;
+        IndexingManager im = IndexingManagerFactory.getIndexingManager();
+        if (im == null) return;
+        String context = (myShepherd != null) ? myShepherd.getContext() : "context0";
+        Shepherd shep = new Shepherd(context);
+        shep.setAction("IndexingManager.queueIndividualsByIdForDeepReindex");
+        shep.beginDBTransaction();
+        try {
+            for (String id : individualIds) {
+                if (id == null) continue;
+                MarkedIndividual indiv = shep.getMarkedIndividualQuiet(id);
+                if (indiv != null) im.addIndexingQueueEntry(indiv, false);
+            }
+        } finally {
+            shep.rollbackAndClose();
+        }
+    }
+
     //Removes an oject's UUID from the queue
     public void removeIndexingQueueEntry(String objectID) {
         if (indexingQueue.contains(objectID)) {

src/main/java/org/ecocean/MarkedIndividual.java

@@ -2606,9 +2606,12 @@ public void removeFromNamesCache() {
 
     // Need request to record which user did it
     public void mergeIndividual(MarkedIndividual other, String username, Shepherd myShepherd) {
+        IndexingManager im = IndexingManagerFactory.getIndexingManager();
         for (Encounter enc : other.getEncounters()) {
             other.removeEncounter(enc);
             enc.setIndividual(this);
+            // Reindex each transferred encounter
+            if (im != null) im.addIndexingQueueEntry(enc, false);
         }
         this.names.merge(other.getNames());
         this.setComments(getMergedComments(other, username));
@@ -2653,6 +2656,11 @@ public void mergeIndividual(MarkedIndividual other, String username, Shepherd my
             myShepherd.updateDBTransaction();
         }
         refreshDependentProperties();
+        // Reindex both individuals after merge
+        if (im != null) {
+            im.addIndexingQueueEntry(this, false);
+            im.addIndexingQueueEntry(other, false);
+        }
     }
 
     public String getMergedComments(MarkedIndividual other, HttpServletRequest request,

src/main/java/org/ecocean/api/BulkImport.java

@@ -143,6 +143,9 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response)
         JSONObject matchingSetFilter = new JSONObject();
         JSONObject encAssets = null;
         String dupId = null; // gets set as bulkImporId to be used in finally block
+        // GH-1514: hoisted so the finally block can queue a post-commit deep
+        // reindex of individuals the foreground importer touched.
+        BulkImporter fgImporter = null;
         long startProcess = System.currentTimeMillis();
         Shepherd myShepherd = new Shepherd(context);
 
@@ -425,6 +428,9 @@ public void run() {
 
                             JSONObject bgEncAssets = null;
                             boolean success = false;
+                            // GH-1514: hoisted so the finally block can queue
+                            // post-commit deep reindex of touched individuals.
+                            BulkImporter bgImporter = null;
                             try {
                                 User bgUser = bgShepherd.getUser(currentUsername);
                                 initializeImportTask(bulkImportId, bgUser, payload,
@@ -448,6 +454,7 @@ public void run() {
                                 rtn.put("numberMediaAssetsCreated", maMap.size());
                                 BulkImporter importer = new BulkImporter(bulkImportId,
                                     validatedRows, maMap, bgUser, bgShepherd);
+                                bgImporter = importer;
                                 JSONObject results = null;
                                 if (!blockedByMAErrors) {
                                     try {
@@ -518,6 +525,15 @@ public void run() {
                                     bgShepherd.rollbackDBTransaction();
                                 }
                                 bgShepherd.closeDBTransaction();
+                                // GH-1514: post-commit, queue deep reindex of touched
+                                // individuals so sibling encounters pick up refreshed
+                                // individualNumberEncounters. The bulkOpensearchIndex
+                                // pass only does shallow individual doc indexing.
+                                if (success && bgImporter != null) {
+                                    org.ecocean.IndexingManager
+                                        .queueIndividualsByIdForDeepReindex(bgShepherd,
+                                            bgImporter.getTouchedIndividualIds());
+                                }
                                 if (success && !bgSkipDetection)
                                     initiateIA(bulkImportId, bgSkipIdentification, bgEncAssets,
                                         matchingSetFilter);
@@ -562,6 +578,7 @@ public void run() {
                     } else {
                         BulkImporter importer = new BulkImporter(bulkImportId, validatedRows, maMap,
                             currentUser, myShepherd);
+                        fgImporter = importer;
                         BulkImporter.logProgress(bulkImportId, "doPost: fg pre-createImport()",
                             startTime);
                         JSONObject results = importer.createImport();
@@ -624,6 +641,12 @@ public void run() {
                 myShepherd.rollbackDBTransaction();
             }
             myShepherd.closeDBTransaction();
+            // GH-1514: post-commit, queue deep reindex of individuals touched by
+            // the foreground import so sibling encounters refresh individualNumberEncounters.
+            if ((statusCode == 200) && !validateOnly && fgImporter != null) {
+                org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd,
+                    fgImporter.getTouchedIndividualIds());
+            }
             if ((statusCode == 200) && !skipDetection)
                 initiateIA(dupId, skipIdentification, encAssets, matchingSetFilter);
         }
@@ -647,6 +670,8 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response
         Shepherd myShepherd = new Shepherd(context);
 
         myShepherd.setAction("api.Bulk.doDelete");
+        java.util.Set<String> touchedSurvivingIndividualIds =
+            new java.util.LinkedHashSet<String>();
         myShepherd.beginDBTransaction();
         try {
             User currentUser = myShepherd.getUser(request);
@@ -655,7 +680,8 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response
             if (args.length < 2) throw new ServletException("bad api path");
             String bulkImportId = args[1];
             // this may throw IOException (and will if currentUser is null or cannot delete)
-            ImportTask.deleteWithRelated(bulkImportId, currentUser, myShepherd);
+            touchedSurvivingIndividualIds = ImportTask.deleteWithRelated(bulkImportId,
+                currentUser, myShepherd);
             statusCode = 204;
             rtn.put("success", true);
         } catch (IOException ex) {
@@ -676,6 +702,12 @@ protected void doDelete(HttpServletRequest request, HttpServletResponse response
             }
             myShepherd.closeDBTransaction();
         }
+        // GH-1514: post-commit, queue deep reindex of individuals that survived
+        // the bulk-import delete so their remaining encounters refresh in OpenSearch.
+        if (statusCode == 204) {
+            org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd,
+                touchedSurvivingIndividualIds);
+        }
         response.setStatus(statusCode);
         response.setCharacterEncoding("UTF-8");
         response.setHeader("Content-Type", "application/json");

src/main/java/org/ecocean/api/bulk/BulkImporter.java

@@ -49,6 +49,20 @@ public class BulkImporter {
     private Map<String, Encounter> encounterCache = new HashMap<String, Encounter>();
     private Map<String, Occurrence> occurrenceCache = new HashMap<String, Occurrence>();
     private Map<String, MarkedIndividual> individualCache = new HashMap<String, MarkedIndividual>();
+
+    // GH-1514: individual ids that were either created OR had encounters added
+    // during this import. Caller should queue these for a deep reindex AFTER
+    // committing, so sibling encounters on pre-existing individuals pick up
+    // refreshed individualNumberEncounters.
+    public java.util.Set<String> getTouchedIndividualIds() {
+        java.util.Set<String> ids = new java.util.LinkedHashSet<String>();
+        for (MarkedIndividual indiv : individualCache.values()) {
+            if (indiv != null && indiv.getIndividualID() != null) {
+                ids.add(indiv.getIndividualID());
+            }
+        }
+        return ids;
+    }
     private Map<String, User> userCache = new HashMap<String, User>();
     private Map<String, Keyword> keywordCache = new HashMap<String, Keyword>();
     private Map<String, Project> projectCache = new HashMap<String, Project>();

src/main/java/org/ecocean/identity/IBEISIA.java

@@ -2559,6 +2559,10 @@ public static JSONObject assignFromIAAPI(JSONObject arg, Shepherd myShepherd, bo
             rtn.put("annotations", ja);
         }
         myShepherd.commitDBTransaction();
+        // GH-1514: post-commit, queue deep reindex of the target individual so
+        // sibling encounters pick up refreshed individualNumberEncounters etc.
+        org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd,
+            java.util.Collections.singleton(indivId));
         return rtn;
     }
 

src/main/java/org/ecocean/servlet/AnnotationEdit.java

@@ -2,6 +2,8 @@
 
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.LinkedHashSet;
+import java.util.Set;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -11,6 +13,7 @@
 import org.ecocean.AccessControl;
 import org.ecocean.Annotation;
 import org.ecocean.Encounter;
+import org.ecocean.IndexingManager;
 import org.ecocean.media.Feature;
 import org.ecocean.media.MediaAsset;
 import org.ecocean.MarkedIndividual;
@@ -53,6 +56,10 @@ public class AnnotationEdit extends HttpServlet {
             out.println("access denied");
         }
         JSONObject rtn = new JSONObject("{\"success\": false}");
+        // GH-1514: collect individual ids touched during this request so we can
+        // queue a post-commit deep reindex, refreshing individualNumberEncounters
+        // (and related denormalized fields) on every sibling encounter.
+        Set<String> touchedIndividualIds = new LinkedHashSet<>();
         String annId = jsonIn.optString("id", null);
         Annotation annot = myShepherd.getAnnotation(annId);
         if (annot == null) {
@@ -83,13 +90,15 @@ public class AnnotationEdit extends HttpServlet {
                             if (indiv != null) {
                                 indiv.removeEncounter(enc1);
                                 indiv.addEncounter(enc2);
+                                touchedIndividualIds.add(indivId1);
                             }
                         }
                         if (indivId2 != null) {
                             MarkedIndividual indiv = myShepherd.getMarkedIndividualQuiet(indivId2);
                             if (indiv != null) {
                                 indiv.removeEncounter(enc2);
                                 indiv.addEncounter(enc1);
+                                touchedIndividualIds.add(indivId2);
                             }
                         }
                         // enc2.setIndividualID(indivId1);
@@ -179,9 +188,12 @@ public class AnnotationEdit extends HttpServlet {
             } else if (Util.stringExists(assignIndivId)) {
                 Encounter enc = annot.findEncounter(myShepherd);
                 if (enc.hasMarkedIndividual()) {
-                    MarkedIndividual oldIndiv = myShepherd.getMarkedIndividualQuiet(
-                        enc.getIndividualID());
-                    oldIndiv.removeEncounter(enc);
+                    String oldIndivId = enc.getIndividualID();
+                    MarkedIndividual oldIndiv = myShepherd.getMarkedIndividualQuiet(oldIndivId);
+                    if (oldIndiv != null) {
+                        oldIndiv.removeEncounter(enc);
+                        touchedIndividualIds.add(oldIndivId);
+                    }
                 }
                 boolean newIndiv = false;
                 MarkedIndividual indiv = myShepherd.getMarkedIndividualQuiet(assignIndivId);
@@ -191,6 +203,7 @@ public class AnnotationEdit extends HttpServlet {
                 } else {
                     indiv.addEncounter(enc);
                 }
+                touchedIndividualIds.add(assignIndivId);
                 // enc.setIndividualID(assignIndivId);
                 System.out.println("INFO: AnnotationEdit assigned " + indiv + " on " + enc +
                     " via " + annot);
@@ -202,6 +215,10 @@ public class AnnotationEdit extends HttpServlet {
         }
         if (rtn.optBoolean("success", false)) {
             myShepherd.commitDBTransaction();
+            // GH-1514: post-commit, queue deep reindex of affected individuals so
+            // sibling encounters pick up refreshed individualNumberEncounters etc.
+            IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd,
+                touchedIndividualIds);
         } else {
             myShepherd.rollbackDBTransaction();
         }

src/main/java/org/ecocean/servlet/EncounterForm.java

@@ -27,6 +27,7 @@
 import org.apache.commons.fileupload.servlet.ServletFileUpload;
 import org.ecocean.CommonConfiguration;
 import org.ecocean.Encounter;
+import org.ecocean.IndexingManager;
 import org.ecocean.ia.Task;
 import org.ecocean.identity.IBEISIA;
 import org.ecocean.IAJsonProperties;
@@ -620,6 +621,10 @@ else if (formValues.get("location") != null) {
                 System.out.println("        ENCOUNTERFORM:");
                 System.out.println("        ENCOUNTERFORM:");
             }
+            // GH-1514: track individual id so we can queue a post-commit deep
+            // reindex; new encounter on existing individual changes sibling
+            // individualNumberEncounters.
+            String manualIndID = null;
             if (formValues.get("manualID") != null &&
                 formValues.get("manualID").toString().length() > 0) {
                 String indID = formValues.get("manualID").toString();
@@ -637,6 +642,7 @@ else if (formValues.get("location") != null) {
                 }
                 if (ind != null) enc.setIndividual(ind);
                 enc.setFieldID(indID);
+                manualIndID = ind != null ? ind.getIndividualID() : indID;
             }
             if (formValues.get("occurrenceID") != null &&
                 formValues.get("occurrenceID").toString().length() > 0) {
@@ -877,6 +883,12 @@ else if (formValues.get("location") != null) {
             if (!spamBot) {
                 newnum = myShepherd.storeNewEncounter(enc, encID);
                 enc.refreshAssetFormats(myShepherd);
+                // GH-1514: post-commit deep reindex so individualNumberEncounters
+                // updates on sibling encounters of the manually-assigned individual.
+                if (!"fail".equals(newnum) && manualIndID != null) {
+                    IndexingManager.queueIndividualsByIdForDeepReindex(myShepherd,
+                        java.util.Collections.singleton(manualIndID));
+                }
 
                 // *after* persisting this madness, then lets kick MediaAssets to IA for whatever fate awaits them
                 // note: we dont send Annotations here, as they are always(forever?) trivial annotations, so pretty disposable

src/main/java/org/ecocean/servlet/EncounterVMData.java

@@ -92,6 +92,10 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
                             matchID + ".</p>");
                         enc.setMatchedBy("Visual Matcher");
                         myShepherd.commitDBTransaction();
+                        // GH-1514: post-commit, queue deep reindex of the individual
+                        // so sibling encounters pick up refreshed denormalized fields.
+                        org.ecocean.IndexingManager.queueIndividualsByIdForDeepReindex(
+                            myShepherd, java.util.Collections.singleton(matchID));
                         redirUrl = "encounters/encounter.jsp?number=" + enc.getCatalogNumber();
                     } else {
                         rtn.put("error", "unauthorized");

src/main/java/org/ecocean/servlet/IndividualAddEncounter.java

@@ -109,6 +109,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
                             addToMe.refreshDependentProperties();
                             myShepherd.updateDBTransaction();
                             System.out.println("Now adding the Encounter to the individual");
+                            // Reindex encounter after individual assignment
+                            IndexingManager im = IndexingManagerFactory.getIndexingManager();
+                            if (im != null) im.addIndexingQueueEntry(enc2add, false);
                         }
                         enc2add.setMatchedBy(request.getParameter("matchType"));
                         enc2add.addComments("<p><em>" + request.getRemoteUser() + " on " +

src/main/java/org/ecocean/servlet/IndividualCreateForProject.java

@@ -79,6 +79,9 @@ public void doPost(HttpServletRequest request, HttpServletResponse response)
                         myShepherd.updateDBTransaction();
                         enc.setIndividual(individual);
                         myShepherd.updateDBTransaction();
+                        // Reindex encounter after individual assignment
+                        IndexingManager im = IndexingManagerFactory.getIndexingManager();
+                        if (im != null) im.addIndexingQueueEntry(enc, false);
 
                         res.put("newIndividualId", individual.getId());
                         res.put("newIndividualName", individual.getName(projectId));