Optimisation for bulk loading

Author: hugomanguinhas

record-api-migration/src/main/java/eu/europeana/api/record/migration/MigrationHandler.java

@@ -3,7 +3,7 @@
 import dev.morphia.query.filters.Filters;
 import eu.europeana.api.edm.EDM;
 import eu.europeana.api.format.RdfFormat;
-import eu.europeana.api.record.db.repository.RecordRepository;
+import eu.europeana.api.record.db.repository.OptimisedRecordRepositoryCopy;
 import eu.europeana.api.record.io.FormatHandlerRegistry;
 import eu.europeana.api.record.migration.RecordDomProcessor.Result;
 import eu.europeana.api.record.model.*;
@@ -68,14 +68,14 @@ public class MigrationHandler {
 
     private final FormatHandlerRegistry registry;
 
-    private final RecordRepository migrationRepository;
+    private final OptimisedRecordRepositoryCopy migrationRepository;
 
     private final TemplateLibrary library;
 
     @Autowired
     public MigrationHandler(MigrationSettings settings
                           , FormatHandlerRegistry registry
-                          , RecordRepository migrationRepository
+                          , OptimisedRecordRepositoryCopy migrationRepository
                           , @Qualifier(BEAN_RECORD_TEMPLATE_LIBRARY) TemplateLibrary library) {
         this.settings = settings;
         this.registry = registry;

record-api-model/src/main/java/eu/europeana/api/record/io/jena/ObjectReferenceCodec.java

@@ -3,6 +3,7 @@
 import dev.morphia.mapping.codec.references.MorphiaProxy;
 import eu.europeana.api.record.model.EDMClass;
 import eu.europeana.api.record.model.data.ObjectReference;
+import eu.europeana.api.record.model.data.SharedObject;
 import eu.europeana.jena.encoder.JenaObjectDecoder.DecoderContext;
 import eu.europeana.jena.encoder.JenaObjectEncoder.EncoderContext;
 import eu.europeana.jena.encoder.codec.JenaCodec;
@@ -32,7 +33,7 @@ public void encode(Model m, ObjectReference ref, EncoderContext context) {
         //Good example to test this is: /221/URN_NBN_SI_DOC_0CEKTP4N
         if ( ref.isDereferenced() ) {
             //TODO: review how to best work with lazy loading
-            EDMClass o = ref.getDereferencedObject();
+            SharedObject o = ref.getDereferencedObject();
             // the object may be null when it has not been stored before and 
             // should be covered on the dereferenced
             if ( o != null ) { 

record-api-model/src/main/java/eu/europeana/api/record/model/data/LocalReference.java

@@ -25,7 +25,7 @@ public class LocalReference implements ObjectReference
 
     @JsonUnwrapped
     @Property(ModelConstants.object)
-    protected EDMClass object;
+    protected SharedObject object;
 
     public LocalReference() {}
 
@@ -48,7 +48,7 @@ public LocalReference(EDMClass object)
 
     public boolean isDereferenced() { return this.object != null; }
 
-    public EDMClass getDereferencedObject() { return this.object; }
+    public SharedObject getDereferencedObject() { return this.object; }
 
     public String toString() { return ("<" + id + ">"); }
 
@@ -61,4 +61,9 @@ public void postPersist(Document doc) {
     public void postLoad(Document doc) {
         if ( this.object != null ) { this.id = this.object.getID(); }
     }
+
+    @Override
+    public void setDereferencedObject(SharedObject obj) {
+        this.object = obj;
+    }
 }

record-api-model/src/main/java/eu/europeana/api/record/model/data/ObjectReference.java

@@ -2,15 +2,17 @@
 
 import dev.morphia.annotations.Entity;
 import eu.europeana.api.edm.RDF;
-import eu.europeana.api.record.model.EDMClass;
 
 @Entity(discriminatorKey = RDF.type)
 public interface ObjectReference extends DataValue {
+
     public String   getID();
 
     public boolean  isLocal();
 
     public boolean  isDereferenced();
 
-    public EDMClass getDereferencedObject();
+    public SharedObject getDereferencedObject();
+
+    public void setDereferencedObject(SharedObject obj);
 }

record-api-model/src/main/java/eu/europeana/api/record/model/data/ObjectRepository.java

@@ -3,10 +3,14 @@
  */
 package eu.europeana.api.record.model.data;
 
+import dev.morphia.annotations.Entity;
+import eu.europeana.api.edm.RDF;
+
 /**
  * @author Hugo
  * @since 17 Jun 2024
  */
+@Entity(discriminatorKey = RDF.type)
 public interface SharedObject
 {
     public String getID();

record-api-model/src/main/java/eu/europeana/api/record/model/data/SharedObject.java

@@ -24,13 +24,13 @@ public class SharedReference implements ObjectReference {
 
     private static final Logger LOGGER = LogManager.getLogger(SharedReference.class);
 
-    public static ThreadLocal<ObjectRepository> repo = new ThreadLocal<>();
+    public static ThreadLocal<SharedReferenceHandler> handler = new ThreadLocal<>();
 
     @Property(ModelConstants.id)
     protected String id;
 
     @Reference(value = ModelConstants.object, lazy = true)
-    protected EDMClass object;
+    protected SharedObject object;
 
 
     public SharedReference() {}
@@ -58,11 +58,12 @@ public boolean isLocal() {
      */
     public boolean isDereferenced() { return (this.object != null); }
 
-    public EDMClass getDereferencedObject() {
-        EDMClass obj = this.object;
+    public SharedObject getDereferencedObject() {
+        SharedObject obj = this.object;
+        //explain why there is a if with MorphiaProxy
         if ( obj instanceof MorphiaProxy ) { 
             try {
-                obj = (EDMClass)((MorphiaProxy)obj).unwrap();
+                obj = (SharedObject)((MorphiaProxy)obj).unwrap();
             }
             catch (ReferenceException e) {
                 LOGGER.warn("Could not dereference: " + this.id);
@@ -72,28 +73,30 @@ public EDMClass getDereferencedObject() {
         return obj;
     }
 
+    public void setDereferencedObject(SharedObject obj) {
+        this.object = obj;
+    }
+
     public String toString() { return ("<" + id + ">"); }
 
     @PrePersist
     public void prePersist(Document doc, Datastore ds) {
         if ( this.object != null ) {
-            ObjectRepository repo = this.repo.get();
-            repo.save(this.object);
+            SharedReferenceHandler handler = this.handler.get();
+            if ( handler != null ) { handler.saveShared(this.object); }
         }
     }
 
     @PostLoad
-    public void postLoad(Document document, Datastore ds)
-    {
+    public void postLoad(Document document, Datastore ds) {
+        
+        if ( this.object != null ) {
+            SharedReferenceHandler handler = this.handler.get();
+            if ( handler != null ) { handler.loadShared(this); }
+        }
         // TODO:
         // Consider again removing the id field since the reference is now based
         // on that. The challenge will be getting the id from the reference without
         // forcing a load from the db.
-        
-        //OUTDATED:
-        // when lazy loading the object still comes with an "artificial" value, for which,
-        // the check against null passes and the call to getID() forces the effective load of the object
-        // which effectively breaks the lazy loading. We need to find a better way to do this!
-        //if ( this.object != null ) { this.id = this.object.getID(); }
     }
 }

record-api-model/src/main/java/eu/europeana/api/record/model/data/SharedReference.java

@@ -0,0 +1,16 @@
+/**
+ * 
+ */
+package eu.europeana.api.record.model.data;
+
+/**
+ * @author Hugo
+ * @since 12 Jun 2025
+ */
+public interface SharedReferenceHandler {
+
+    public void saveShared(SharedObject obj);
+
+    public void loadShared(SharedReference ref);
+
+}