Merge pull request #881 from opensrp/clean-duplicate-mother-ids

clean duplicate mother ids

opensrp-core/build.gradle

@@ -256,6 +256,9 @@ dependencies {
     implementation 'de.hdodenhof:circleimageview:2.2.0'
     implementation 'xerces:xercesImpl:2.12.0'
 
+    def work_version = "2.7.1"
+    implementation "androidx.work:work-runtime:$work_version"
+
     // Add the dependency for the Performance Monitoring library
     implementation 'com.google.firebase:firebase-perf:19.0.7'
 

opensrp-core/src/main/java/org/smartregister/AllConstants.java

@@ -148,6 +148,8 @@ public class AllConstants {
 
     public static final String GPS = "gps";
 
+    public static final String IDENTIFIERS = "identifiers";
+
 
     public static class Immunizations {
         public static final String BCG = "bcg";
@@ -586,4 +588,14 @@ public interface SyncInfo {
         String TASK_UNPROCESSED_EVENTS = "taskUnprocessedEvents";
         String NULL_EVENT_SYNC_STATUS = "nullEventSyncStatus";
     }
+
+    public static class EventType {
+        public static final String BITRH_REGISTRATION = "Birth Registration";
+        public static final String NEW_WOMAN_REGISTRATION = "New Woman Registration";
+    }
+
+    public static class Entity {
+        public static final String MOTHER = "mother";
+    }
+
 }

opensrp-core/src/main/java/org/smartregister/Context.java

@@ -51,6 +51,7 @@
 import org.smartregister.repository.TaskRepository;
 import org.smartregister.repository.TimelineEventRepository;
 import org.smartregister.repository.UniqueIdRepository;
+import org.smartregister.repository.ZeirIdCleanupRepository;
 import org.smartregister.service.ANMService;
 import org.smartregister.service.ActionService;
 import org.smartregister.service.AlertService;
@@ -229,6 +230,7 @@ public class Context {
     private ManifestRepository manifestRepository;
     private ClientFormRepository clientFormRepository;
     private ClientRelationshipRepository clientRelationshipRepository;
+    private ZeirIdCleanupRepository zeirIdCleanupRepository;
 
     private static final String SHARED_PREFERENCES_FILENAME = "%s_preferences";
 
@@ -1252,5 +1254,13 @@ public ClientRelationshipRepository getClientRelationshipRepository() {
         return clientRelationshipRepository;
     }
 
+
+    public ZeirIdCleanupRepository zeirIdCleanupRepository() {
+        if (zeirIdCleanupRepository == null) {
+            zeirIdCleanupRepository = new ZeirIdCleanupRepository();
+        }
+        return zeirIdCleanupRepository;
+    }
+
     ///////////////////////////////////////////////////////////////////////////////
 }

opensrp-core/src/main/java/org/smartregister/domain/DuplicateZeirIdStatus.java

@@ -0,0 +1,14 @@
+package org.smartregister.domain;
+
+public enum DuplicateZeirIdStatus {
+    CLEANED("CLEANED"), PENDING("PENDING");
+    private String value;
+
+    DuplicateZeirIdStatus(String value) {
+        this.value = value;
+    }
+
+    public String value() {
+        return value;
+    }
+}

opensrp-core/src/main/java/org/smartregister/job/DuplicateCleanerWorker.java

@@ -0,0 +1,33 @@
+package org.smartregister.job;
+
+import android.content.Context;
+
+import androidx.annotation.NonNull;
+import androidx.work.WorkManager;
+import androidx.work.Worker;
+import androidx.work.WorkerParameters;
+
+import org.smartregister.domain.DuplicateZeirIdStatus;
+import org.smartregister.util.AppHealthUtils;
+
+import timber.log.Timber;
+
+public class DuplicateCleanerWorker extends Worker {
+    private Context mContext;
+
+    public DuplicateCleanerWorker(@NonNull Context context, @NonNull WorkerParameters workerParams) {
+        super(context, workerParams);
+        mContext = context;
+    }
+
+    @NonNull
+    @Override
+    public Result doWork() {
+        DuplicateZeirIdStatus duplicateZeirIdStatus = AppHealthUtils.cleanUniqueZeirIds();
+        Timber.i("Doing some cleaning work");
+        if (duplicateZeirIdStatus != null && duplicateZeirIdStatus.equals(DuplicateZeirIdStatus.CLEANED))
+            WorkManager.getInstance(mContext).cancelWorkById(this.getId());
+
+        return Result.success();
+    }
+}

opensrp-core/src/main/java/org/smartregister/repository/EventClientRepository.java

@@ -6,6 +6,7 @@
 import static org.smartregister.AllConstants.ROWID;
 
 import android.content.ContentValues;
+import android.content.Intent;
 import android.database.Cursor;
 import android.text.TextUtils;
 import android.util.Pair;
@@ -25,19 +26,24 @@
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.smartregister.AllConstants;
+import org.smartregister.Context;
 import org.smartregister.CoreLibrary;
 import org.smartregister.clientandeventmodel.DateUtil;
 import org.smartregister.domain.Client;
 import org.smartregister.domain.ClientRelationship;
+import org.smartregister.domain.DuplicateZeirIdStatus;
 import org.smartregister.domain.Event;
+import org.smartregister.domain.UniqueId;
 import org.smartregister.domain.db.Column;
 import org.smartregister.domain.db.ColumnAttribute;
 import org.smartregister.domain.db.EventClient;
 import org.smartregister.p2p.sync.data.JsonData;
 import org.smartregister.sync.intent.P2pProcessRecordsService;
+import org.smartregister.sync.intent.PullUniqueIdsIntentService;
 import org.smartregister.util.DatabaseMigrationUtils;
 import org.smartregister.util.JsonFormUtils;
 import org.smartregister.util.Utils;
+import org.smartregister.view.activity.DrishtiApplication;
 
 import java.lang.reflect.Type;
 import java.text.ParseException;
@@ -65,6 +71,10 @@ public class EventClientRepository extends BaseRepository {
 
     public static final String VARCHAR = "VARCHAR";
 
+    public static final String ZEIR_ID = "ZEIR_ID";
+
+    public static final String M_ZEIR_ID = "M_ZEIR_ID";
+
     protected Table clientTable;
     protected Table eventTable;
 
@@ -2346,4 +2356,94 @@ public List<Event> getEventsByTaskIds(Set<String> taskIds) {
                 + event_column.taskId.name()
                 + " IN (" + StringUtils.repeat("?", ",", taskIds.size()) + ")", taskIds.toArray(new String[0]));
     }
+
+    public DuplicateZeirIdStatus cleanDuplicateMotherIds() throws Exception {
+        String username = Context.getInstance().userService().getAllSharedPreferences().fetchRegisteredANM();
+
+        UniqueIdRepository uniqueIdRepository = Context.getInstance().getUniqueIdRepository();
+
+        Map<String, String> duplicates = Context.getInstance().zeirIdCleanupRepository().getClientsWithDuplicateZeirIds();
+        long unusedIdsCount = uniqueIdRepository.countUnUsedIds();
+
+        Timber.e("%d duplicates for provider: %s - %s", duplicates.size(), username, duplicates.toString());
+
+        if (duplicates.size() > 0) {
+            Timber.e(
+                    "%s: %d duplicates for provider: %s - %s\nUnused Unique IDs: %d",
+                    this.getClass().getSimpleName(),
+                    duplicates.size(),
+                    username,
+                    duplicates,
+                    unusedIdsCount
+            );
+        }
+
+        for (Map.Entry<String, String> duplicate : duplicates.entrySet()) {
+            String baseEntityId = duplicate.getKey();
+            String zeirId = duplicate.getValue();
+
+            JSONObject clientJson = getClientByBaseEntityId(baseEntityId);
+            JSONObject identifiers = clientJson.getJSONObject(AllConstants.IDENTIFIERS);
+
+            long unusedIds = uniqueIdRepository.countUnUsedIds();
+            if (unusedIds <= 30) { // Mske sure we have enough unused IDs left
+                Timber.e("%s: No more unique IDs available to assign to %s - %s; provider: %s", this.getClass().getSimpleName(), baseEntityId, zeirId, username);
+                android.content.Context applicationContext = CoreLibrary.getInstance().context().applicationContext();
+                applicationContext.startService(new Intent(applicationContext, PullUniqueIdsIntentService.class));
+            }
+
+            UniqueId uniqueId = uniqueIdRepository.getNextUniqueId();
+            String newZeirId = uniqueId != null ? uniqueId.getOpenmrsId() : null;
+
+            if (StringUtils.isBlank(newZeirId)) {
+                Timber.e("No unique ID found to assign to %s; provider: %s", baseEntityId, username);
+                return DuplicateZeirIdStatus.PENDING;
+            }
+
+            String eventType = AllConstants.EventType.BITRH_REGISTRATION;
+            String clientType = clientJson.getString(AllConstants.CLIENT_TYPE);
+
+            if (AllConstants.CHILD_TYPE.equals(clientType)) {
+                identifiers.put(ZEIR_ID, newZeirId.replaceAll("-", ""));
+            } else if (AllConstants.Entity.MOTHER.equals(clientType)) {
+                identifiers.put(M_ZEIR_ID, newZeirId);
+                eventType = AllConstants.EventType.NEW_WOMAN_REGISTRATION;
+            }
+            clientJson.put(AllConstants.IDENTIFIERS, identifiers);
+
+            // Add events to process this
+            addorUpdateClient(baseEntityId, clientJson);
+
+            // fetch the birth/new woman registration event
+            List<EventClient> registrationEvent = getEvents(
+                    Collections.singletonList(baseEntityId),
+                    Collections.singletonList(BaseRepository.TYPE_Synced),
+                    Collections.singletonList(eventType)
+            );
+            Event event = null;
+            if (!registrationEvent.isEmpty())
+                event = registrationEvent.get(0).getEvent();
+
+            Client client = convert(clientJson, Client.class);
+
+            // reprocess the event
+            DrishtiApplication.getInstance().getClientProcessor().processClient(Collections.singletonList(new EventClient(event, client)));
+            markClientValidationStatus(baseEntityId, false);
+
+            uniqueIdRepository.close(newZeirId);
+
+            Timber.e("%s: %s - %s updated to %s; provider: %s", this.getClass().getSimpleName(), baseEntityId, zeirId, newZeirId, username);
+        }
+
+        if (duplicates.size() > 0) {
+            Timber.d("%s: Successfully processed %d duplicates for provider: %s - %s",
+                    this.getClass().getSimpleName(),
+                    duplicates.size(),
+                    username,
+                    duplicates
+            );
+        }
+        return DuplicateZeirIdStatus.CLEANED;
+    }
+
 }

opensrp-core/src/main/java/org/smartregister/repository/ZeirIdCleanupRepository.java

@@ -0,0 +1,62 @@
+package org.smartregister.repository;
+
+import net.sqlcipher.Cursor;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import timber.log.Timber;
+
+public class ZeirIdCleanupRepository extends BaseRepository {
+
+    private static final String BASE_ENTITY_ID = "baseEntityId";
+
+    private static final String DUPLICATES_SQL =
+            "WITH duplicates AS ( " +
+                    "  WITH clients AS ( " +
+                    "    SELECT baseEntityId, COALESCE(json_extract(json, '$.identifiers.ZEIR_ID'), json_extract(json, '$.identifiers.M_ZEIR_ID')) zeir_id " +
+                    "    FROM client " +
+                    "  ) " +
+                    "  SELECT b.* FROM (SELECT baseEntityId, zeir_id FROM clients GROUP BY zeir_id HAVING count(zeir_id) > 1) a " +
+                    "  INNER JOIN clients b ON a.zeir_id=b.zeir_id " +
+                    "  UNION " +
+                    "  SELECT * FROM clients WHERE zeir_id IS NULL " +
+                    ") " +
+                    "SELECT baseEntityId, zeir_id, lag(zeir_id) over(order by zeir_id) AS prev_zeir_id FROM duplicates";
+
+    public Map<String, String> getClientsWithDuplicateZeirIds() {
+        Map<String, String> duplicates = new HashMap<>();
+
+        Cursor cursor = null;
+        try {
+            cursor = getWritableDatabase().rawQuery(DUPLICATES_SQL, new String[]{});
+
+            while (cursor.moveToNext()) {
+                String baseEntityId = cursor.getString(cursor.getColumnIndex(BASE_ENTITY_ID));
+                String zeirId = cursor.getString(cursor.getColumnIndex("zeir_id"));
+
+                duplicates.put(baseEntityId, zeirId);
+
+                String prevZeirId = null;
+                try {
+                    prevZeirId = cursor.getString(cursor.getColumnIndex("prev_zeir_id"));
+                } catch (NullPointerException e) {
+                    Timber.e(e, "null prev_zeir_id");
+                }
+
+                if (StringUtils.isNotEmpty(prevZeirId) && (prevZeirId.equals(zeirId))) {
+                    duplicates.put(baseEntityId, prevZeirId);
+                }
+            }
+        } catch (Exception e) {
+            Timber.e(e);
+        } finally {
+            if (cursor != null && !cursor.isClosed())
+                cursor.close();
+        }
+
+        return duplicates;
+    }
+}

opensrp-core/src/main/java/org/smartregister/sync/intent/PullUniqueIdsIntentService.java

@@ -5,7 +5,6 @@
  */
 
 import android.content.Intent;
-
 import androidx.annotation.VisibleForTesting;
 
 import org.json.JSONArray;
@@ -27,7 +26,6 @@ public class PullUniqueIdsIntentService extends BaseSyncIntentService {
     public static final String IDENTIFIERS = "identifiers";
     private UniqueIdRepository uniqueIdRepo;
 
-
     public PullUniqueIdsIntentService() {
         super("PullUniqueOpenMRSUniqueIdsService");
     }
@@ -99,5 +97,4 @@ public int onStartCommand(Intent intent, int flags, int startId) {
     protected HTTPAgent getHttpAgent() {
         return CoreLibrary.getInstance().context().getHttpAgent();
     }
-
 }

opensrp-core/src/main/java/org/smartregister/util/AppHealthUtils.java

@@ -11,6 +11,7 @@
 import android.view.View;
 import android.widget.ArrayAdapter;
 
+import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
 import androidx.appcompat.app.AlertDialog;
 import androidx.appcompat.view.ContextThemeWrapper;
@@ -19,13 +20,16 @@
 import org.smartregister.AllConstants;
 import org.smartregister.CoreLibrary;
 import org.smartregister.R;
+import org.smartregister.domain.DuplicateZeirIdStatus;
 
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Locale;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Executors;
 
+import timber.log.Timber;
+
 /**
  * Created by ndegwamartin on 26/06/2021.
  */
@@ -115,6 +119,18 @@ public static void refreshFileSystem(Context context, @VisibleForTesting boolean
         }
     }
 
+    @Nullable
+    public static DuplicateZeirIdStatus cleanUniqueZeirIds(){
+        try {
+            return CoreLibrary.getInstance().context().getEventClientRepository()
+                    .cleanDuplicateMotherIds();
+        } catch (Exception e) {
+            Timber.e(e);
+        }
+
+        return null;
+    }
+
     public interface HealthStatsView {
 
         void performDatabaseDownload();

opensrp-core/src/main/java/org/smartregister/view/fragment/BaseRegisterFragment.java

@@ -115,7 +115,6 @@ public DialogOption[] serviceModeOptions() {
                 return new DialogOption[]{
                 };
             }
-
             @Override
             public DialogOption[] sortingOptions() {
                 return new DialogOption[]{