Merge pull request #230 from JetBrains/graph-store-clear-data-after-m…

…igration

#XD-1178 fixed

query/build.gradle.kts

@@ -5,6 +5,7 @@ dependencies {
     api(project(":xodus-openAPI"))
     implementation("com.github.penemue:keap:0.3.0")
     implementation(project(":xodus-environment"))
+    implementation("commons-io:commons-io:2.15.1")
     implementation(libs.slf4j.simple)
 
     testImplementation(project(":xodus-utils", "testArtifacts"))

query/src/main/kotlin/jetbrains/exodus/query/metadata/XodusToOrientDataMigratorLauncher.kt

@@ -20,27 +20,50 @@ import jetbrains.exodus.entitystore.PersistentEntityStores
 import jetbrains.exodus.entitystore.orientdb.*
 import jetbrains.exodus.env.Environments
 import jetbrains.exodus.env.newEnvironmentConfig
+import jetbrains.exodus.log.BackupMetadata
+import jetbrains.exodus.log.StartupMetadata
 import mu.KotlinLogging
+import org.apache.commons.io.FileUtils
 import java.io.File
 import kotlin.time.measureTimedValue
 
 private val log = KotlinLogging.logger { }
 
 val VERTEX_CLASSES_TO_SKIP_MIGRATION = 10
 
+
+/**
+ * Configuration for migrating to OrientDB from another system.
+ *
+ * @param databaseProvider Provides database connections.
+ * @param db Instance of the OrientDB to which data is being migrated.
+ * @param orientConfig Configuration settings specific to the OrientDB database.
+ * @param closeOnFinish Flag indicating whether to close the database upon completion of migration.
+ */
 data class MigrateToOrientConfig(
     val databaseProvider: ODatabaseProvider,
     val db: OrientDB,
     val orientConfig: ODatabaseConfig,
     val closeOnFinish: Boolean = false
 )
 
+/**
+ * Configuration for migrating from Xodus to another system.
+ *
+ * @param databaseDirectory The directory where the Xodus database is located.
+ * @param storeName The name of the store within the Xodus database.
+ * @param cipherKey Optional cipher key used for encrypting the database.
+ * @param cipherIV The initialization vector used for encryption if a cipher key is provided.
+ * @param memoryUsagePercentage Percentage of memory that can be used by the process.
+ * @param dropOldDatabaseAfterMigration Flag indicating whether the old database should be deleted after migration.
+ */
 data class MigrateFromXodusConfig(
     val databaseDirectory: String,
     val storeName: String,
     val cipherKey: String?,
     val cipherIV: Long,
-    val memoryUsagePercentage: Int
+    val memoryUsagePercentage: Int,
+    val dropOldDatabaseAfterMigration: Boolean = false
 )
 
 class XodusToOrientDataMigratorLauncher(
@@ -67,25 +90,11 @@ class XodusToOrientDataMigratorLauncher(
         val classesCount = dbProvider.withSession {
             it.metadata.schema.classes.filter { !it.name.startsWith("O") }.size
         }
-        if (classesCount > VERTEX_CLASSES_TO_SKIP_MIGRATION){
+        if (classesCount > VERTEX_CLASSES_TO_SKIP_MIGRATION) {
             log.info { "There are already $classesCount classes in the database so it's considered as migrated" }
             return
         }
 
-        // 1.2 Create OModelMetadata
-        // it is important to disable autoInitialize for the schemaBuddy,
-        // dataMigrator does not like anything existing in the database before it migrated the data
-        val schemaBuddy = OSchemaBuddyImpl(dbProvider, autoInitialize = false)
-        val oModelMetadata = OModelMetaData(dbProvider, schemaBuddy)
-
-        // 1.3 Create OPersistentEntityStore
-        // it is important to pass the oModelMetadata to the entityStore as schemaBuddy.
-        // it (oModelMetadata) must handle all the schema-related logic.
-        val oEntityStore = OPersistentEntityStore(dbProvider, dbName, schemaBuddy = oModelMetadata)
-
-        // 1.4 Create TransientEntityStore
-        // val oTransientEntityStore = TransientEntityStoreImpl(oModelMetadata, oEntityStore)
-
 
         // 2. Where we migrate the data from
 
@@ -104,6 +113,21 @@ class XodusToOrientDataMigratorLauncher(
         })
         val xEntityStore = PersistentEntityStores.newInstance(env, xodus.storeName)
 
+        // 1.2 Create OModelMetadata
+        // it is important to disable autoInitialize for the schemaBuddy,
+        // dataMigrator does not like anything existing in the database before it migrated the data
+        val schemaBuddy = OSchemaBuddyImpl(dbProvider, autoInitialize = false)
+        val oModelMetadata = OModelMetaData(dbProvider, schemaBuddy)
+
+        // 1.3 Create OPersistentEntityStore
+        // it is important to pass the oModelMetadata to the entityStore as schemaBuddy.
+        // it (oModelMetadata) must handle all the schema-related logic.
+        val oEntityStore = OPersistentEntityStore(dbProvider, dbName, schemaBuddy = oModelMetadata)
+
+        // 1.4 Create TransientEntityStore
+        // val oTransientEntityStore = TransientEntityStoreImpl(oModelMetadata, oEntityStore)
+
+
         try {
             xEntityStore.computeInTransaction { tx ->
                 require(tx.entityTypes.size > 0) { "The Xodus database contains 0 entity types. Looks like a misconfiguration." }
@@ -210,11 +234,73 @@ class XodusToOrientDataMigratorLauncher(
         } finally {
             // cleanup
             xEntityStore.close()
+            cleanUpXodusDB(xodus.dropOldDatabaseAfterMigration)
             if (orient.closeOnFinish) {
                 orient.db.close()
             }
         }
     }
+
+    private fun cleanUpXodusDB(delete:Boolean) {
+        val xodusSystemFilenames = hashSetOf(
+            StartupMetadata.ZERO_FILE_NAME,
+            StartupMetadata.FIRST_FILE_NAME,
+            BackupMetadata.BACKUP_METADATA_FILE_NAME,
+            BackupMetadata.START_BACKUP_METADATA_FILE_NAME,
+            "xd.lck"
+        )
+        if (delete){
+            log.info("Removing blobs...")
+            val blobsDir = File(xodus.databaseDirectory, "blobs")
+            if (blobsDir.exists()) {
+                FileUtils.deleteDirectory(blobsDir)
+            }
+            log.info("Blobs removed")
+
+            log.info("Removing text index...")
+            val textIndex = File(xodus.databaseDirectory, "textindex")
+            if (textIndex.exists()){
+                FileUtils.deleteDirectory(textIndex)
+            }
+            log.info("Text index removed...")
+
+            log.info("Cleanup xd files...")
+            val dbDirectory = File(xodus.databaseDirectory)
+            if (dbDirectory.exists() && dbDirectory.isDirectory){
+                dbDirectory.listFiles { file ->
+                    val name = file.name
+                    name.endsWith(".xd") || xodusSystemFilenames.contains(name)
+                }?.forEach {
+                    it.delete()
+                }
+            }
+            log.info("DB Files cleanup finished...")
+        } else {
+            log.info("Moving xodus files to backup directory")
+            val newRoot = File(xodus.databaseDirectory, "xodus_backup")
+            log.info("Moving blobs...")
+            val blobsDir = File(xodus.databaseDirectory, "blobs")
+            if (blobsDir.exists() && blobsDir.isDirectory){
+                FileUtils.moveDirectory(blobsDir, File(newRoot, "blobs"))
+            }
+            log.info("Moving text-index...")
+            val textIndex = File(xodus.databaseDirectory, "textindex")
+            if (textIndex.exists()){
+                FileUtils.moveDirectory(textIndex, File(newRoot, "textIndex"))
+            }
+            log.info("Moving xd files...")
+            val dbDirectory = File(xodus.databaseDirectory)
+            if (dbDirectory.exists() && dbDirectory.isDirectory){
+                dbDirectory.listFiles { file ->
+                    val name = file.name
+                    name.endsWith(".xd") || xodusSystemFilenames.contains(name)
+                }?.forEach {
+                    FileUtils.moveFile(it, File(newRoot, it.name))
+                }
+            }
+            log.info("DB files moved")
+        }
+    }
 }
 
 private fun percent(value: Double): String = "%.${2}f".format(value * 100) + "%"

settings.gradle.kts

@@ -32,7 +32,7 @@ dependencyResolutionManagement {
             version("commons-lang", "3.12.0")
             version("commons-compress", "1.22")
             version("bouncyCastle", "1.70")
-            version("commons-io", "2.11.0")
+            version("commons-io", "2.15.1")
             version("lucene", "8.10.0")
             version("fastutil", "8.5.12")