diff --git a/query/build.gradle.kts b/query/build.gradle.kts index 300e8420f..09d1eac69 100644 --- a/query/build.gradle.kts +++ b/query/build.gradle.kts @@ -5,6 +5,7 @@ dependencies { api(project(":xodus-openAPI")) implementation("com.github.penemue:keap:0.3.0") implementation(project(":xodus-environment")) + implementation("commons-io:commons-io:2.15.1") implementation(libs.slf4j.simple) testImplementation(project(":xodus-utils", "testArtifacts")) diff --git a/query/src/main/kotlin/jetbrains/exodus/query/metadata/XodusToOrientDataMigratorLauncher.kt b/query/src/main/kotlin/jetbrains/exodus/query/metadata/XodusToOrientDataMigratorLauncher.kt index 8ad50d8ff..a69d7a839 100644 --- a/query/src/main/kotlin/jetbrains/exodus/query/metadata/XodusToOrientDataMigratorLauncher.kt +++ b/query/src/main/kotlin/jetbrains/exodus/query/metadata/XodusToOrientDataMigratorLauncher.kt @@ -20,7 +20,10 @@ import jetbrains.exodus.entitystore.PersistentEntityStores import jetbrains.exodus.entitystore.orientdb.* import jetbrains.exodus.env.Environments import jetbrains.exodus.env.newEnvironmentConfig +import jetbrains.exodus.log.BackupMetadata +import jetbrains.exodus.log.StartupMetadata import mu.KotlinLogging +import org.apache.commons.io.FileUtils import java.io.File import kotlin.time.measureTimedValue @@ -28,6 +31,15 @@ private val log = KotlinLogging.logger { } val VERTEX_CLASSES_TO_SKIP_MIGRATION = 10 + +/** + * Configuration for migrating to OrientDB from another system. + * + * @param databaseProvider Provides database connections. + * @param db Instance of the OrientDB to which data is being migrated. + * @param orientConfig Configuration settings specific to the OrientDB database. + * @param closeOnFinish Flag indicating whether to close the database upon completion of migration. + */ data class MigrateToOrientConfig( val databaseProvider: ODatabaseProvider, val db: OrientDB, @@ -35,12 +47,23 @@ data class MigrateToOrientConfig( val closeOnFinish: Boolean = false ) +/** + * Configuration for migrating from Xodus to another system. + * + * @param databaseDirectory The directory where the Xodus database is located. + * @param storeName The name of the store within the Xodus database. + * @param cipherKey Optional cipher key used for encrypting the database. + * @param cipherIV The initialization vector used for encryption if a cipher key is provided. + * @param memoryUsagePercentage Percentage of memory that can be used by the process. + * @param dropOldDatabaseAfterMigration Flag indicating whether the old database should be deleted after migration. + */ data class MigrateFromXodusConfig( val databaseDirectory: String, val storeName: String, val cipherKey: String?, val cipherIV: Long, - val memoryUsagePercentage: Int + val memoryUsagePercentage: Int, + val dropOldDatabaseAfterMigration: Boolean = false ) class XodusToOrientDataMigratorLauncher( @@ -67,25 +90,11 @@ class XodusToOrientDataMigratorLauncher( val classesCount = dbProvider.withSession { it.metadata.schema.classes.filter { !it.name.startsWith("O") }.size } - if (classesCount > VERTEX_CLASSES_TO_SKIP_MIGRATION){ + if (classesCount > VERTEX_CLASSES_TO_SKIP_MIGRATION) { log.info { "There are already $classesCount classes in the database so it's considered as migrated" } return } - // 1.2 Create OModelMetadata - // it is important to disable autoInitialize for the schemaBuddy, - // dataMigrator does not like anything existing in the database before it migrated the data - val schemaBuddy = OSchemaBuddyImpl(dbProvider, autoInitialize = false) - val oModelMetadata = OModelMetaData(dbProvider, schemaBuddy) - - // 1.3 Create OPersistentEntityStore - // it is important to pass the oModelMetadata to the entityStore as schemaBuddy. - // it (oModelMetadata) must handle all the schema-related logic. - val oEntityStore = OPersistentEntityStore(dbProvider, dbName, schemaBuddy = oModelMetadata) - - // 1.4 Create TransientEntityStore - // val oTransientEntityStore = TransientEntityStoreImpl(oModelMetadata, oEntityStore) - // 2. Where we migrate the data from @@ -104,6 +113,21 @@ class XodusToOrientDataMigratorLauncher( }) val xEntityStore = PersistentEntityStores.newInstance(env, xodus.storeName) + // 1.2 Create OModelMetadata + // it is important to disable autoInitialize for the schemaBuddy, + // dataMigrator does not like anything existing in the database before it migrated the data + val schemaBuddy = OSchemaBuddyImpl(dbProvider, autoInitialize = false) + val oModelMetadata = OModelMetaData(dbProvider, schemaBuddy) + + // 1.3 Create OPersistentEntityStore + // it is important to pass the oModelMetadata to the entityStore as schemaBuddy. + // it (oModelMetadata) must handle all the schema-related logic. + val oEntityStore = OPersistentEntityStore(dbProvider, dbName, schemaBuddy = oModelMetadata) + + // 1.4 Create TransientEntityStore + // val oTransientEntityStore = TransientEntityStoreImpl(oModelMetadata, oEntityStore) + + try { xEntityStore.computeInTransaction { tx -> require(tx.entityTypes.size > 0) { "The Xodus database contains 0 entity types. Looks like a misconfiguration." } @@ -210,11 +234,73 @@ class XodusToOrientDataMigratorLauncher( } finally { // cleanup xEntityStore.close() + cleanUpXodusDB(xodus.dropOldDatabaseAfterMigration) if (orient.closeOnFinish) { orient.db.close() } } } + + private fun cleanUpXodusDB(delete:Boolean) { + val xodusSystemFilenames = hashSetOf( + StartupMetadata.ZERO_FILE_NAME, + StartupMetadata.FIRST_FILE_NAME, + BackupMetadata.BACKUP_METADATA_FILE_NAME, + BackupMetadata.START_BACKUP_METADATA_FILE_NAME, + "xd.lck" + ) + if (delete){ + log.info("Removing blobs...") + val blobsDir = File(xodus.databaseDirectory, "blobs") + if (blobsDir.exists()) { + FileUtils.deleteDirectory(blobsDir) + } + log.info("Blobs removed") + + log.info("Removing text index...") + val textIndex = File(xodus.databaseDirectory, "textindex") + if (textIndex.exists()){ + FileUtils.deleteDirectory(textIndex) + } + log.info("Text index removed...") + + log.info("Cleanup xd files...") + val dbDirectory = File(xodus.databaseDirectory) + if (dbDirectory.exists() && dbDirectory.isDirectory){ + dbDirectory.listFiles { file -> + val name = file.name + name.endsWith(".xd") || xodusSystemFilenames.contains(name) + }?.forEach { + it.delete() + } + } + log.info("DB Files cleanup finished...") + } else { + log.info("Moving xodus files to backup directory") + val newRoot = File(xodus.databaseDirectory, "xodus_backup") + log.info("Moving blobs...") + val blobsDir = File(xodus.databaseDirectory, "blobs") + if (blobsDir.exists() && blobsDir.isDirectory){ + FileUtils.moveDirectory(blobsDir, File(newRoot, "blobs")) + } + log.info("Moving text-index...") + val textIndex = File(xodus.databaseDirectory, "textindex") + if (textIndex.exists()){ + FileUtils.moveDirectory(textIndex, File(newRoot, "textIndex")) + } + log.info("Moving xd files...") + val dbDirectory = File(xodus.databaseDirectory) + if (dbDirectory.exists() && dbDirectory.isDirectory){ + dbDirectory.listFiles { file -> + val name = file.name + name.endsWith(".xd") || xodusSystemFilenames.contains(name) + }?.forEach { + FileUtils.moveFile(it, File(newRoot, it.name)) + } + } + log.info("DB files moved") + } + } } private fun percent(value: Double): String = "%.${2}f".format(value * 100) + "%" diff --git a/settings.gradle.kts b/settings.gradle.kts index 2784fca5c..986e0559f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -32,7 +32,7 @@ dependencyResolutionManagement { version("commons-lang", "3.12.0") version("commons-compress", "1.22") version("bouncyCastle", "1.70") - version("commons-io", "2.11.0") + version("commons-io", "2.15.1") version("lucene", "8.10.0") version("fastutil", "8.5.12")