From aadcf7475c037e5d349f245843a0fa71a503ab4f Mon Sep 17 00:00:00 2001 From: andrii0lomakin Date: Thu, 22 Aug 2024 15:29:04 +0200 Subject: [PATCH] Tool to export/import data to/from version neutral binary format was added. --- .../jetbrains/exodus/env/DatabaseRoot.java | 8 +- .../jetbrains/exodus/env/EnvExportImport.java | 236 +++++++++++++----- .../jetbrains/exodus/env/EnvironmentImpl.java | 2 +- .../jetbrains/exodus/env/MetaTreeImpl.java | 4 +- .../src/main/kotlin/jetbrains/exodus/Main.kt | 6 +- .../kotlin/jetbrains/exodus/crypto/Scytale.kt | 2 +- .../exodus/entityStore/ApplyRefactorings.kt | 2 +- .../jetbrains/exodus/env/export/Export.kt | 37 +++ .../kotlin/jetbrains/exodus/env/imp/Import.kt | 38 +++ .../exodus/env/{ => reflect}/Reflect.kt | 7 +- 10 files changed, 266 insertions(+), 76 deletions(-) create mode 100644 tools/src/main/kotlin/jetbrains/exodus/env/export/Export.kt create mode 100644 tools/src/main/kotlin/jetbrains/exodus/env/imp/Import.kt rename tools/src/main/kotlin/jetbrains/exodus/env/{ => reflect}/Reflect.kt (99%) diff --git a/environment/src/main/java/jetbrains/exodus/env/DatabaseRoot.java b/environment/src/main/java/jetbrains/exodus/env/DatabaseRoot.java index 282345ab4..1ed967e36 100644 --- a/environment/src/main/java/jetbrains/exodus/env/DatabaseRoot.java +++ b/environment/src/main/java/jetbrains/exodus/env/DatabaseRoot.java @@ -22,9 +22,9 @@ import jetbrains.exodus.util.LightOutputStream; import org.jetbrains.annotations.NotNull; -final class DatabaseRoot { +public final class DatabaseRoot { - static final byte DATABASE_ROOT_TYPE = 1; + public static final byte DATABASE_ROOT_TYPE = 1; private static final long MAGIC_DIFF = 199L; @@ -32,9 +32,9 @@ final class DatabaseRoot { private final Loggable loggable; private final long rootAddress; private final int lastStructureId; - private final boolean isValid; + public final boolean isValid; - DatabaseRoot(@NotNull final Loggable loggable) { + public DatabaseRoot(@NotNull final Loggable loggable) { this(loggable, loggable.getData().iterator()); } diff --git a/environment/src/main/java/jetbrains/exodus/env/EnvExportImport.java b/environment/src/main/java/jetbrains/exodus/env/EnvExportImport.java index 066351cca..adda575f7 100644 --- a/environment/src/main/java/jetbrains/exodus/env/EnvExportImport.java +++ b/environment/src/main/java/jetbrains/exodus/env/EnvExportImport.java @@ -18,37 +18,40 @@ import jetbrains.exodus.ArrayByteIterable; import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.List; +import java.util.zip.CRC32; public class EnvExportImport { - public static void exportEnvironment(Path envPath, EnvironmentConfig envConf, Path exportPath) { - try (Environment env = Environments.newInstance(envPath.toFile(), envConf)) { - env.executeInReadonlyTransaction(txn -> { - if (Files.exists(exportPath)) { - throw new IllegalStateException("Export file already exists: " + exportPath); - } + public static void exportEnvironment(Path envPath, EnvironmentConfig envConf, Path exportPath) throws IOException { + System.out.printf("Exporting database located in path: %s into file: %s%n", envPath, exportPath); + if (Files.exists(exportPath)) { + throw new IllegalStateException("File already exists in path : " + exportPath); + } + if (!Files.exists(envPath)) { + throw new IllegalStateException("Database does not exist in path: " + envPath); + } - Path parent = exportPath.getParent(); - if (parent != null) { - try { - Files.createDirectories(parent); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - try { - Files.createFile(exportPath); - } catch (IOException e) { - throw new RuntimeException(e); - } + Path parent = exportPath.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + Files.createFile(exportPath); + int[] totalEntriesCount = new int[]{0}; + try (Environment env = Environments.newInstance(envPath.toFile(), envConf)) { + env.executeInReadonlyTransaction(txn -> { try { try (OutputStream outputStream = Files.newOutputStream(exportPath)) { try (DataOutputStream dataOutputStream = new DataOutputStream(outputStream)) { - final List stores = env.getAllStoreNames(txn); + dataOutputStream.writeChar('V'); + dataOutputStream.writeChar('S'); + final List stores = env.getAllStoreNames(txn); for (String storeName : stores) { final Store store = env.openStore(storeName, StoreConfig.USE_EXISTING, txn); @@ -73,6 +76,7 @@ public static void exportEnvironment(Path envPath, EnvironmentConfig envConf, Pa dataOutputStream.write(value); entryCount++; + totalEntriesCount[0]++; } } @@ -85,60 +89,168 @@ public static void exportEnvironment(Path envPath, EnvironmentConfig envConf, Pa } }); } + + try (FileChannel fileChannel = FileChannel.open(exportPath, StandardOpenOption.READ, + StandardOpenOption.WRITE)) { + ByteBuffer totalEntriesCountBuffer = ByteBuffer.allocate(Integer.BYTES).putInt(totalEntriesCount[0]); + totalEntriesCountBuffer.flip(); + + fileChannel.position(fileChannel.size()); + int written = 0; + while (written < Integer.BYTES) { + written += fileChannel.write(totalEntriesCountBuffer); + } + + final ByteBuffer buffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size()); + CRC32 crc32 = new CRC32(); + crc32.update(buffer); + + fileChannel.position(fileChannel.size()); + long fileCrc = crc32.getValue(); + ByteBuffer crcBuffer = ByteBuffer.allocate(Long.BYTES).putLong(fileCrc); + crcBuffer.flip(); + + written = 0; + while (written < Long.BYTES) { + written += fileChannel.write(crcBuffer); + } + + fileChannel.force(true); + } + + System.out.printf("Export complete. %d entries were exported from database located in path: %s into file: %s%n", + totalEntriesCount[0], envPath, exportPath); } - public static void importEnvironment(Path envPath, EnvironmentConfig envConf, Path importPath) { + public static void importEnvironment(Path envPath, EnvironmentConfig envConf, Path importPath) throws IOException { + if (Files.exists(envPath)) { + throw new IllegalStateException("Database already exists in path : " + envPath); + } + if (!Files.exists(importPath)) { + throw new IllegalStateException("Import file does not exist in path : " + importPath); + } + + System.out.printf("Importing database located in file: %s into database located into path: %s%n", importPath, envPath); + int totalEntriesCount; + + try (FileChannel fileChannel = FileChannel.open(importPath, StandardOpenOption.READ)) { + fileChannel.position(fileChannel.size() - Integer.BYTES - Long.BYTES); + ByteBuffer totalEntriesCountBuffer = ByteBuffer.allocate(Integer.BYTES); + int read = 0; + while (read < Integer.BYTES) { + int r = fileChannel.read(totalEntriesCountBuffer); + if (r < 0) { + throw new EOFException(importPath + " - unexpected end of file."); + } + + read += r; + } + totalEntriesCountBuffer.flip(); + totalEntriesCount = totalEntriesCountBuffer.getInt(); + + final ByteBuffer buffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, + fileChannel.size() - Long.BYTES); + + CRC32 crc32 = new CRC32(); + crc32.update(buffer); + long fileCrc = crc32.getValue(); + + fileChannel.position(fileChannel.size() - Long.BYTES); + ByteBuffer crcBuffer = ByteBuffer.allocate(Long.BYTES); + + read = 0; + while (read < Long.BYTES) { + int r = fileChannel.read(crcBuffer); + if (r < 0) { + throw new EOFException(importPath + " - unexpected end of file"); + } + + read += r; + } + + crcBuffer.flip(); + if (fileCrc != crcBuffer.getLong()) { + throw new IllegalStateException("Import file : " + importPath + " is broken, CRC mismatch."); + } + } + + System.out.printf("Importing %d entries from file: %s into database located in path: %s%n", totalEntriesCount, + importPath, envPath); + try (Environment env = Environments.newInstance(envPath.toFile(), envConf)) { - env.executeInTransaction(txn -> { - try { - try (InputStream inputStream = Files.newInputStream(importPath)) { - try (DataInputStream dataInputStream = new DataInputStream(inputStream)) { - String currentStoreName = null; - Store currentStore = null; - int entryCount = 0; - try { - while (true) { - final String storeName = dataInputStream.readUTF(); - String configName = dataInputStream.readUTF(); - - final StoreConfig storeConfig = StoreConfig.valueOf(configName); - if (currentStoreName == null) { - currentStoreName = storeName; - currentStore = env.openStore(storeName, storeConfig, txn); - } else if (!currentStoreName.equals(storeName)) { - System.out.println("Imported " + entryCount + " entries to store " + currentStoreName); - - currentStoreName = storeName; - currentStore = env.openStore(storeName, storeConfig, txn); - entryCount = 0; - } + try { + try (InputStream inputStream = Files.newInputStream(importPath)) { + try (DataInputStream dataInputStream = new DataInputStream(inputStream)) { + char magic1 = dataInputStream.readChar(); + char magic2 = dataInputStream.readChar(); - final int keyLength = dataInputStream.readInt(); - final int rawKeyLength = dataInputStream.readInt(); - final byte[] key = new byte[rawKeyLength]; - dataInputStream.readFully(key); + if (magic1 != 'V' || magic2 != 'S') { + throw new IllegalStateException(importPath + " - invalid export file format"); + } + + String currentStoreName = null; + Store currentStore = null; + int entryCount = 0; + Transaction txn = null; + try { + for (int i = 0; i < totalEntriesCount; i++) { + final String storeName = dataInputStream.readUTF(); + String configName = dataInputStream.readUTF(); - final int valueLength = dataInputStream.readInt(); - final int rawValueLength = dataInputStream.readInt(); - final byte[] value = new byte[rawValueLength]; - dataInputStream.readFully(value); + final StoreConfig storeConfig = StoreConfig.valueOf(configName); + if (currentStoreName == null) { + currentStoreName = storeName; + txn = env.beginTransaction(); + currentStore = env.openStore(storeName, storeConfig, txn); + } else if (!currentStoreName.equals(storeName)) { + System.out.println("Imported " + entryCount + " entries to store " + currentStoreName); + + if (!txn.commit()) { + throw new IllegalStateException("Failed to commit transaction for store " + + currentStoreName); + } - currentStore.put(txn, new ArrayByteIterable(key, keyLength), - new ArrayByteIterable(value, valueLength)); - entryCount += 1; + txn = env.beginTransaction(); + currentStoreName = storeName; + currentStore = env.openStore(storeName, storeConfig, txn); + entryCount = 0; } - } catch (EOFException e) { - // end of file + + final int keyLength = dataInputStream.readInt(); + final int rawKeyLength = dataInputStream.readInt(); + final byte[] key = new byte[rawKeyLength]; + dataInputStream.readFully(key); + + final int valueLength = dataInputStream.readInt(); + final int rawValueLength = dataInputStream.readInt(); + final byte[] value = new byte[rawValueLength]; + dataInputStream.readFully(value); + + currentStore.put(txn, new ArrayByteIterable(key, keyLength), + new ArrayByteIterable(value, valueLength)); + entryCount += 1; } - if (currentStoreName != null) { - System.out.println("Imported " + entryCount + " entries to store " + currentStoreName); + } catch (EOFException e) { + // end of file + } + if (currentStoreName != null) { + if (!txn.isFinished()) { + if (!txn.commit()) { + throw new IllegalStateException("Failed to commit transaction for store " + + currentStoreName); + } } + + System.out.println("Imported " + entryCount + " entries to store " + currentStoreName); } } - } catch (IOException e) { - throw new RuntimeException(e); } - }); + } catch (IOException e) { + throw new RuntimeException(e); + } } + + System.out.printf("Import complete. Data located in file: %s imported to database located in path: %s%n", importPath, + envPath); } } diff --git a/environment/src/main/java/jetbrains/exodus/env/EnvironmentImpl.java b/environment/src/main/java/jetbrains/exodus/env/EnvironmentImpl.java index 7f9978cb5..7322c8135 100644 --- a/environment/src/main/java/jetbrains/exodus/env/EnvironmentImpl.java +++ b/environment/src/main/java/jetbrains/exodus/env/EnvironmentImpl.java @@ -802,7 +802,7 @@ public MetaTree getMetaTree() { } } - MetaTreeImpl getMetaTreeInternal() { + public MetaTreeImpl getMetaTreeInternal() { return metaTree; } diff --git a/environment/src/main/java/jetbrains/exodus/env/MetaTreeImpl.java b/environment/src/main/java/jetbrains/exodus/env/MetaTreeImpl.java index 1c9b6cf12..30b21bc0e 100644 --- a/environment/src/main/java/jetbrains/exodus/env/MetaTreeImpl.java +++ b/environment/src/main/java/jetbrains/exodus/env/MetaTreeImpl.java @@ -33,7 +33,7 @@ import java.util.Collections; import java.util.List; -final class MetaTreeImpl implements MetaTree { +public final class MetaTreeImpl implements MetaTree { private static final int EMPTY_LOG_BOUND = 5; @@ -158,7 +158,7 @@ public long rootAddress() { return root; } - LongIterator addressIterator() { + public LongIterator addressIterator() { return tree.addressIterator(); } diff --git a/tools/src/main/kotlin/jetbrains/exodus/Main.kt b/tools/src/main/kotlin/jetbrains/exodus/Main.kt index 4aeb7eff2..46b6766e4 100644 --- a/tools/src/main/kotlin/jetbrains/exodus/Main.kt +++ b/tools/src/main/kotlin/jetbrains/exodus/Main.kt @@ -31,7 +31,9 @@ fun main(args: Array) { printUsage() } when (args[0].lowercase()) { - "reflect" -> jetbrains.exodus.env.main(args.skipFirst) + "reflect" -> jetbrains.exodus.env.export.main(args.skipFirst) + "export" -> jetbrains.exodus.env.export.main(args.skipFirst) + "import" -> jetbrains.exodus.env.imp.main(args.skipFirst) "refactorings" -> jetbrains.exodus.entityStore.main(args.skipFirst) "scytale" -> jetbrains.exodus.crypto.main(args.skipFirst) "vfs" -> jetbrains.exodus.vfs.main(args.skipFirst) @@ -52,7 +54,7 @@ fun main(args: Array) { internal fun printUsage() { println("Usage: [tool parameters]") - println("Available tools: Reflect | Refactorings | Scytale | Vfs | EnvironmentJSConsole | EntityStoreJSConsole") + println("Available tools: reflect | refactorings | scytale | Vws | environmentJSConsole | entityStoreJSConsole | import | export") exitProcess(1) } diff --git a/tools/src/main/kotlin/jetbrains/exodus/crypto/Scytale.kt b/tools/src/main/kotlin/jetbrains/exodus/crypto/Scytale.kt index ca90bde77..a94789365 100644 --- a/tools/src/main/kotlin/jetbrains/exodus/crypto/Scytale.kt +++ b/tools/src/main/kotlin/jetbrains/exodus/crypto/Scytale.kt @@ -19,7 +19,7 @@ import jetbrains.exodus.crypto.convert.* import jetbrains.exodus.crypto.streamciphers.CHACHA_CIPHER_ID import jetbrains.exodus.crypto.streamciphers.SALSA20_CIPHER_ID import jetbrains.exodus.entitystore.PersistentEntityStoreImpl -import jetbrains.exodus.env.Reflect +import jetbrains.exodus.env.reflect.Reflect import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream import java.io.BufferedOutputStream import java.io.File diff --git a/tools/src/main/kotlin/jetbrains/exodus/entityStore/ApplyRefactorings.kt b/tools/src/main/kotlin/jetbrains/exodus/entityStore/ApplyRefactorings.kt index 143588fe1..2e199b6ac 100644 --- a/tools/src/main/kotlin/jetbrains/exodus/entityStore/ApplyRefactorings.kt +++ b/tools/src/main/kotlin/jetbrains/exodus/entityStore/ApplyRefactorings.kt @@ -16,7 +16,7 @@ package jetbrains.exodus.entityStore import jetbrains.exodus.entitystore.PersistentEntityStoreImpl -import jetbrains.exodus.env.Reflect +import jetbrains.exodus.env.reflect.Reflect import java.io.File fun main(args: Array) { diff --git a/tools/src/main/kotlin/jetbrains/exodus/env/export/Export.kt b/tools/src/main/kotlin/jetbrains/exodus/env/export/Export.kt new file mode 100644 index 000000000..6b65ec43f --- /dev/null +++ b/tools/src/main/kotlin/jetbrains/exodus/env/export/Export.kt @@ -0,0 +1,37 @@ +/* + * Copyright 2010 - 2024 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package jetbrains.exodus.env.export + +import jetbrains.exodus.env.EnvExportImport +import jetbrains.exodus.env.EnvironmentConfig +import java.nio.file.Paths +import kotlin.system.exitProcess + +fun main(args: Array) { + if (args.size < 2) { + printUsage() + } + + val envPath = args[0] + val exportPath = args[1] + + EnvExportImport.exportEnvironment(Paths.get(envPath), EnvironmentConfig(), Paths.get(exportPath)) +} + +private fun printUsage() { + println("Usage: export ") + exitProcess(1) +} diff --git a/tools/src/main/kotlin/jetbrains/exodus/env/imp/Import.kt b/tools/src/main/kotlin/jetbrains/exodus/env/imp/Import.kt new file mode 100644 index 000000000..b35f920ab --- /dev/null +++ b/tools/src/main/kotlin/jetbrains/exodus/env/imp/Import.kt @@ -0,0 +1,38 @@ +/* + * Copyright 2010 - 2024 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package jetbrains.exodus.env.imp + +import jetbrains.exodus.env.EnvExportImport +import jetbrains.exodus.env.EnvironmentConfig +import java.nio.file.Paths +import kotlin.system.exitProcess + +fun main(args: Array) { + if (args.size < 2) { + printUsage() + } + + val envPath = args[0] + val importPath = args[1] + + EnvExportImport.importEnvironment(Paths.get(envPath), EnvironmentConfig(), Paths.get(importPath)) + +} + +private fun printUsage() { + println("Usage: import ") + exitProcess(1) +} \ No newline at end of file diff --git a/tools/src/main/kotlin/jetbrains/exodus/env/Reflect.kt b/tools/src/main/kotlin/jetbrains/exodus/env/reflect/Reflect.kt similarity index 99% rename from tools/src/main/kotlin/jetbrains/exodus/env/Reflect.kt rename to tools/src/main/kotlin/jetbrains/exodus/env/reflect/Reflect.kt index 7cd3273ae..016a3d455 100644 --- a/tools/src/main/kotlin/jetbrains/exodus/env/Reflect.kt +++ b/tools/src/main/kotlin/jetbrains/exodus/env/reflect/Reflect.kt @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package jetbrains.exodus.env +package jetbrains.exodus.env.reflect import jetbrains.exodus.ExodusException import jetbrains.exodus.bindings.IntegerBinding import jetbrains.exodus.bindings.StringBinding import jetbrains.exodus.core.dataStructures.hash.IntHashMap import jetbrains.exodus.core.dataStructures.hash.LinkedHashSet +import jetbrains.exodus.env.* import jetbrains.exodus.gc.GarbageCollector import jetbrains.exodus.io.FileDataReader import jetbrains.exodus.io.FileDataWriter @@ -121,8 +122,8 @@ fun main(args: Array) { exitProcess(0) } -internal fun printUsage() { - println("Usage: Reflect [-options] [environment path 2]") +private fun printUsage() { + println("Usage: reflect [-options] [environment path 2]") println("Options:") println(" -ls collect Log Stats") println(" -r validate Roots")