Skip to content

Commit

Permalink
Tool to export/import data to/from version neutral binary format was …
Browse files Browse the repository at this point in the history 
…added.
  • Loading branch information
@andrii0lomakin
andrii0lomakin committed Aug 22, 2024
1 parent 1d4c7c0 commit aadcf74
Show file tree
Hide file tree
Showing 10 changed files with 266 additions and 76 deletions.
8 changes: 4 additions & 4 deletions environment/src/main/java/jetbrains/exodus/env/DatabaseRoot.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,19 @@
import jetbrains.exodus.util.LightOutputStream;
import org.jetbrains.annotations.NotNull;

final class DatabaseRoot {
public final class DatabaseRoot {

static final byte DATABASE_ROOT_TYPE = 1;
public static final byte DATABASE_ROOT_TYPE = 1;

private static final long MAGIC_DIFF = 199L;

@NotNull
private final Loggable loggable;
private final long rootAddress;
private final int lastStructureId;
private final boolean isValid;
public final boolean isValid;

DatabaseRoot(@NotNull final Loggable loggable) {
public DatabaseRoot(@NotNull final Loggable loggable) {
this(loggable, loggable.getData().iterator());
}

Expand Down
236 changes: 174 additions & 62 deletions environment/src/main/java/jetbrains/exodus/env/EnvExportImport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,37 +18,40 @@
import jetbrains.exodus.ArrayByteIterable;

import java.io.*;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.List;
import java.util.zip.CRC32;

public class EnvExportImport {
public static void exportEnvironment(Path envPath, EnvironmentConfig envConf, Path exportPath) {
try (Environment env = Environments.newInstance(envPath.toFile(), envConf)) {
env.executeInReadonlyTransaction(txn -> {
if (Files.exists(exportPath)) {
throw new IllegalStateException("Export file already exists: " + exportPath);
}
public static void exportEnvironment(Path envPath, EnvironmentConfig envConf, Path exportPath) throws IOException {
System.out.printf("Exporting database located in path: %s into file: %s%n", envPath, exportPath);
if (Files.exists(exportPath)) {
throw new IllegalStateException("File already exists in path : " + exportPath);
}
if (!Files.exists(envPath)) {
throw new IllegalStateException("Database does not exist in path: " + envPath);
}

Path parent = exportPath.getParent();
if (parent != null) {
try {
Files.createDirectories(parent);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
try {
Files.createFile(exportPath);
} catch (IOException e) {
throw new RuntimeException(e);
}
Path parent = exportPath.getParent();
if (parent != null) {
Files.createDirectories(parent);
}
Files.createFile(exportPath);

int[] totalEntriesCount = new int[]{0};
try (Environment env = Environments.newInstance(envPath.toFile(), envConf)) {
env.executeInReadonlyTransaction(txn -> {
try {
try (OutputStream outputStream = Files.newOutputStream(exportPath)) {
try (DataOutputStream dataOutputStream = new DataOutputStream(outputStream)) {
final List<String> stores = env.getAllStoreNames(txn);
dataOutputStream.writeChar('V');
dataOutputStream.writeChar('S');

final List<String> stores = env.getAllStoreNames(txn);
for (String storeName : stores) {
final Store store = env.openStore(storeName, StoreConfig.USE_EXISTING, txn);

Expand All @@ -73,6 +76,7 @@ public static void exportEnvironment(Path envPath, EnvironmentConfig envConf, Pa
dataOutputStream.write(value);

entryCount++;
totalEntriesCount[0]++;
}
}

Expand All @@ -85,60 +89,168 @@ public static void exportEnvironment(Path envPath, EnvironmentConfig envConf, Pa
}
});
}

try (FileChannel fileChannel = FileChannel.open(exportPath, StandardOpenOption.READ,
StandardOpenOption.WRITE)) {
ByteBuffer totalEntriesCountBuffer = ByteBuffer.allocate(Integer.BYTES).putInt(totalEntriesCount[0]);
totalEntriesCountBuffer.flip();

fileChannel.position(fileChannel.size());
int written = 0;
while (written < Integer.BYTES) {
written += fileChannel.write(totalEntriesCountBuffer);
}

final ByteBuffer buffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, fileChannel.size());
CRC32 crc32 = new CRC32();
crc32.update(buffer);

fileChannel.position(fileChannel.size());
long fileCrc = crc32.getValue();
ByteBuffer crcBuffer = ByteBuffer.allocate(Long.BYTES).putLong(fileCrc);
crcBuffer.flip();

written = 0;
while (written < Long.BYTES) {
written += fileChannel.write(crcBuffer);
}

fileChannel.force(true);
}

System.out.printf("Export complete. %d entries were exported from database located in path: %s into file: %s%n",
totalEntriesCount[0], envPath, exportPath);
}

public static void importEnvironment(Path envPath, EnvironmentConfig envConf, Path importPath) {
public static void importEnvironment(Path envPath, EnvironmentConfig envConf, Path importPath) throws IOException {
if (Files.exists(envPath)) {
throw new IllegalStateException("Database already exists in path : " + envPath);
}
if (!Files.exists(importPath)) {
throw new IllegalStateException("Import file does not exist in path : " + importPath);
}

System.out.printf("Importing database located in file: %s into database located into path: %s%n", importPath, envPath);
int totalEntriesCount;

try (FileChannel fileChannel = FileChannel.open(importPath, StandardOpenOption.READ)) {
fileChannel.position(fileChannel.size() - Integer.BYTES - Long.BYTES);
ByteBuffer totalEntriesCountBuffer = ByteBuffer.allocate(Integer.BYTES);
int read = 0;
while (read < Integer.BYTES) {
int r = fileChannel.read(totalEntriesCountBuffer);
if (r < 0) {
throw new EOFException(importPath + " - unexpected end of file.");
}

read += r;
}
totalEntriesCountBuffer.flip();
totalEntriesCount = totalEntriesCountBuffer.getInt();

final ByteBuffer buffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0,
fileChannel.size() - Long.BYTES);

CRC32 crc32 = new CRC32();
crc32.update(buffer);
long fileCrc = crc32.getValue();

fileChannel.position(fileChannel.size() - Long.BYTES);
ByteBuffer crcBuffer = ByteBuffer.allocate(Long.BYTES);

read = 0;
while (read < Long.BYTES) {
int r = fileChannel.read(crcBuffer);
if (r < 0) {
throw new EOFException(importPath + " - unexpected end of file");
}

read += r;
}

crcBuffer.flip();
if (fileCrc != crcBuffer.getLong()) {
throw new IllegalStateException("Import file : " + importPath + " is broken, CRC mismatch.");
}
}

System.out.printf("Importing %d entries from file: %s into database located in path: %s%n", totalEntriesCount,
importPath, envPath);

try (Environment env = Environments.newInstance(envPath.toFile(), envConf)) {
env.executeInTransaction(txn -> {
try {
try (InputStream inputStream = Files.newInputStream(importPath)) {
try (DataInputStream dataInputStream = new DataInputStream(inputStream)) {
String currentStoreName = null;
Store currentStore = null;
int entryCount = 0;
try {
while (true) {
final String storeName = dataInputStream.readUTF();
String configName = dataInputStream.readUTF();

final StoreConfig storeConfig = StoreConfig.valueOf(configName);
if (currentStoreName == null) {
currentStoreName = storeName;
currentStore = env.openStore(storeName, storeConfig, txn);
} else if (!currentStoreName.equals(storeName)) {
System.out.println("Imported " + entryCount + " entries to store " + currentStoreName);

currentStoreName = storeName;
currentStore = env.openStore(storeName, storeConfig, txn);
entryCount = 0;
}
try {
try (InputStream inputStream = Files.newInputStream(importPath)) {
try (DataInputStream dataInputStream = new DataInputStream(inputStream)) {
char magic1 = dataInputStream.readChar();
char magic2 = dataInputStream.readChar();

final int keyLength = dataInputStream.readInt();
final int rawKeyLength = dataInputStream.readInt();
final byte[] key = new byte[rawKeyLength];
dataInputStream.readFully(key);
if (magic1 != 'V' || magic2 != 'S') {
throw new IllegalStateException(importPath + " - invalid export file format");
}

String currentStoreName = null;
Store currentStore = null;
int entryCount = 0;
Transaction txn = null;
try {
for (int i = 0; i < totalEntriesCount; i++) {
final String storeName = dataInputStream.readUTF();
String configName = dataInputStream.readUTF();

final int valueLength = dataInputStream.readInt();
final int rawValueLength = dataInputStream.readInt();
final byte[] value = new byte[rawValueLength];
dataInputStream.readFully(value);
final StoreConfig storeConfig = StoreConfig.valueOf(configName);
if (currentStoreName == null) {
currentStoreName = storeName;
txn = env.beginTransaction();
currentStore = env.openStore(storeName, storeConfig, txn);
} else if (!currentStoreName.equals(storeName)) {
System.out.println("Imported " + entryCount + " entries to store " + currentStoreName);

if (!txn.commit()) {
throw new IllegalStateException("Failed to commit transaction for store "
+ currentStoreName);
}

currentStore.put(txn, new ArrayByteIterable(key, keyLength),
new ArrayByteIterable(value, valueLength));
entryCount += 1;
txn = env.beginTransaction();
currentStoreName = storeName;
currentStore = env.openStore(storeName, storeConfig, txn);
entryCount = 0;
}
} catch (EOFException e) {
// end of file

final int keyLength = dataInputStream.readInt();
final int rawKeyLength = dataInputStream.readInt();
final byte[] key = new byte[rawKeyLength];
dataInputStream.readFully(key);

final int valueLength = dataInputStream.readInt();
final int rawValueLength = dataInputStream.readInt();
final byte[] value = new byte[rawValueLength];
dataInputStream.readFully(value);

currentStore.put(txn, new ArrayByteIterable(key, keyLength),
new ArrayByteIterable(value, valueLength));
entryCount += 1;
}
if (currentStoreName != null) {
System.out.println("Imported " + entryCount + " entries to store " + currentStoreName);
} catch (EOFException e) {
// end of file
}
if (currentStoreName != null) {
if (!txn.isFinished()) {
if (!txn.commit()) {
throw new IllegalStateException("Failed to commit transaction for store "
+ currentStoreName);
}
}

System.out.println("Imported " + entryCount + " entries to store " + currentStoreName);
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
}

System.out.printf("Import complete. Data located in file: %s imported to database located in path: %s%n", importPath,
envPath);
}
}
2 changes: 1 addition & 1 deletion environment/src/main/java/jetbrains/exodus/env/EnvironmentImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -802,7 +802,7 @@ public MetaTree getMetaTree() {
}
}

MetaTreeImpl getMetaTreeInternal() {
public MetaTreeImpl getMetaTreeInternal() {
return metaTree;
}

Expand Down
4 changes: 2 additions & 2 deletions environment/src/main/java/jetbrains/exodus/env/MetaTreeImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
import java.util.Collections;
import java.util.List;

final class MetaTreeImpl implements MetaTree {
public final class MetaTreeImpl implements MetaTree {

private static final int EMPTY_LOG_BOUND = 5;

Expand Down Expand Up @@ -158,7 +158,7 @@ public long rootAddress() {
return root;
}

LongIterator addressIterator() {
public LongIterator addressIterator() {
return tree.addressIterator();
}

Expand Down
6 changes: 4 additions & 2 deletions tools/src/main/kotlin/jetbrains/exodus/Main.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ fun main(args: Array<String>) {
printUsage()
}
when (args[0].lowercase()) {
"reflect" -> jetbrains.exodus.env.main(args.skipFirst)
"reflect" -> jetbrains.exodus.env.export.main(args.skipFirst)
"export" -> jetbrains.exodus.env.export.main(args.skipFirst)
"import" -> jetbrains.exodus.env.imp.main(args.skipFirst)
"refactorings" -> jetbrains.exodus.entityStore.main(args.skipFirst)
"scytale" -> jetbrains.exodus.crypto.main(args.skipFirst)
"vfs" -> jetbrains.exodus.vfs.main(args.skipFirst)
Expand All @@ -52,7 +54,7 @@ fun main(args: Array<String>) {

internal fun printUsage() {
println("Usage: <tool name> [tool parameters]")
println("Available tools: Reflect | Refactorings | Scytale | Vfs | EnvironmentJSConsole | EntityStoreJSConsole")
println("Available tools: reflect | refactorings | scytale | Vws | environmentJSConsole | entityStoreJSConsole | import | export")
exitProcess(1)
}

Expand Down
2 changes: 1 addition & 1 deletion tools/src/main/kotlin/jetbrains/exodus/crypto/Scytale.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import jetbrains.exodus.crypto.convert.*
import jetbrains.exodus.crypto.streamciphers.CHACHA_CIPHER_ID
import jetbrains.exodus.crypto.streamciphers.SALSA20_CIPHER_ID
import jetbrains.exodus.entitystore.PersistentEntityStoreImpl
import jetbrains.exodus.env.Reflect
import jetbrains.exodus.env.reflect.Reflect
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream
import java.io.BufferedOutputStream
import java.io.File
Expand Down
2 changes: 1 addition & 1 deletion tools/src/main/kotlin/jetbrains/exodus/entityStore/ApplyRefactorings.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
package jetbrains.exodus.entityStore

import jetbrains.exodus.entitystore.PersistentEntityStoreImpl
import jetbrains.exodus.env.Reflect
import jetbrains.exodus.env.reflect.Reflect
import java.io.File

fun main(args: Array<String>) {
Expand Down
Loading

0 comments on commit aadcf74

Please sign in to comment.