diff --git a/environment/src/main/java/jetbrains/exodus/log/Log.java b/environment/src/main/java/jetbrains/exodus/log/Log.java index b80254ec7..4194b4dad 100644 --- a/environment/src/main/java/jetbrains/exodus/log/Log.java +++ b/environment/src/main/java/jetbrains/exodus/log/Log.java @@ -53,6 +53,7 @@ public final class Log implements Closeable { final LogCache cache; private int logIdentity; + @SuppressWarnings("NullableProblems") @NotNull private TransactionalDataWriter bufferedWriter; /** @@ -356,7 +357,7 @@ ArrayByteIterable getHighPage(long alignedAddress) { return bufferedWriter.getHighPage(alignedAddress); } - public final int getCachePageSize() { + final int getCachePageSize() { return cachePageSize; } @@ -614,7 +615,7 @@ public void removeFile(final long address, @NotNull final RemoveBlockType rbt) { } } - public void truncateFile(final long address, final long length) { + private void truncateFile(final long address, final long length) { // truncate physical file reader.truncateBlock(address, length); // clear cache @@ -632,8 +633,11 @@ public void truncateFile(final long address, final long length) { * loggable can begin in one file and end in another. Also, this simplifies reading algorithm: * if we started reading by address it definitely should finish within current file. */ - public void padWithNulls() { + void padWithNulls() { long bytesToWrite = fileLengthBound - getLastFileLength(); + if (bytesToWrite == 0L) { + throw new ExodusException("Nothing to pad"); + } if (bytesToWrite >= cachePageSize) { final ArrayByteIterable cachedTailPage = LogCache.getCachedTailPage(cachePageSize); if (cachedTailPage != null) { @@ -645,8 +649,13 @@ public void padWithNulls() { } while (bytesToWrite >= cachePageSize); } } - while (bytesToWrite-- > 0) { - writeContinuously(NullLoggable.create()); + if (bytesToWrite == 0) { + bufferedWriter.commit(); + createNewFileIfNecessary(); + } else { + while (bytesToWrite-- > 0) { + writeContinuously(NullLoggable.create()); + } } } @@ -808,21 +817,7 @@ public long writeContinuously(final Loggable loggable) { } bufferedWriter.commit(); highAddress += recordLength; - if (getLastFileLength() == 0 || System.currentTimeMillis() > lastSyncTicks + config.getSyncPeriod()) { - flush(true); - if (getLastFileLength() == 0) { - bufferedWriter.close(); - if (config.isFullFileReadonly()) { - final Long lastFile; - synchronized (blockAddrs) { - lastFile = blockAddrs.getMaximum(); - } - if (lastFile != null) { - reader.getBlock(lastFile).setWritable(false); - } - } - } - } + createNewFileIfNecessary(); return result; } catch (Throwable e) { highAddress = result; @@ -834,6 +829,25 @@ public long writeContinuously(final Loggable loggable) { } } + private void createNewFileIfNecessary() { + final boolean shouldCreateNewFile = getLastFileLength() == 0; + if (shouldCreateNewFile) { + flush(true); + bufferedWriter.close(); + if (config.isFullFileReadonly()) { + final Long lastFile; + synchronized (blockAddrs) { + lastFile = blockAddrs.getMaximum(); + } + if (lastFile != null) { + reader.getBlock(lastFile).setWritable(false); + } + } + } else if (System.currentTimeMillis() > lastSyncTicks + config.getSyncPeriod()) { + flush(true); + } + } + /** * Sets LogTestConfig. * Is destined for tests only, please don't set a not-null value in application code. diff --git a/environment/src/test/java/jetbrains/exodus/tree/TreePutTest.java b/environment/src/test/java/jetbrains/exodus/tree/TreePutTest.java index c76a78ed0..50f014bf4 100644 --- a/environment/src/test/java/jetbrains/exodus/tree/TreePutTest.java +++ b/environment/src/test/java/jetbrains/exodus/tree/TreePutTest.java @@ -16,10 +16,12 @@ package jetbrains.exodus.tree; import jetbrains.exodus.ByteIterable; +import jetbrains.exodus.TestFor; import jetbrains.exodus.TestUtil; import jetbrains.exodus.bindings.IntegerBinding; import jetbrains.exodus.bindings.LongBinding; import jetbrains.exodus.core.dataStructures.hash.IntHashMap; +import jetbrains.exodus.core.dataStructures.hash.IntHashSet; import org.junit.Assert; import org.junit.Test; @@ -312,7 +314,7 @@ public void testPutRandomWithoutDuplicates() throws Throwable { final IntHashMap map = new IntHashMap<>(); final int count = 200000; - TestUtil.time("Put took ", new Runnable() { + TestUtil.time("put()", new Runnable() { @Override public void run() { for (int i = 0; i < count; ++i) { @@ -325,7 +327,7 @@ public void run() { }); Assert.assertEquals(map.size(), tm.getSize()); - TestUtil.time("Get took ", new Runnable() { + TestUtil.time("get()", new Runnable() { @Override public void run() { for (final Map.Entry entry : map.entrySet()) { @@ -344,7 +346,7 @@ public void testPutRandomWithoutDuplicates2() throws Throwable { final IntHashMap map = new IntHashMap<>(); final int count = 200000; - TestUtil.time("Put took ", new Runnable() { + TestUtil.time("put()", new Runnable() { @Override public void run() { for (int i = 0; i < count; ++i) { @@ -362,7 +364,7 @@ public void run() { Assert.assertEquals(map.size(), t.getSize()); - TestUtil.time("Get took ", new Runnable() { + TestUtil.time("get()", new Runnable() { @Override public void run() { for (final Map.Entry entry : map.entrySet()) { @@ -379,14 +381,14 @@ public void testPutRightRandomWithoutDuplicates() throws Throwable { tm = createMutableTree(false, 1); final IntHashMap map = new IntHashMap<>(); - final int count = 200000; + final int count = 99999; - TestUtil.time("PutRight took ", new Runnable() { + TestUtil.time("putRight()", new Runnable() { @Override public void run() { for (int i = 0; i < count; ++i) { final String value = Integer.toString(i); - tm.put(key(Integer.toString(i)), value(value)); + tm.putRight(key(i), value(value)); map.put(i, value); } } @@ -398,13 +400,13 @@ public void run() { Assert.assertEquals(map.size(), t.getSize()); - TestUtil.time("Get took ", new Runnable() { + TestUtil.time("get()", new Runnable() { @Override public void run() { for (final Map.Entry entry : map.entrySet()) { final Integer key = entry.getKey(); final String value = entry.getValue(); - valueEquals(value, t.get(key(Integer.toString(key)))); + valueEquals(value, t.get(key(key))); } } }); @@ -417,7 +419,7 @@ public void testAddRandomWithoutDuplicates() throws Throwable { final IntHashMap map = new IntHashMap<>(); final int count = 50000; - TestUtil.time("Add took ", new Runnable() { + TestUtil.time("add()", new Runnable() { @Override public void run() { for (int i = 0; i < count; ++i) { @@ -437,7 +439,7 @@ public void run() { Assert.assertEquals(map.size(), t.getSize()); - TestUtil.time("Get took ", new Runnable() { + TestUtil.time("get()", new Runnable() { @Override public void run() { for (final Map.Entry entry : map.entrySet()) { @@ -450,7 +452,7 @@ public void run() { tm = t.getMutableCopy(); - TestUtil.time("Failing add took ", new Runnable() { + TestUtil.time("Failing add()", new Runnable() { @Override public void run() { for (final Map.Entry entry : map.entrySet()) { @@ -461,4 +463,45 @@ public void run() { } }); } + + @Test + @TestFor(issues = "XD-539") + public void createHugeTree() throws Throwable { + if (Runtime.getRuntime().maxMemory() < 4000000000L) { + return; + } + + tm = createMutableTree(false, 1); + + final IntHashSet set = new IntHashSet(); + final int count = 20000; + final StringBuilder builder = new StringBuilder("value"); + + TestUtil.time("put()", new Runnable() { + @Override + public void run() { + for (int i = 0; i < count; ++i) { + tm.put(key(Integer.toString(i)), value(builder.toString())); + set.add(i); + builder.append(i); + } + } + }); + + final long address = tm.save(); + System.out.println("Log size: " + tm.getLog().getHighAddress()); + reopen(); + t = openTree(address, false); + + Assert.assertEquals(set.size(), t.getSize()); + + TestUtil.time("get()", new Runnable() { + @Override + public void run() { + for (Integer i : set) { + assertTrue(t.hasKey(key(Integer.toString(i)))); + } + } + }); + } }