From a278936af3b46ab9912a11aa38a6070cb90c148b Mon Sep 17 00:00:00 2001 From: Atanas Atanasov Date: Fri, 6 Dec 2024 16:14:46 +0200 Subject: [PATCH] feat: new BlockWriter implementation for block-as-file (#355) Signed-off-by: Atanas Atanasov --- .../block/common/utils/Preconditions.java | 88 +++++-- .../block/common/utils/PreconditionsTest.java | 38 +++ server/build.gradle.kts | 2 + server/docker/logging.properties | 2 +- server/docker/update-env.sh | 2 + server/docs/configuration.md | 20 +- .../com/hedera/block/server/BlockNodeApp.java | 1 - .../com/hedera/block/server/Constants.java | 7 +- .../ServerMappedConfigSourceInitializer.java | 4 +- .../block/server/metrics/MetricsService.java | 4 +- .../server/metrics/MetricsServiceImpl.java | 26 +- .../PersistenceInjectionModule.java | 101 ++++++-- .../StreamPersistenceHandlerImpl.java | 7 +- .../storage/PersistenceStorageConfig.java | 140 ++++++++--- .../path/BlockAsLocalDirPathResolver.java | 59 +++++ .../path/BlockAsLocalFilePathResolver.java | 65 +++++ .../storage/path/BlockPathResolver.java | 39 +++ .../storage/path/NoOpBlockPathResolver.java | 52 ++++ .../storage/read/BlockAsDirReaderBuilder.java | 75 ------ ...Reader.java => BlockAsLocalDirReader.java} | 75 +++--- .../storage/read/BlockAsLocalFileReader.java | 50 ++++ .../persistence/storage/read/BlockReader.java | 8 +- .../storage/read/LocalBlockReader.java | 25 ++ .../storage/read/NoOpBlockReader.java | 50 ++++ ...mover.java => BlockAsLocalDirRemover.java} | 50 ++-- .../remove/BlockAsLocalFileRemover.java | 54 ++++ .../storage/remove/BlockRemover.java | 6 +- .../storage/remove/LocalBlockRemover.java | 23 ++ .../storage/remove/NoOpBlockRemover.java | 45 ++++ .../write/BlockAsDirWriterBuilder.java | 101 -------- ...Writer.java => BlockAsLocalDirWriter.java} | 178 +++++++------ .../storage/write/BlockAsLocalFileWriter.java | 115 +++++++++ .../storage/write/BlockWriter.java | 16 +- .../storage/write/LocalBlockWriter.java | 23 ++ .../storage/write/NoOpBlockWriter.java | 33 +-- server/src/main/java/module-info.java | 1 + server/src/main/resources/app.properties | 6 + ...rverMappedConfigSourceInitializerTest.java | 4 +- .../server/grpc/BlockAccessServiceTest.java | 36 +-- .../PbjBlockStreamServiceIntegrationTest.java | 43 ++-- .../PersistenceInjectionModuleTest.java | 233 +++++++++++++---- .../storage/PersistenceStorageConfigTest.java | 231 +++++++++++++---- .../path/BlockAsLocalDirPathResolverTest.java | 135 ++++++++++ .../BlockAsLocalFilePathResolverTest.java | 177 +++++++++++++ .../path/NoOpBlockPathResolverTest.java | 116 +++++++++ ...st.java => BlockAsLocalDirReaderTest.java} | 144 +++++++---- .../read/BlockAsLocalFileReaderTest.java | 127 ++++++++++ .../storage/read/NoOpBlockReaderTest.java | 117 +++++++++ .../storage/remove/BlockAsDirRemoverTest.java | 95 ------- .../remove/BlockAsLocalDirRemoverTest.java | 148 +++++++++++ .../remove/BlockAsLocalFileRemoverTest.java | 127 ++++++++++ .../storage/remove/NoOpBlockRemoverTest.java | 111 ++++++++ ...st.java => BlockAsLocalDirWriterTest.java} | 220 ++++++++-------- .../write/BlockAsLocalFileWriterTest.java | 236 ++++++++++++++++++ .../storage/write/NoOpBlockWriterTest.java | 77 ++++++ .../block/server/util/PersistTestUtils.java | 2 +- .../block/server/util/TestConfigUtil.java | 14 -- server/src/test/resources/app.properties | 2 + .../PositiveDataPersistenceTests.java | 7 +- 59 files changed, 3133 insertions(+), 860 deletions(-) create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolver.java create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolver.java create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockPathResolver.java create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolver.java delete mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java rename server/src/main/java/com/hedera/block/server/persistence/storage/read/{BlockAsDirReader.java => BlockAsLocalDirReader.java} (81%) create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReader.java create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/read/LocalBlockReader.java create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReader.java rename server/src/main/java/com/hedera/block/server/persistence/storage/remove/{BlockAsDirRemover.java => BlockAsLocalDirRemover.java} (51%) create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemover.java create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/remove/LocalBlockRemover.java create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemover.java delete mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java rename server/src/main/java/com/hedera/block/server/persistence/storage/write/{BlockAsDirWriter.java => BlockAsLocalDirWriter.java} (57%) create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriter.java create mode 100644 server/src/main/java/com/hedera/block/server/persistence/storage/write/LocalBlockWriter.java create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolverTest.java create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolverTest.java create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolverTest.java rename server/src/test/java/com/hedera/block/server/persistence/storage/read/{BlockAsDirReaderTest.java => BlockAsLocalDirReaderTest.java} (61%) create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReaderTest.java create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReaderTest.java delete mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemoverTest.java create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemoverTest.java create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemoverTest.java rename server/src/test/java/com/hedera/block/server/persistence/storage/write/{BlockAsDirWriterTest.java => BlockAsLocalDirWriterTest.java} (60%) create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriterTest.java create mode 100644 server/src/test/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriterTest.java diff --git a/common/src/main/java/com/hedera/block/common/utils/Preconditions.java b/common/src/main/java/com/hedera/block/common/utils/Preconditions.java index 08be1e517..348a9c69f 100644 --- a/common/src/main/java/com/hedera/block/common/utils/Preconditions.java +++ b/common/src/main/java/com/hedera/block/common/utils/Preconditions.java @@ -19,7 +19,7 @@ import java.util.Objects; /** A utility class used to assert various preconditions. */ -public final class Preconditions { +public final class Preconditions { // @todo(381) change the APIs to accept non-null error messages /** * This method asserts a given {@link String} is not blank, meaning it is * not {@code null} or does not contain only whitespaces as defined by @@ -75,6 +75,65 @@ public static int requirePositive(final int toCheck) { return requirePositive(toCheck, null); } + /** + * This method asserts a given integer is a positive. An integer is positive + * if it is NOT equal to zero and is greater than zero. + * + * @param toCheck the integer to check if it is a positive power of two + * @param errorMessage the error message to be used in the exception if the + * input integer to check is not positive, if null, a default message will + * be used + * @return the number to check if it is positive + * @throws IllegalArgumentException if the input number to check is not + * positive + */ + public static int requirePositive(final int toCheck, final String errorMessage) { + if (0 >= toCheck) { + final String message = Objects.isNull(errorMessage) + ? "The input integer [%d] is required be positive.".formatted(toCheck) + : errorMessage; + throw new IllegalArgumentException(message); + } else { + return toCheck; + } + } + + /** + * This method asserts a given long is a positive. A long is positive + * if it is NOT equal to zero and is greater than zero. + * + * @param toCheck the long to check if it is a positive power of two + * @return the long to check if it is positive + * @throws IllegalArgumentException if the input long to check is not + * positive + */ + public static long requirePositive(final long toCheck) { + return requirePositive(toCheck, null); + } + + /** + * This method asserts a given long is a positive. A long is positive + * if it is NOT equal to zero and is greater than zero. + * + * @param toCheck the long to check if it is a positive power of two + * @param errorMessage the error message to be used in the exception if the + * input long to check is not positive, if null, a default message will + * be used + * @return the long to check if it is positive + * @throws IllegalArgumentException if the input long to check is not + * positive + */ + public static long requirePositive(final long toCheck, final String errorMessage) { + if (0L >= toCheck) { + final String message = Objects.isNull(errorMessage) + ? "The input long [%d] is required be positive.".formatted(toCheck) + : errorMessage; + throw new IllegalArgumentException(message); + } else { + return toCheck; + } + } + /** * This method asserts a given long is a whole number. A long is whole * if it is greater or equal to zero. @@ -103,34 +162,11 @@ public static long requireWhole(final long toCheck) { public static long requireWhole(final long toCheck, final String errorMessage) { if (toCheck >= 0) { return toCheck; - } - - final String message = Objects.isNull(errorMessage) - ? "The input integer [%d] is required be whole.".formatted(toCheck) - : errorMessage; - throw new IllegalArgumentException(message); - } - - /** - * This method asserts a given integer is a positive. An integer is positive - * if it is NOT equal to zero and is greater than zero. - * - * @param toCheck the integer to check if it is a positive power of two - * @param errorMessage the error message to be used in the exception if the - * input integer to check is not positive, if null, a default message will - * be used - * @return the number to check if it is positive - * @throws IllegalArgumentException if the input number to check is not - * positive - */ - public static int requirePositive(final int toCheck, final String errorMessage) { - if (0 >= toCheck) { + } else { final String message = Objects.isNull(errorMessage) - ? "The input integer [%d] is required be positive.".formatted(toCheck) + ? "The input integer [%d] is required be whole.".formatted(toCheck) : errorMessage; throw new IllegalArgumentException(message); - } else { - return toCheck; } } diff --git a/common/src/test/java/com/hedera/block/common/utils/PreconditionsTest.java b/common/src/test/java/com/hedera/block/common/utils/PreconditionsTest.java index b6c2dd8e2..31b6f609e 100644 --- a/common/src/test/java/com/hedera/block/common/utils/PreconditionsTest.java +++ b/common/src/test/java/com/hedera/block/common/utils/PreconditionsTest.java @@ -145,6 +145,44 @@ void testRequirePositiveFail(final int toTest) { .withMessage(testErrorMessage); } + /** + * This test aims to verify that the + * {@link Preconditions#requirePositive(long)} will return the input 'toTest' + * parameter if the positive check passes. Test includes overloads. + * + * @param toTest parameterized, the number to test + */ + @ParameterizedTest + @MethodSource("com.hedera.block.common.CommonsTestUtility#positiveIntegers") + void testRequirePositiveLongPass(final long toTest) { + final Consumer asserts = actual -> assertThat(actual).isPositive().isEqualTo(toTest); + + final long actual = Preconditions.requirePositive(toTest); + assertThat(actual).satisfies(asserts); + + final long actualOverload = Preconditions.requirePositive(toTest, "test error message"); + assertThat(actualOverload).satisfies(asserts); + } + + /** + * This test aims to verify that the + * {@link Preconditions#requirePositive(long)} will throw an + * {@link IllegalArgumentException} if the positive check fails. Test + * includes overloads. + * + * @param toTest parameterized, the number to test + */ + @ParameterizedTest + @MethodSource("com.hedera.block.common.CommonsTestUtility#zeroAndNegativeIntegers") + void testRequirePositiveLongFail(final long toTest) { + assertThatIllegalArgumentException().isThrownBy(() -> Preconditions.requirePositive(toTest)); + + final String testErrorMessage = "test error message"; + assertThatIllegalArgumentException() + .isThrownBy(() -> Preconditions.requirePositive(toTest, testErrorMessage)) + .withMessage(testErrorMessage); + } + /** * This test aims to verify that the * {@link Preconditions#requirePowerOfTwo(int)} will return the input diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 24aad29a3..25592c617 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -36,8 +36,10 @@ mainModuleInfo { testModuleInfo { annotationProcessor("dagger.compiler") requires("org.junit.jupiter.api") + requires("org.junit.jupiter.params") requires("org.mockito") requires("org.mockito.junit.jupiter") + requires("org.assertj.core") requiresStatic("com.github.spotbugs.annotations") } diff --git a/server/docker/logging.properties b/server/docker/logging.properties index 60bb91bd6..c703a1553 100644 --- a/server/docker/logging.properties +++ b/server/docker/logging.properties @@ -24,7 +24,7 @@ io.helidon.common.level=INFO # Configure specific Block Node loggers #com.hedera.block.server.producer.ProducerBlockItemObserver.level=FINE #com.hedera.block.server.mediator.LiveStreamMediatorImpl.level=FINE -#com.hedera.block.server.persistence.storage.write.BlockAsDirWriter.level=FINE +#com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter.level=FINE #com.hedera.block.server.consumer.ConsumerStreamResponseObserver.level=FINE #com.hedera.block.server.pbj.PbjBlockStreamServiceProxy.level=FINE diff --git a/server/docker/update-env.sh b/server/docker/update-env.sh index 29a593980..0fa641586 100755 --- a/server/docker/update-env.sh +++ b/server/docker/update-env.sh @@ -28,6 +28,8 @@ fi if [ true = "$is_smoke_test" ]; then # add smoke test variables + # @todo(#372) - default persistence type should be BLOCK_AS_LOCAL_FILE + echo "PERSISTENCE_STORAGE_TYPE=BLOCK_AS_LOCAL_DIRECTORY" >> .env echo "MEDIATOR_RING_BUFFER_SIZE=1024" >> .env echo "NOTIFIER_RING_BUFFER_SIZE=1024" >> .env echo "JAVA_OPTS='-Xms4G -Xmx4G'" >> .env diff --git a/server/docs/configuration.md b/server/docs/configuration.md index cda8487c1..c68f65f4b 100644 --- a/server/docs/configuration.md +++ b/server/docs/configuration.md @@ -9,12 +9,14 @@ The default configuration allows users to quickly get up and running without hav ease of use at the trade-off of some insecure default configuration. Most configuration settings have appropriate defaults and can be left unchanged. It is recommended to browse the properties below and adjust to your needs. -| Environment Variable | Description | Default Value | -|:----------------------------------|:--------------------------------------------------------------------------------------------------------------|:--------------| -| PERSISTENCE_STORAGE_ROOT_PATH | The root path for the storage, if not provided will attempt to create a `data` on the working dir of the app. | | -| CONSUMER_TIMEOUT_THRESHOLD_MILLIS | Time to wait for subscribers before disconnecting in milliseconds | 1500 | -| SERVICE_DELAY_MILLIS | Service shutdown delay in milliseconds | 500 | -| MEDIATOR_RING_BUFFER_SIZE | Size of the ring buffer used by the mediator (must be a power of 2) | 67108864 | -| NOTIFIER_RING_BUFFER_SIZE | Size of the ring buffer used by the notifier (must be a power of 2) | 2048 | -| SERVER_PORT | The port the server will listen on | 8080 | -| SERVER_MAX_MESSAGE_SIZE_BYTES | The maximum size of a message frame in bytes | 1048576 | +| Environment Variable | Description | Default Value | +|:---|:---|---:| +| PERSISTENCE_STORAGE_LIVE_ROOT_PATH | The root path for the live storage. | | +| PERSISTENCE_STORAGE_ARCHIVE_ROOT_PATH | The root path for the archive storage. | | +| PERSISTENCE_STORAGE_TYPE | Type of the persistence storage | BLOCK_AS_LOCAL_FILE | +| CONSUMER_TIMEOUT_THRESHOLD_MILLIS | Time to wait for subscribers before disconnecting in milliseconds | 1500 | +| SERVICE_DELAY_MILLIS | Service shutdown delay in milliseconds | 500 | +| MEDIATOR_RING_BUFFER_SIZE | Size of the ring buffer used by the mediator (must be a power of 2) | 67108864 | +| NOTIFIER_RING_BUFFER_SIZE | Size of the ring buffer used by the notifier (must be a power of 2) | 2048 | +| SERVER_PORT | The port the server will listen on | 8080 | +| SERVER_MAX_MESSAGE_SIZE_BYTES | The maximum size of a message frame in bytes | 1048576 | diff --git a/server/src/main/java/com/hedera/block/server/BlockNodeApp.java b/server/src/main/java/com/hedera/block/server/BlockNodeApp.java index 91485e3bc..5f18136b7 100644 --- a/server/src/main/java/com/hedera/block/server/BlockNodeApp.java +++ b/server/src/main/java/com/hedera/block/server/BlockNodeApp.java @@ -96,7 +96,6 @@ public void start() throws IOException { .build(); // Build the web server - // TODO: make port server a configurable value. final WebServer webServer = webServerBuilder .port(serverConfig.port()) .addProtocol(pbjConfig) diff --git a/server/src/main/java/com/hedera/block/server/Constants.java b/server/src/main/java/com/hedera/block/server/Constants.java index 047243a4d..1870db7ef 100644 --- a/server/src/main/java/com/hedera/block/server/Constants.java +++ b/server/src/main/java/com/hedera/block/server/Constants.java @@ -18,8 +18,11 @@ /** Constants used in the BlockNode service. */ public final class Constants { - /** Constant mapped to the semantic name of the Block Node root directory */ - public static final String BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME = "Block Node Root Directory"; + /** Constant mapped to the semantic name of the Block Node live root directory */ + public static final String BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME = "Block Node Live Root Directory"; + + /** Constant mapped to the semantic name of the Block Node archive root directory */ + public static final String BLOCK_NODE_ARCHIVE_ROOT_DIRECTORY_SEMANTIC_NAME = "Block Node Archive Root Directory"; /** Constant mapped to PbjProtocolProvider.CONFIG_NAME in the PBJ Helidon Plugin */ public static final String PBJ_PROTOCOL_PROVIDER_CONFIG_NAME = "pbj"; diff --git a/server/src/main/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializer.java b/server/src/main/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializer.java index 67b2d8aa0..1b441de09 100644 --- a/server/src/main/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializer.java +++ b/server/src/main/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializer.java @@ -28,7 +28,9 @@ public final class ServerMappedConfigSourceInitializer { private static final List MAPPINGS = List.of( new ConfigMapping("consumer.timeoutThresholdMillis", "CONSUMER_TIMEOUT_THRESHOLD_MILLIS"), - new ConfigMapping("persistence.storage.rootPath", "PERSISTENCE_STORAGE_ROOT_PATH"), + new ConfigMapping("persistence.storage.liveRootPath", "PERSISTENCE_STORAGE_LIVE_ROOT_PATH"), + new ConfigMapping("persistence.storage.archiveRootPath", "PERSISTENCE_STORAGE_ARCHIVE_ROOT_PATH"), + new ConfigMapping("persistence.storage.type", "PERSISTENCE_STORAGE_TYPE"), new ConfigMapping("service.delayMillis", "SERVICE_DELAY_MILLIS"), new ConfigMapping("mediator.ringBufferSize", "MEDIATOR_RING_BUFFER_SIZE"), new ConfigMapping("notifier.ringBufferSize", "NOTIFIER_RING_BUFFER_SIZE"), diff --git a/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java b/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java index 205c17e1a..59dea3097 100644 --- a/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java +++ b/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java @@ -28,7 +28,7 @@ public interface MetricsService { * @param key to get a specific counter * @return the counter */ - Counter get(@NonNull BlockNodeMetricTypes.Counter key); + Counter get(@NonNull final BlockNodeMetricTypes.Counter key); /** * Use this method to get a specific gauge for the given metric type. @@ -36,5 +36,5 @@ public interface MetricsService { * @param key to get a specific gauge * @return the gauge */ - LongGauge get(@NonNull BlockNodeMetricTypes.Gauge key); + LongGauge get(@NonNull final BlockNodeMetricTypes.Gauge key); } diff --git a/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java b/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java index 776318333..0a38cd5ee 100644 --- a/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java +++ b/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java @@ -21,6 +21,7 @@ import com.swirlds.metrics.api.Metrics; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.EnumMap; +import java.util.Objects; import javax.inject.Inject; /** @@ -29,10 +30,8 @@ *

Metrics are updated by calling the appropriate method on the metric object instance. For * example, to increment a counter, call {@link Counter#increment()}. */ -public class MetricsServiceImpl implements MetricsService { - +public final class MetricsServiceImpl implements MetricsService { private static final String CATEGORY = "hedera_block_node"; - private final EnumMap counters = new EnumMap<>(BlockNodeMetricTypes.Counter.class); private final EnumMap gauges = @@ -45,22 +44,21 @@ public class MetricsServiceImpl implements MetricsService { */ @Inject public MetricsServiceImpl(@NonNull final Metrics metrics) { + Objects.requireNonNull(metrics); // Initialize the counters - for (BlockNodeMetricTypes.Counter counter : BlockNodeMetricTypes.Counter.values()) { + for (final BlockNodeMetricTypes.Counter counter : BlockNodeMetricTypes.Counter.values()) { counters.put( counter, - metrics.getOrCreate( - new Counter.Config(CATEGORY, counter.grafanaLabel()) - .withDescription(counter.description()))); + metrics.getOrCreate(new Counter.Config(CATEGORY, counter.grafanaLabel()) + .withDescription(counter.description()))); } // Initialize the gauges - for (BlockNodeMetricTypes.Gauge gauge : BlockNodeMetricTypes.Gauge.values()) { + for (final BlockNodeMetricTypes.Gauge gauge : BlockNodeMetricTypes.Gauge.values()) { gauges.put( gauge, metrics.getOrCreate( - new LongGauge.Config(CATEGORY, gauge.grafanaLabel()) - .withDescription(gauge.description()))); + new LongGauge.Config(CATEGORY, gauge.grafanaLabel()).withDescription(gauge.description()))); } } @@ -72,8 +70,8 @@ public MetricsServiceImpl(@NonNull final Metrics metrics) { */ @NonNull @Override - public Counter get(@NonNull BlockNodeMetricTypes.Counter key) { - return counters.get(key); + public Counter get(@NonNull final BlockNodeMetricTypes.Counter key) { + return counters.get(Objects.requireNonNull(key)); } /** @@ -84,7 +82,7 @@ public Counter get(@NonNull BlockNodeMetricTypes.Counter key) { */ @NonNull @Override - public LongGauge get(@NonNull BlockNodeMetricTypes.Gauge key) { - return gauges.get(key); + public LongGauge get(@NonNull final BlockNodeMetricTypes.Gauge key) { + return gauges.get(Objects.requireNonNull(key)); } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java b/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java index 518033e8c..2f35925d8 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java @@ -20,9 +20,21 @@ import com.hedera.block.server.events.BlockNodeEventHandler; import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig.StorageType; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalFilePathResolver; +import com.hedera.block.server.persistence.storage.path.BlockPathResolver; +import com.hedera.block.server.persistence.storage.path.NoOpBlockPathResolver; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReader; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalFileReader; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; +import com.hedera.block.server.persistence.storage.read.NoOpBlockReader; +import com.hedera.block.server.persistence.storage.remove.BlockAsLocalDirRemover; +import com.hedera.block.server.persistence.storage.remove.BlockAsLocalFileRemover; +import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.block.server.persistence.storage.remove.NoOpBlockRemover; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalFileWriter; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.persistence.storage.write.NoOpBlockWriter; import com.hedera.hapi.block.BlockItemUnparsed; @@ -31,47 +43,106 @@ import dagger.Binds; import dagger.Module; import dagger.Provides; +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; import java.util.List; +import java.util.Objects; import javax.inject.Singleton; /** A Dagger module for providing dependencies for Persistence Module. */ @Module public interface PersistenceInjectionModule { - /** * Provides a block writer singleton using the block node context. * * @param blockNodeContext the application context + * @param blockRemover the block remover + * @param blockPathResolver the block path resolver * @return a block writer singleton */ @Provides @Singleton - static BlockWriter> providesBlockWriter(BlockNodeContext blockNodeContext) { - final String persistenceType = blockNodeContext + static BlockWriter> providesBlockWriter( + @NonNull final BlockNodeContext blockNodeContext, + @NonNull final BlockRemover blockRemover, + @NonNull final BlockPathResolver blockPathResolver) { + Objects.requireNonNull(blockRemover); + Objects.requireNonNull(blockPathResolver); + final StorageType persistenceType = blockNodeContext .configuration() .getConfigData(PersistenceStorageConfig.class) .type(); - if ("NOOP".equalsIgnoreCase(persistenceType)) { - return new NoOpBlockWriter(blockNodeContext); - } try { - return BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - } catch (IOException e) { - throw new RuntimeException("Failed to create block writer", e); + return switch (persistenceType) { + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFileWriter.of(blockNodeContext, blockPathResolver); + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirWriter.of( + blockNodeContext, blockRemover, blockPathResolver); + case NO_OP -> NoOpBlockWriter.newInstance(); + }; + } catch (final IOException e) { + // we cannot have checked exceptions with dagger @Provides + throw new UncheckedIOException("Failed to create BlockWriter", e); } } /** * Provides a block reader singleton using the persistence storage config. * - * @param config the persistence storage configuration needed to build the block reader + * @param config the persistence storage configuration needed to build the + * block reader * @return a block reader singleton */ @Provides @Singleton - static BlockReader providesBlockReader(PersistenceStorageConfig config) { - return BlockAsDirReaderBuilder.newBuilder(config).build(); + static BlockReader providesBlockReader(@NonNull final PersistenceStorageConfig config) { + final StorageType persistenceType = config.type(); + return switch (persistenceType) { + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFileReader.newInstance(); + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirReader.of(config); + case NO_OP -> NoOpBlockReader.newInstance(); + }; + } + + /** + * Provides a block remover singleton using the persistence storage config. + * + * @param config the persistence storage configuration needed to build the + * block remover + * @param blockPathResolver the block path resolver + * @return a block remover singleton + */ + @Provides + @Singleton + static BlockRemover providesBlockRemover( + @NonNull final PersistenceStorageConfig config, @NonNull final BlockPathResolver blockPathResolver) { + Objects.requireNonNull(blockPathResolver); + final StorageType persistenceType = config.type(); + return switch (persistenceType) { + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFileRemover.newInstance(); + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirRemover.of(blockPathResolver); + case NO_OP -> NoOpBlockRemover.newInstance(); + }; + } + + /** + * Provides a path resolver singleton using the persistence storage config. + * + * @param config the persistence storage configuration needed to build the + * path resolver + * @return a path resolver singleton + */ + @Provides + @Singleton + static BlockPathResolver providesPathResolver(@NonNull final PersistenceStorageConfig config) { + final StorageType persistenceType = config.type(); + final Path blockStorageRoot = Path.of(config.liveRootPath()); + return switch (persistenceType) { + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFilePathResolver.of(blockStorageRoot); + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirPathResolver.of(blockStorageRoot); + case NO_OP -> NoOpBlockPathResolver.newInstance(); + }; } /** @@ -83,5 +154,5 @@ static BlockReader providesBlockReader(PersistenceStorageConfig c @Binds @Singleton BlockNodeEventHandler> bindBlockNodeEventHandler( - StreamPersistenceHandlerImpl streamPersistenceHandler); + @NonNull final StreamPersistenceHandlerImpl streamPersistenceHandler); } diff --git a/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java b/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java index 76d3e96ac..6f4174a86 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java +++ b/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java @@ -32,9 +32,7 @@ import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.hedera.pbj.runtime.OneOf; -import com.hedera.pbj.runtime.ParseException; import edu.umd.cs.findbugs.annotations.NonNull; -import java.io.IOException; import java.util.List; import java.util.Optional; import javax.inject.Inject; @@ -102,7 +100,6 @@ public StreamPersistenceHandlerImpl( public void onEvent(ObjectEvent event, long l, boolean b) { try { if (serviceStatus.isRunning()) { - final SubscribeStreamResponseUnparsed subscribeStreamResponse = event.get(); final OneOf oneOfTypeOneOf = subscribeStreamResponse.response(); @@ -137,9 +134,7 @@ public void onEvent(ObjectEvent event, long l, } else { LOGGER.log(ERROR, "Service is not running. Block item will not be processed further."); } - - } catch (BlockStreamProtocolException | IOException | ParseException e) { - + } catch (final Exception e) { LOGGER.log(ERROR, "Failed to persist BlockItems: ", e); metricsService.get(StreamPersistenceHandlerError).increment(); diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java b/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java index 2ae5711a9..c385378b1 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java @@ -16,56 +16,134 @@ package com.hedera.block.server.persistence.storage; -import static com.hedera.block.server.Constants.BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME; -import static java.lang.System.Logger.Level.ERROR; -import static java.lang.System.Logger.Level.INFO; +import static com.hedera.block.server.Constants.BLOCK_NODE_ARCHIVE_ROOT_DIRECTORY_SEMANTIC_NAME; +import static com.hedera.block.server.Constants.BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME; -import com.hedera.block.common.utils.FileUtilities; +import com.hedera.block.common.utils.StringUtilities; import com.swirlds.config.api.ConfigData; import com.swirlds.config.api.ConfigProperty; +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; +import java.util.Objects; /** - * Use this configuration across the persistent storage package + * Use this configuration across the persistence storage package. * - * @param rootPath provides the root path for saving block data, if you want to override it need to - * set it as persistence.storage.rootPath + * @param liveRootPath provides the root path for saving blocks live + * @param archiveRootPath provides the root path for archived blocks * @param type use a predefined type string to replace the persistence component implementation. - * Non-PRODUCTION values should only be used for troubleshooting and development purposes. + * Non-PRODUCTION values should only be used for troubleshooting and development purposes. */ @ConfigData("persistence.storage") public record PersistenceStorageConfig( - @ConfigProperty(defaultValue = "") String rootPath, @ConfigProperty(defaultValue = "PRODUCTION") String type) { - private static final System.Logger LOGGER = System.getLogger(PersistenceStorageConfig.class.getName()); + // @todo(#371) - the default life/archive root path must be absolute starting from /opt + @ConfigProperty(defaultValue = "") String liveRootPath, + // @todo(#371) - the default life/archive root path must be absolute starting from /opt + @ConfigProperty(defaultValue = "") String archiveRootPath, + @ConfigProperty(defaultValue = "BLOCK_AS_LOCAL_FILE") StorageType type) { + // @todo(#371) - the default life/archive root path must be absolute starting from /opt + private static final String LIVE_ROOT_PATH = + Path.of("hashgraph/blocknode/data/live/").toAbsolutePath().toString(); + // @todo(#371) - the default life/archive root path must be absolute starting from /opt + private static final String ARCHIVE_ROOT_PATH = + Path.of("hashgraph/blocknode/data/archive/").toAbsolutePath().toString(); /** - * Constructor to set the default root path if not provided, it will be set to the data - * directory in the current working directory + * Constructor. */ public PersistenceStorageConfig { - // verify rootPath prop - Path path = Path.of(rootPath); - if (rootPath.isEmpty()) { - path = Paths.get(rootPath).toAbsolutePath().resolve("data"); - } - // Check if absolute - if (!path.isAbsolute()) { - throw new IllegalArgumentException(rootPath + " Root path must be absolute"); - } - // Create Directory if it does not exist + Objects.requireNonNull(type); + liveRootPath = resolvePath(liveRootPath, LIVE_ROOT_PATH, BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME); + archiveRootPath = + resolvePath(archiveRootPath, ARCHIVE_ROOT_PATH, BLOCK_NODE_ARCHIVE_ROOT_DIRECTORY_SEMANTIC_NAME); + } + + /** + * This method attempts to resolve a given configured path. If the input + * path is blank, a default path is used. The resolved path must be + * absolute! If the path resolution is successful, at attempt is made to + * create the directory path. If the directory path cannot be created, an + * {@link UncheckedIOException} is thrown. + * + * @param pathToResolve the path to resolve + * @param defaultIfBlank the default path if the path to resolve is blank + * @param semanticPathName the semantic name of the path used for logging + * @return the resolved path + * @throws IllegalArgumentException if the resolved path is not absolute + * @throws UncheckedIOException if the resolved path cannot be created + */ + @NonNull + private String resolvePath( + final String pathToResolve, @NonNull final String defaultIfBlank, @NonNull final String semanticPathName) { + final Path normalized = getNormalizedPath(pathToResolve, defaultIfBlank); + createDirectoryPath(normalized, semanticPathName); + return normalized.toString(); + } + + /** + * This method normalizes a given path. If the path to normalize is blank, + * a default path is used. The normalized path must be absolute! + * + * @param pathToNormalize the path to normalize + * @param defaultIfBlank the default path if the path to normalize is blank + * @throws IllegalArgumentException if the path to normalize is not absolute + */ + @NonNull + private Path getNormalizedPath(final String pathToNormalize, @NonNull final String defaultIfBlank) { + final String actualToNormalize = StringUtilities.isBlank(pathToNormalize) ? defaultIfBlank : pathToNormalize; + return Path.of(actualToNormalize).normalize().toAbsolutePath(); + } + + /** + * This method creates a directory path at the given target path. If the + * directory path cannot be created, an {@link UncheckedIOException} is + * thrown. + * + * @param targetPath the target path to create the directory path + * @param semanticPathName the semantic name of the path used for logging + * @throws UncheckedIOException if the directory path cannot be created + */ + private void createDirectoryPath(@NonNull final Path targetPath, @NonNull final String semanticPathName) { try { - FileUtilities.createFolderPathIfNotExists(path, ERROR, BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME); + Files.createDirectories(targetPath); } catch (final IOException e) { - final String message = - "Unable to instantiate [%s]! Unable to create the root directory for the block storage [%s]" - .formatted(this.getClass().getName(), path); - throw new RuntimeException(message, e); + final String classname = this.getClass().getName(); + final String message = "Unable to instantiate [%s]! Unable to create the [%s] path that was provided!" + .formatted(classname, semanticPathName); + throw new UncheckedIOException(message, e); } + } - LOGGER.log(INFO, "Persistence Storage configuration persistence.storage.rootPath: " + path); - rootPath = path.toString(); - LOGGER.log(INFO, "Persistence configuration persistence.storage.type: " + type); + /** + * An enum that reflects the type of Block Storage Persistence that is + * currently used within the given server instance. During runtime one + * should only query for the storage type that was configured by calling + * {@link PersistenceStorageConfig#type()} on an instance of the persistence + * storage config that was only constructed via + * {@link com.swirlds.config.api.Configuration#getConfigData(Class)}! + */ + public enum StorageType { + /** + * This type of storage stores Blocks as individual files with the Block + * number as a unique file name and persisted in a trie structure with + * digit-per-folder + * (see #125). + * This is also the default setting for the server if it is not + * explicitly specified via an environment variable or app.properties. + */ + BLOCK_AS_LOCAL_FILE, + /** + * This type of storage stores Blocks as directories with the Block + * number being the directory number. Block Items are stored as files + * within a given Block directory. Used primarily for testing purposes. + */ + BLOCK_AS_LOCAL_DIRECTORY, + /** + * This type of storage does nothing. + */ + NO_OP } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolver.java b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolver.java new file mode 100644 index 000000000..03299ac17 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolver.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import com.hedera.block.common.utils.Preconditions; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.file.Path; +import java.util.Objects; + +/** + * A Block path resolver for block-as-dir. + */ +public final class BlockAsLocalDirPathResolver implements BlockPathResolver { + private final Path liveRootPath; + + /** + * Constructor. + * + * @param liveRootPath valid, {@code non-null} instance of {@link Path} + * that points to the live root of the block storage + */ + private BlockAsLocalDirPathResolver(@NonNull final Path liveRootPath) { + this.liveRootPath = Objects.requireNonNull(liveRootPath); + } + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalDirPathResolver}. + * + * @param liveRootPath valid, {@code non-null} instance of {@link Path} + * that points to the live root of the block storage + * @return a new, fully initialized instance of + * {@link BlockAsLocalDirPathResolver} + */ + public static BlockAsLocalDirPathResolver of(@NonNull final Path liveRootPath) { + return new BlockAsLocalDirPathResolver(liveRootPath); + } + + @NonNull + @Override + public Path resolvePathToBlock(final long blockNumber) { + Preconditions.requireWhole(blockNumber); + return liveRootPath.resolve(String.valueOf(blockNumber)); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolver.java b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolver.java new file mode 100644 index 000000000..0a5966dae --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolver.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import com.hedera.block.common.utils.Preconditions; +import com.hedera.block.server.Constants; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.file.Path; +import java.util.Objects; + +/** + * A Block path resolver for block-as-file. + */ +public final class BlockAsLocalFilePathResolver implements BlockPathResolver { + private static final int MAX_LONG_DIGITS = 19; + private final Path liveRootPath; + + /** + * Constructor. + * + * @param liveRootPath valid, {@code non-null} instance of {@link Path} + * that points to the live root of the block storage + */ + private BlockAsLocalFilePathResolver(@NonNull final Path liveRootPath) { + this.liveRootPath = Objects.requireNonNull(liveRootPath); + } + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalFilePathResolver}. + * + * @param liveRootPath valid, {@code non-null} instance of {@link Path} + * that points to the live root of the block storage + * @return a new, fully initialized instance of + * {@link BlockAsLocalFilePathResolver} + */ + public static BlockAsLocalFilePathResolver of(@NonNull final Path liveRootPath) { + return new BlockAsLocalFilePathResolver(liveRootPath); + } + + @NonNull + @Override + public Path resolvePathToBlock(final long blockNumber) { + Preconditions.requireWhole(blockNumber); + final String rawBlockNumber = String.format("%0" + MAX_LONG_DIGITS + "d", blockNumber); + final String[] blockPath = rawBlockNumber.split(""); + final String blockFileName = rawBlockNumber.concat(Constants.BLOCK_FILE_EXTENSION); + blockPath[blockPath.length - 1] = blockFileName; + return Path.of(liveRootPath.toString(), blockPath); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockPathResolver.java b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockPathResolver.java new file mode 100644 index 000000000..4a141b3db --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockPathResolver.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.file.Path; + +/** + * A Block path resolver. Used to resolve path to a given Block and all the + * supporting related operations. + */ +public interface BlockPathResolver { + /** + * This method resolves the fs {@link Path} to a Block by a given input + * number. This method does not guarantee that the returned {@link Path} + * exists! This method is guaranteed to return a {@code non-null} + * {@link Path}. + * + * @param blockNumber to be resolved the path for + * @return the resolved path to the given Block by a number + * @throws IllegalArgumentException if the blockNumber IS NOT a whole number + */ + @NonNull + Path resolvePathToBlock(final long blockNumber); +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolver.java b/server/src/main/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolver.java new file mode 100644 index 000000000..4386cc244 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolver.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.file.Path; + +/** + * A no-op path resolver. + */ +public final class NoOpBlockPathResolver implements BlockPathResolver { + /** + * Constructor. + */ + private NoOpBlockPathResolver() {} + + /** + * This method creates and returns a new instance of + * {@link NoOpBlockPathResolver}. + * + * @return a new, fully initialized instance of + * {@link NoOpBlockPathResolver} + */ + public static NoOpBlockPathResolver newInstance() { + return new NoOpBlockPathResolver(); + } + + /** + * No-op resolver. Does nothing and always returns a path under '/tmp' that + * resolves to 'blockNumber.tmp.blk'. No preconditions check also. + */ + @NonNull + @Override + public Path resolvePathToBlock(final long blockNumber) { + final String blockName = String.format("%d.tmp.blk", blockNumber); + return Path.of("/tmp/hashgraph/blocknode/data/").resolve(blockName); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java deleted file mode 100644 index 8cb90cbee..000000000 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (C) 2024 Hedera Hashgraph, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.hedera.block.server.persistence.storage.read; - -import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.hapi.block.BlockUnparsed; -import edu.umd.cs.findbugs.annotations.NonNull; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.util.Objects; -import java.util.Set; - -/** - * Use builder methods to create a {@link BlockReader} to read blocks from storage. - * - *

When a block reader is created, it will provide access to read blocks from storage. - */ -public final class BlockAsDirReaderBuilder { - private final PersistenceStorageConfig config; - private FileAttribute> folderPermissions; - - private BlockAsDirReaderBuilder(@NonNull final PersistenceStorageConfig config) { - this.config = Objects.requireNonNull(config); - } - - /** - * Creates a new block reader builder using the minimum required parameters. - * - * @param config is required to supply pertinent configuration info for the block reader to - * access storage. - * @return a block reader builder configured with required parameters. - */ - @NonNull - public static BlockAsDirReaderBuilder newBuilder(@NonNull final PersistenceStorageConfig config) { - return new BlockAsDirReaderBuilder(config); - } - - /** - * Optionally, provide file permissions for the block reader to use when managing block files - * and directories. - * - * @param folderPermissions the folder permissions to use when managing block files as directories. - * @return a block reader builder configured with required parameters. - */ - @NonNull - public BlockAsDirReaderBuilder folderPermissions( - @NonNull final FileAttribute> folderPermissions) { - this.folderPermissions = Objects.requireNonNull(folderPermissions); - return this; - } - - /** - * Use the build method to construct a block reader to read blocks from storage. - * - * @return a new block reader configured with the parameters provided to the builder. - */ - @NonNull - public BlockReader build() { - return new BlockAsDirReader(config, folderPermissions); - } -} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReader.java similarity index 81% rename from server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java rename to server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReader.java index 3a082c4b0..dc5b19645 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReader.java @@ -22,6 +22,7 @@ import static java.lang.System.Logger.Level.ERROR; import static java.lang.System.Logger.Level.INFO; +import com.hedera.block.common.utils.Preconditions; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.BlockUnparsed; @@ -39,7 +40,6 @@ import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -47,35 +47,10 @@ * The BlockAsDirReader class reads a block from the file system. The block is stored as a directory * containing block items. The block items are stored as files within the block directory. */ -class BlockAsDirReader implements BlockReader { - private final Logger LOGGER = System.getLogger(getClass().getName()); - private final Path blockNodeRootPath; - private final FileAttribute> folderPermissions; - - /** - * Constructor for the BlockAsDirReader class. It initializes the BlockAsDirReader with the - * given parameters. - * - * @param config the configuration to retrieve the block node root path - * @param folderPermissions the folder permissions to set on the block node root path, default will be used if null provided - */ - BlockAsDirReader( - @NonNull final PersistenceStorageConfig config, - final FileAttribute> folderPermissions) { - LOGGER.log(INFO, "Initializing FileSystemBlockReader"); - - final Path blockNodeRootPath = Path.of(config.rootPath()); - - LOGGER.log(INFO, config.toString()); - LOGGER.log(INFO, "Block Node Root Path: " + blockNodeRootPath); - - this.blockNodeRootPath = blockNodeRootPath; - - if (Objects.nonNull(folderPermissions)) { - this.folderPermissions = folderPermissions; - } else { - // default permissions for folders - this.folderPermissions = PosixFilePermissions.asFileAttribute(Set.of( +public class BlockAsLocalDirReader implements LocalBlockReader { + private static final Logger LOGGER = System.getLogger(BlockAsLocalDirReader.class.getName()); + private static final FileAttribute> DEFAULT_FOLDER_PERMISSIONS = + PosixFilePermissions.asFileAttribute(Set.of( PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, @@ -83,7 +58,30 @@ class BlockAsDirReader implements BlockReader { PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_EXECUTE)); - } + private final Path liveRootPath; + + /** + * Constructor. + * + * @param config the configuration to retrieve the root paths + */ + protected BlockAsLocalDirReader(@NonNull final PersistenceStorageConfig config) { + LOGGER.log(INFO, "Initializing %s...".formatted(getClass().getName())); + this.liveRootPath = Path.of(config.liveRootPath()); + } + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalDirReader}. + * + * @param config valid, {@code non-null} instance of + * {@link PersistenceStorageConfig} used to retrieve the block node root + * path + * @return a new, fully initialized instance of + * {@link BlockAsLocalDirReader} + */ + public static BlockAsLocalDirReader of(@NonNull final PersistenceStorageConfig config) { + return new BlockAsLocalDirReader(config); } /** @@ -97,15 +95,15 @@ class BlockAsDirReader implements BlockReader { @NonNull @Override public Optional read(final long blockNumber) throws IOException, ParseException { - - // Verify path attributes of the block node root path - if (isPathDisqualified(blockNodeRootPath)) { + Preconditions.requireWhole(blockNumber); + // Verify path attributes of the block node live root path + if (isPathDisqualified(liveRootPath)) { return Optional.empty(); } // Verify path attributes of the block directory within the - // block node root path - final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); + // block node live root path + final Path blockPath = liveRootPath.resolve(String.valueOf(blockNumber)); if (isPathDisqualified(blockPath)) { return Optional.empty(); } @@ -130,7 +128,6 @@ public Optional read(final long blockNumber) throws IOException, blockItems.add(blockItemOpt.get()); continue; } - break; } @@ -138,7 +135,7 @@ public Optional read(final long blockNumber) throws IOException, // Return the Block return Optional.of(builder.build()); - } catch (IOException io) { + } catch (final IOException io) { LOGGER.log(ERROR, "Error reading block: " + blockPath, io); throw io; } @@ -187,7 +184,7 @@ private boolean isPathDisqualified(@NonNull final Path path) { try { // If resetting the permissions fails or // if the path is still unreadable, return true. - setPerm(path, folderPermissions.value()); + setPerm(path, DEFAULT_FOLDER_PERMISSIONS.value()); if (!path.toFile().canRead()) { return true; } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReader.java new file mode 100644 index 000000000..eb568cabc --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReader.java @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.read; + +import com.hedera.block.common.utils.Preconditions; +import com.hedera.hapi.block.BlockUnparsed; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.Optional; + +/** + * A Block reader that reads block-as-file. + */ +public final class BlockAsLocalFileReader implements LocalBlockReader { + /** + * Constructor. + */ + private BlockAsLocalFileReader() {} + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalFileReader}. + * + * @return a new, fully initialized instance of + * {@link BlockAsLocalFileReader} + */ + public static BlockAsLocalFileReader newInstance() { + return new BlockAsLocalFileReader(); + } + + @NonNull + @Override + public Optional read(final long blockNumber) { + Preconditions.requireWhole(blockNumber); + throw new UnsupportedOperationException("Not implemented yet"); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockReader.java index 1a24bccdb..2c7a42bc8 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockReader.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockReader.java @@ -24,10 +24,9 @@ /** * The BlockReader interface defines the contract for reading a block from storage. * - * @param the type of the block to read + * @param the type to be returned after reading the block */ -public interface BlockReader { - +public interface BlockReader { /** * Reads the block with the given block number. * @@ -37,7 +36,8 @@ public interface BlockReader { * @throws ParseException if the PBJ codec encounters a problem caused by I/O issues, malformed * input data, or any other reason that prevents the parse() method from completing the * operation when fetching the block. + * @throws IllegalArgumentException if the blockNumber IS NOT a whole number */ @NonNull - Optional read(final long blockNumber) throws IOException, ParseException; + Optional read(final long blockNumber) throws IOException, ParseException; } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/LocalBlockReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/LocalBlockReader.java new file mode 100644 index 000000000..095d26b9f --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/LocalBlockReader.java @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.read; + +/** + * A marker interface that groups all writers that operate on a local file + * system. + * + * @param the type that will be returned after reading the block + */ +public interface LocalBlockReader extends BlockReader {} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReader.java new file mode 100644 index 000000000..6d67ad348 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReader.java @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.read; + +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.pbj.runtime.ParseException; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.IOException; +import java.util.Optional; + +/** + * A no-op Block reader. + */ +public final class NoOpBlockReader implements BlockReader { + /** + * Constructor. + */ + private NoOpBlockReader() {} + + /** + * This method creates and returns a new instance of + * {@link NoOpBlockReader}. + * + * @return a new, fully initialized instance of + * {@link NoOpBlockReader} + */ + public static NoOpBlockReader newInstance() { + return new NoOpBlockReader(); + } + + @NonNull + @Override + public Optional read(final long blockNumber) throws IOException, ParseException { + return Optional.empty(); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemover.java b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemover.java similarity index 51% rename from server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemover.java rename to server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemover.java index 5fbc4a161..85aa5cbce 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemover.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemover.java @@ -19,6 +19,8 @@ import static java.lang.System.Logger; import static java.lang.System.Logger.Level.ERROR; +import com.hedera.block.common.utils.Preconditions; +import com.hedera.block.server.persistence.storage.path.BlockPathResolver; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.File; import java.io.IOException; @@ -30,37 +32,51 @@ * The BlockAsDirRemover class removes a block from the file system. The block is stored as a * directory containing block items. The block items are stored as files within the block directory. */ -public class BlockAsDirRemover implements BlockRemover { +public final class BlockAsLocalDirRemover implements LocalBlockRemover { private final Logger LOGGER = System.getLogger(getClass().getName()); - private final Path blockNodeRootPath; + private final BlockPathResolver blockPathResolver; /** - * Create a block remover to manage removing blocks from storage. + * Constructor. * - * @param blockNodeRootPath the root path where blocks are stored. + * @param blockPathResolver valid, {@code non-null} instance of + * {@link BlockPathResolver} to be used internally to resolve paths to Block */ - public BlockAsDirRemover(@NonNull final Path blockNodeRootPath) { - this.blockNodeRootPath = Objects.requireNonNull(blockNodeRootPath); + private BlockAsLocalDirRemover(@NonNull final BlockPathResolver blockPathResolver) { + this.blockPathResolver = Objects.requireNonNull(blockPathResolver); + } + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalDirRemover}. + * + * @param blockPathResolver valid, {@code non-null} instance of + * {@link BlockPathResolver} to be used internally to resolve paths to Block + * @return a new, fully initialized instance of + * {@link BlockAsLocalDirRemover} + */ + public static BlockAsLocalDirRemover of(@NonNull final BlockPathResolver blockPathResolver) { + return new BlockAsLocalDirRemover(blockPathResolver); } /** * Removes a block from the file system. * - * @param id the id of the block to remove + * @param blockNumber the id of the block to remove * @throws IOException if an I/O error occurs + * @throws IllegalArgumentException if the blockNumber IS NOT a whole number */ @Override - public void remove(final long id) throws IOException { - // Calculate the block path and proactively set the permissions - // for removal - final Path blockPath = blockNodeRootPath.resolve(String.valueOf(id)); + public void remove(final long blockNumber) throws IOException { + Preconditions.requireWhole(blockNumber); + final Path blockPath = blockPathResolver.resolvePathToBlock(blockNumber); if (Files.notExists(blockPath)) { - LOGGER.log(ERROR, "Block does not exist: {0}", id); - return; - } - // Best effort to delete the block - if (!delete(blockPath.toFile())) { - LOGGER.log(ERROR, "Failed to delete block: {0}", id); + LOGGER.log(ERROR, "Block cannot be deleted as it does not exist: {0}", blockNumber); + } else { + final boolean deleted = delete(blockPath.toFile()); + if (!deleted) { + LOGGER.log(ERROR, "Failed to delete block: {0}", blockNumber); + } } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemover.java b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemover.java new file mode 100644 index 000000000..c6065489f --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemover.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +import com.hedera.block.common.utils.Preconditions; +import java.io.IOException; + +/** + * A Block remover that handles block-as-local-file. + */ +public final class BlockAsLocalFileRemover implements LocalBlockRemover { + /** + * Constructor. + */ + private BlockAsLocalFileRemover() {} + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalFileRemover}. + * + * @return a new fully initialized instance of + * {@link BlockAsLocalFileRemover} + */ + public static BlockAsLocalFileRemover newInstance() { + return new BlockAsLocalFileRemover(); + } + + /** + * Remove a block with the given block number. + * + * @param blockNumber the block number of the block to remove + * @throws IOException when failing to remove a block + * @throws IllegalArgumentException if the blockNumber IS NOT a whole number + */ + @Override + public void remove(final long blockNumber) throws IOException { + Preconditions.requireWhole(blockNumber); + throw new UnsupportedOperationException("Not implemented yet"); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockRemover.java b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockRemover.java index bee39bed8..9ece1c3fa 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockRemover.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockRemover.java @@ -20,12 +20,12 @@ /** The BlockRemover interface defines the contract for removing a block from storage. */ public interface BlockRemover { - /** * Remove a block with the given block number. * - * @param blockNumber the block number of the block to remove. - * @throws IOException when failing to remove a block. + * @param blockNumber the block number of the block to remove + * @throws IOException when failing to remove a block + * @throws IllegalArgumentException if the blockNumber IS NOT a whole number */ void remove(final long blockNumber) throws IOException; } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/LocalBlockRemover.java b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/LocalBlockRemover.java new file mode 100644 index 000000000..f996f1d81 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/LocalBlockRemover.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +/** + * A marker interface that groups all removers that operate on a local file + * system. + */ +public interface LocalBlockRemover extends BlockRemover {} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemover.java b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemover.java new file mode 100644 index 000000000..45313b651 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemover.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +/** + * A no-op Block remover. + */ +public final class NoOpBlockRemover implements BlockRemover { + /** + * Constructor. + */ + private NoOpBlockRemover() {} + + /** + * This method creates and returns a new instance of {@link NoOpBlockRemover}. + * + * @return a new, fully initialized instance of {@link NoOpBlockRemover} + */ + public static NoOpBlockRemover newInstance() { + return new NoOpBlockRemover(); + } + + /** + * No-op remover. Does nothing and returns immediately. No preconditions + * check also. + */ + @Override + public void remove(final long blockNumber) { + // do nothing + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java deleted file mode 100644 index b6e826807..000000000 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (C) 2024 Hedera Hashgraph, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.hedera.block.server.persistence.storage.write; - -import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.remove.BlockAsDirRemover; -import com.hedera.block.server.persistence.storage.remove.BlockRemover; -import com.hedera.hapi.block.BlockItemUnparsed; -import edu.umd.cs.findbugs.annotations.NonNull; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.util.List; -import java.util.Objects; -import java.util.Set; - -/** - * Use builder methods to create a {@link BlockWriter} to write blocks to storage. - * - *

When a block writer is created, it will provide access to write blocks to storage. - */ -public final class BlockAsDirWriterBuilder { - private final BlockNodeContext blockNodeContext; - private FileAttribute> folderPermissions; - private BlockRemover blockRemover; - - private BlockAsDirWriterBuilder(@NonNull final BlockNodeContext blockNodeContext) { - this.blockNodeContext = Objects.requireNonNull(blockNodeContext); - final PersistenceStorageConfig config = - blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); - this.blockRemover = new BlockAsDirRemover(Path.of(config.rootPath())); - } - - /** - * Creates a new block writer builder using the minimum required parameters. - * - * @param blockNodeContext is required to provide metrics reporting mechanisms . - * @return a block writer builder configured with required parameters. - */ - @NonNull - public static BlockAsDirWriterBuilder newBuilder(@NonNull final BlockNodeContext blockNodeContext) { - return new BlockAsDirWriterBuilder(blockNodeContext); - } - - /** - * Optionally, provide file permissions for the block writer to use when managing block files - * and directories. - * - * @param folderPermissions the folder permissions to use when managing block files as directories. - * @return a block writer builder configured with required parameters. - */ - @NonNull - public BlockAsDirWriterBuilder folderPermissions( - @NonNull final FileAttribute> folderPermissions) { - this.folderPermissions = Objects.requireNonNull(folderPermissions); - return this; - } - - /** - * Optionally, provide a block remover to remove blocks from storage. - * - *

By default, the block writer will use the block remover defined in {@link - * BlockAsDirRemover}. This method is primarily used for testing purposes. Default values should - * be sufficient for production use. - * - * @param blockRemover the block remover to use when removing blocks from storage. - * @return a block writer builder configured with required parameters. - */ - @NonNull - public BlockAsDirWriterBuilder blockRemover(@NonNull final BlockRemover blockRemover) { - this.blockRemover = Objects.requireNonNull(blockRemover); - return this; - } - - /** - * Use the build method to construct a block writer to write blocks to storage. - * - * @return a new block writer configured with the parameters provided to the builder. - * @throws IOException when an error occurs while persisting block items to storage. - */ - @NonNull - public BlockWriter> build() throws IOException { - return new BlockAsDirWriter(blockRemover, folderPermissions, blockNodeContext); - } -} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriter.java similarity index 57% rename from server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java rename to server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriter.java index 461c74911..dc468328f 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriter.java @@ -17,7 +17,7 @@ package com.hedera.block.server.persistence.storage.write; import static com.hedera.block.server.Constants.BLOCK_FILE_EXTENSION; -import static com.hedera.block.server.Constants.BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME; +import static com.hedera.block.server.Constants.BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.BlocksPersisted; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; @@ -25,9 +25,11 @@ import static java.lang.System.Logger.Level.INFO; import com.hedera.block.common.utils.FileUtilities; +import com.hedera.block.common.utils.Preconditions; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.persistence.storage.path.BlockPathResolver; import com.hedera.block.server.persistence.storage.remove.BlockRemover; import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.stream.output.BlockHeader; @@ -36,6 +38,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.io.FileOutputStream; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; @@ -55,139 +58,137 @@ * to remove the current, incomplete block (directory) before re-throwing the exception to the * caller. */ -class BlockAsDirWriter implements BlockWriter> { - private final Logger LOGGER = System.getLogger(getClass().getName()); - private final Path blockNodeRootPath; - private final FileAttribute> folderPermissions; - private final BlockRemover blockRemover; +public class BlockAsLocalDirWriter implements LocalBlockWriter> { + private static final Logger LOGGER = System.getLogger(BlockAsLocalDirWriter.class.getName()); + private static final FileAttribute> DEFAULT_FOLDER_PERMISSIONS = + PosixFilePermissions.asFileAttribute(Set.of( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE, + PosixFilePermission.GROUP_READ, + PosixFilePermission.GROUP_EXECUTE, + PosixFilePermission.OTHERS_READ, + PosixFilePermission.OTHERS_EXECUTE)); + private final Path liveRootPath; private final MetricsService metricsService; + private final BlockRemover blockRemover; + private final BlockPathResolver blockPathResolver; private long blockNodeFileNameIndex; - private Path currentBlockDir; + private long currentBlockNumber; /** * Use the corresponding builder to construct a new BlockAsDirWriter with the given parameters. * - * @param blockRemover the block remover to use for removing blocks - * @param folderPermissions the folder permissions to use for writing blocks, if null provided then defaults will be used * @param blockNodeContext the block node context to use for writing blocks + * @param blockRemover the block remover to use for removing blocks + * @param blockPathResolver used internally to resolve paths + * * @throws IOException if an error occurs while initializing the BlockAsDirWriter */ - BlockAsDirWriter( + protected BlockAsLocalDirWriter( + @NonNull final BlockNodeContext blockNodeContext, @NonNull final BlockRemover blockRemover, - final FileAttribute> folderPermissions, - @NonNull final BlockNodeContext blockNodeContext) + @NonNull final BlockPathResolver blockPathResolver) throws IOException { - LOGGER.log(INFO, "Initializing FileSystemBlockStorage"); + LOGGER.log(INFO, "Initializing %s...".formatted(getClass().getName())); + + this.metricsService = Objects.requireNonNull(blockNodeContext.metricsService()); + this.blockRemover = Objects.requireNonNull(blockRemover); + this.blockPathResolver = Objects.requireNonNull(blockPathResolver); final PersistenceStorageConfig config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); - - final Path blockNodeRootPath = Path.of(config.rootPath()); - LOGGER.log(INFO, "Block Node Root Path: " + blockNodeRootPath); - - this.blockNodeRootPath = blockNodeRootPath; - this.blockRemover = blockRemover; - this.metricsService = blockNodeContext.metricsService(); - - if (Objects.nonNull(folderPermissions)) { - this.folderPermissions = folderPermissions; - } else { - // default permissions for folders - this.folderPermissions = PosixFilePermissions.asFileAttribute(Set.of( - PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, - PosixFilePermission.OWNER_EXECUTE, - PosixFilePermission.GROUP_READ, - PosixFilePermission.GROUP_EXECUTE, - PosixFilePermission.OTHERS_READ, - PosixFilePermission.OTHERS_EXECUTE)); - } + this.liveRootPath = Path.of(config.liveRootPath()); // Initialize the block node root directory if it does not exist FileUtilities.createFolderPathIfNotExists( - blockNodeRootPath, INFO, this.folderPermissions, BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME); + liveRootPath, INFO, DEFAULT_FOLDER_PERMISSIONS, BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME); + } + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalDirWriter}. + * + * @param blockNodeContext valid, {@code non-null} instance of + * {@link BlockNodeContext} used to get the {@link MetricsService} + * @param blockRemover valid, {@code non-null} instance of + * {@link BlockRemover} used to remove blocks in case of cleanup + * @param blockPathResolver valid, {@code non-null} instance of + * {@link BlockPathResolver} used to resolve paths to Blocks + * @return a new, fully initialized instance of {@link BlockAsLocalDirWriter} + * @throws IOException if an error occurs while initializing the BlockAsDirWriter + */ + public static BlockAsLocalDirWriter of( + @NonNull final BlockNodeContext blockNodeContext, + @NonNull final BlockRemover blockRemover, + @NonNull final BlockPathResolver blockPathResolver) + throws IOException { + return new BlockAsLocalDirWriter(blockNodeContext, blockRemover, blockPathResolver); } /** * Writes the given block item to the filesystem. * - * @param blockItems the block item to write + * @param valueToWrite the block item to write * @throws IOException if an error occurs while writing the block item */ + @NonNull @Override - public Optional> write(@NonNull final List blockItems) + public Optional> write(@NonNull final List valueToWrite) throws IOException, ParseException { - - final Bytes unparsedBlockHeader = blockItems.getFirst().blockHeader(); + final Bytes unparsedBlockHeader = valueToWrite.getFirst().blockHeader(); if (unparsedBlockHeader != null) { resetState(BlockHeader.PROTOBUF.parse(unparsedBlockHeader)); } - for (BlockItemUnparsed blockItemUnparsed : blockItems) { + for (final BlockItemUnparsed blockItemUnparsed : valueToWrite) { final Path blockItemFilePath = calculateBlockItemPath(); - for (int retries = 0; ; retries++) { - try { - write(blockItemFilePath, blockItemUnparsed); - break; - } catch (IOException e) { - - LOGGER.log(ERROR, "Error writing the BlockItem protobuf to a file: ", e); - - // Remove the block if repairing the permissions fails - if (retries > 0) { - // Attempt to remove the block - blockRemover.remove(Long.parseLong(currentBlockDir.toString())); - throw e; - } else { - // Attempt to repair the permissions on the block path - // and the blockItem path - repairPermissions(blockNodeRootPath); - repairPermissions(calculateBlockPath()); - LOGGER.log(INFO, "Retrying to write the BlockItem protobuf to a file"); - } - } - } + doWrite(blockItemFilePath, blockItemUnparsed, 0); } - if (blockItems.getLast().hasBlockProof()) { + if (valueToWrite.getLast().hasBlockProof()) { metricsService.get(BlocksPersisted).increment(); - return Optional.of(blockItems); + return Optional.of(valueToWrite); + } else { + return Optional.empty(); } - - return Optional.empty(); } - /** - * Writes the given block item to the filesystem. This method is protected to allow for testing. - * - * @param blockItemFilePath the path to the block item file - * @param blockItem the block item to write - * @throws IOException if an error occurs while writing the block item - */ - protected void write(@NonNull final Path blockItemFilePath, @NonNull final BlockItemUnparsed blockItem) - throws IOException { - try (final FileOutputStream fos = new FileOutputStream(blockItemFilePath.toString())) { - // Write the Bytes directly - BlockItemUnparsed.PROTOBUF.toBytes(blockItem).writeTo(fos); - LOGGER.log(DEBUG, "Successfully wrote the block item file: {0}", blockItemFilePath); - } catch (IOException e) { + private void doWrite(final Path targetPath, final BlockItemUnparsed toWrite, final int attempt) throws IOException { + try (final FileOutputStream fos = new FileOutputStream(targetPath.toString())) { + BlockItemUnparsed.PROTOBUF.toBytes(toWrite).writeTo(fos); + LOGGER.log(DEBUG, "Successfully wrote the block item file: {0}", targetPath); + } catch (final IOException | UncheckedIOException e) { LOGGER.log(ERROR, "Error writing the BlockItem protobuf to a file: ", e); - throw e; + if (attempt > 0) { + // If the write operation fails after a retry, attempt to remove the current block + blockRemover.remove(currentBlockNumber); + throw e; + } else { + repairPermissions(liveRootPath); + repairPermissions(blockPathResolver.resolvePathToBlock(currentBlockNumber)); + LOGGER.log(INFO, "Retrying to write the BlockItem protobuf to a file"); + doWrite(targetPath, toWrite, attempt + 1); + } } } private void resetState(@NonNull final BlockHeader blockHeader) throws IOException { // Here a "block" is represented as a directory of BlockItems. // Create the "block" directory based on the block_number - currentBlockDir = Path.of(String.valueOf(blockHeader.number())); + // Block Number must be a whole number! + currentBlockNumber = Preconditions.requireWhole(blockHeader.number()); // Check the blockNodeRootPath permissions and // attempt to repair them if possible - repairPermissions(blockNodeRootPath); + repairPermissions(liveRootPath); // Construct the path to the block directory FileUtilities.createFolderPathIfNotExists( - calculateBlockPath(), DEBUG, folderPermissions, BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME); + blockPathResolver.resolvePathToBlock(currentBlockNumber), + DEBUG, + DEFAULT_FOLDER_PERMISSIONS, + BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME); // Reset blockNodeFileNameIndex = 0; @@ -199,20 +200,15 @@ private void repairPermissions(@NonNull final Path path) throws IOException { LOGGER.log(ERROR, "Block node root directory is not writable. Attempting to change the" + " permissions."); // Attempt to restore the permissions on the block node root directory - Files.setPosixFilePermissions(path, folderPermissions.value()); + Files.setPosixFilePermissions(path, DEFAULT_FOLDER_PERMISSIONS.value()); } } @NonNull private Path calculateBlockItemPath() { // Build the path to a .blk file - final Path blockPath = calculateBlockPath(); + final Path blockPath = blockPathResolver.resolvePathToBlock(currentBlockNumber); blockNodeFileNameIndex++; return blockPath.resolve(blockNodeFileNameIndex + BLOCK_FILE_EXTENSION); } - - @NonNull - private Path calculateBlockPath() { - return blockNodeRootPath.resolve(currentBlockDir); - } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriter.java new file mode 100644 index 000000000..fb9fed0a1 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriter.java @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.write; + +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.BlocksPersisted; + +import com.hedera.block.common.utils.Preconditions; +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.metrics.MetricsService; +import com.hedera.block.server.persistence.storage.path.BlockPathResolver; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.hapi.block.stream.output.BlockHeader; +import com.hedera.pbj.runtime.ParseException; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * A Block writer that handles writing of block-as-file. + */ +public final class BlockAsLocalFileWriter implements LocalBlockWriter> { + private final MetricsService metricsService; + private final BlockPathResolver blockPathResolver; + private List currentBlockItems; + private long currentBlockNumber = -1; + + /** + * Constructor. + * + * @param blockNodeContext valid, {@code non-null} instance of {@link BlockNodeContext} used to + * get the {@link MetricsService} + * @param blockPathResolver valid, {@code non-null} instance of {@link BlockPathResolver} used + * to resolve paths to Blocks + */ + private BlockAsLocalFileWriter( + @NonNull final BlockNodeContext blockNodeContext, @NonNull final BlockPathResolver blockPathResolver) { + this.metricsService = Objects.requireNonNull(blockNodeContext.metricsService()); + this.blockPathResolver = Objects.requireNonNull(blockPathResolver); + } + + /** + * This method creates and returns a new instance of {@link BlockAsLocalFileWriter}. + * + * @param blockNodeContext valid, {@code non-null} instance of + * {@link BlockNodeContext} used to get the {@link MetricsService} + * @param blockPathResolver valid, {@code non-null} instance of + * {@link BlockPathResolver} used to resolve paths to Blocks + * @return a new, fully initialized instance of {@link BlockAsLocalFileWriter} + */ + public static BlockAsLocalFileWriter of( + @NonNull final BlockNodeContext blockNodeContext, @NonNull final BlockPathResolver blockPathResolver) { + return new BlockAsLocalFileWriter(blockNodeContext, blockPathResolver); + } + + @NonNull + @Override + public Optional> write(@NonNull final List valueToWrite) + throws IOException, ParseException { + final BlockItemUnparsed firstItem = valueToWrite.getFirst(); + if (firstItem.hasBlockHeader()) { + currentBlockNumber = Preconditions.requireWhole( + BlockHeader.PROTOBUF.parse(firstItem.blockHeader()).number()); + currentBlockItems = new LinkedList<>(valueToWrite); + } else { + currentBlockItems.addAll(valueToWrite); + } + + if (valueToWrite.getLast().hasBlockProof()) { + final Optional> result = Optional.of(writeToFs()); + metricsService.get(BlocksPersisted).increment(); + resetState(); + return result; + } else { + return Optional.empty(); + } + } + + private List writeToFs() throws IOException { + final Path blockToWritePathResolved = blockPathResolver.resolvePathToBlock(currentBlockNumber); + Files.createDirectories(blockToWritePathResolved.getParent()); + Files.createFile(blockToWritePathResolved); + try (final FileOutputStream fos = new FileOutputStream(blockToWritePathResolved.toFile())) { + final BlockUnparsed blockToWrite = + BlockUnparsed.newBuilder().blockItems(currentBlockItems).build(); + BlockUnparsed.PROTOBUF.toBytes(blockToWrite).writeTo(fos); + } + return currentBlockItems; + } + + private void resetState() { + currentBlockItems = null; + currentBlockNumber = -1; + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java index eccbe4291..a05f6ae23 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java @@ -24,18 +24,18 @@ /** * BlockWriter defines the contract for writing block items to storage. * - * @param the type of the block item to write + * @param the type of the block item to write */ -public interface BlockWriter { - +public interface BlockWriter { /** * Write the block item to storage. * - * @param blockItem the block item to write to storage. - * @return an optional containing the block item written to storage if the block item was a - * block proof signaling the end of the block, an empty optional otherwise. - * @throws IOException when failing to write the block item to storage. + * @param valueToWrite to storage. + * @return an optional containing the item written to storage if the item + * was a block proof signaling the end of the block, an empty optional otherwise. + * @throws IOException when failing to write the item to storage. * @throws ParseException when failing to parse a block item. */ - Optional write(@NonNull final V blockItem) throws IOException, ParseException; + @NonNull + Optional write(@NonNull final T valueToWrite) throws IOException, ParseException; } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/LocalBlockWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/LocalBlockWriter.java new file mode 100644 index 000000000..d6c6536df --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/LocalBlockWriter.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.write; + +/** + * A marker interface that groups all writers that operate on a local file + * system. + */ +interface LocalBlockWriter extends BlockWriter {} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java index a805c1fd6..eb8d0b7eb 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java @@ -16,9 +16,6 @@ package com.hedera.block.server.persistence.storage.write; -import static java.lang.System.Logger.Level.INFO; - -import com.hedera.block.server.config.BlockNodeContext; import com.hedera.hapi.block.BlockItemUnparsed; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; @@ -30,29 +27,33 @@ * designed to isolate the Producer and Mediator components from storage implementation during testing while still * providing metrics and logging for troubleshooting. */ -public class NoOpBlockWriter implements BlockWriter> { +public final class NoOpBlockWriter implements BlockWriter> { + /** + * Constructor. + */ + private NoOpBlockWriter() {} /** - * Creates a new NoOpBlockWriter instance for testing and troubleshooting only. + * This method creates and returns a new instance of + * {@link NoOpBlockWriter}. * - * @param blockNodeContext the block node context + * @return a new, fully initialized instance of {@link NoOpBlockWriter} */ - public NoOpBlockWriter(BlockNodeContext blockNodeContext) { - System.getLogger(getClass().getName()).log(INFO, "Using " + getClass().getSimpleName()); + public static NoOpBlockWriter newInstance() { + return new NoOpBlockWriter(); } - /** - * {@inheritDoc} - */ + @NonNull @Override - public Optional> write(@NonNull List blockItems) throws IOException { - if (blockItems.getLast().hasBlockProof()) { + public Optional> write(@NonNull final List valueToWrite) + throws IOException { + if (valueToWrite.getLast().hasBlockProof()) { // Returning the BlockItems triggers a // PublishStreamResponse to be sent to the // upstream producer. - return Optional.of(blockItems); + return Optional.of(valueToWrite); + } else { + return Optional.empty(); } - - return Optional.empty(); } } diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 642467866..d00c922e4 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -6,6 +6,7 @@ exports com.hedera.block.server.consumer; exports com.hedera.block.server.exception; exports com.hedera.block.server.persistence.storage; + exports com.hedera.block.server.persistence.storage.path; exports com.hedera.block.server.persistence.storage.write; exports com.hedera.block.server.persistence.storage.read; exports com.hedera.block.server.persistence.storage.remove; diff --git a/server/src/main/resources/app.properties b/server/src/main/resources/app.properties index ce2fe915a..fcdc53254 100644 --- a/server/src/main/resources/app.properties +++ b/server/src/main/resources/app.properties @@ -8,3 +8,9 @@ prometheus.endpointPortNumber=9999 # Timeout for consumers to wait for block item before timing out. # Default is 1500 milliseconds. #consumer.timeoutThresholdMillis=1500 + +#Persistence Storage Config +#persistence.storage.liveRootPath= +#persistence.storage.archiveRootPath= +# @todo(#372) - default persistence type should be BLOCK_AS_LOCAL_FILE +persistence.storage.type=BLOCK_AS_LOCAL_DIRECTORY diff --git a/server/src/test/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializerTest.java b/server/src/test/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializerTest.java index 5c2f03dde..47b86af65 100644 --- a/server/src/test/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializerTest.java +++ b/server/src/test/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializerTest.java @@ -30,7 +30,9 @@ class ServerMappedConfigSourceInitializerTest { private static final ConfigMapping[] SUPPORTED_MAPPINGS = { new ConfigMapping("consumer.timeoutThresholdMillis", "CONSUMER_TIMEOUT_THRESHOLD_MILLIS"), - new ConfigMapping("persistence.storage.rootPath", "PERSISTENCE_STORAGE_ROOT_PATH"), + new ConfigMapping("persistence.storage.liveRootPath", "PERSISTENCE_STORAGE_LIVE_ROOT_PATH"), + new ConfigMapping("persistence.storage.archiveRootPath", "PERSISTENCE_STORAGE_ARCHIVE_ROOT_PATH"), + new ConfigMapping("persistence.storage.type", "PERSISTENCE_STORAGE_TYPE"), new ConfigMapping("service.delayMillis", "SERVICE_DELAY_MILLIS"), new ConfigMapping("mediator.ringBufferSize", "MEDIATOR_RING_BUFFER_SIZE"), new ConfigMapping("notifier.ringBufferSize", "NOTIFIER_RING_BUFFER_SIZE"), diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java index 98094390f..ad30047b6 100644 --- a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java @@ -18,9 +18,13 @@ import static com.hedera.block.server.Constants.FULL_SERVICE_NAME_BLOCK_ACCESS; import static com.hedera.block.server.Constants.SERVICE_NAME_BLOCK_ACCESS; +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -29,9 +33,11 @@ import com.hedera.block.server.pbj.PbjBlockAccessService; import com.hedera.block.server.pbj.PbjBlockAccessServiceProxy; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReader; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; +import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; @@ -57,6 +63,7 @@ @ExtendWith(MockitoExtension.class) class BlockAccessServiceTest { + private BlockAsLocalDirPathResolver pathResolverMock; @Mock private Pipeline responseObserver; @@ -64,29 +71,28 @@ class BlockAccessServiceTest { @Mock private BlockReader blockReader; - @Mock - private BlockWriter> blockWriter; - @Mock private ServiceStatus serviceStatus; - private static final int testTimeout = 1000; - @TempDir - private Path testPath; + private Path testLiveRootPath; private BlockNodeContext blockNodeContext; - private PersistenceStorageConfig config; + private PersistenceStorageConfig testConfig; private PbjBlockAccessService blockAccessService; @BeforeEach public void setUp() throws IOException { - blockNodeContext = - TestConfigUtil.getTestBlockNodeContext(Map.of("persistence.storage.rootPath", testPath.toString())); - config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + blockNodeContext = TestConfigUtil.getTestBlockNodeContext( + Map.of(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString())); + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); blockAccessService = new PbjBlockAccessServiceProxy(serviceStatus, blockReader, blockNodeContext); + + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = spy(BlockAsLocalDirPathResolver.of(testLiveRootPath)); } @Test @@ -106,9 +112,7 @@ public void testMethods() { @Test void testSingleBlockHappyPath() throws IOException, ParseException { - - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); final PbjBlockAccessService blockAccessService = new PbjBlockAccessServiceProxy(serviceStatus, blockReader, blockNodeContext); @@ -118,7 +122,7 @@ void testSingleBlockHappyPath() throws IOException, ParseException { // Generate and persist a block final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); final List blockItems = generateBlockItemsUnparsed(1); blockWriter.write(blockItems); diff --git a/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java b/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java index 13feee431..0f7312d69 100644 --- a/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java +++ b/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java @@ -19,13 +19,14 @@ import static com.hedera.block.server.util.PbjProtoTestUtils.buildAck; import static com.hedera.block.server.util.PbjProtoTestUtils.buildEmptyPublishStreamRequest; import static com.hedera.block.server.util.PbjProtoTestUtils.buildEmptySubscribeStreamRequest; +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; -import static java.lang.System.Logger; -import static java.lang.System.Logger.Level.INFO; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; @@ -39,13 +40,15 @@ import com.hedera.block.server.notifier.Notifier; import com.hedera.block.server.notifier.NotifierImpl; import com.hedera.block.server.persistence.StreamPersistenceHandlerImpl; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; +import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.service.ServiceStatusImpl; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.Acknowledgement; import com.hedera.hapi.block.BlockItemSetUnparsed; import com.hedera.hapi.block.BlockItemUnparsed; @@ -66,7 +69,6 @@ import com.lmax.disruptor.BatchEventProcessor; import io.helidon.webserver.WebServer; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.security.NoSuchAlgorithmException; import java.util.HashMap; @@ -75,17 +77,17 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) public class PbjBlockStreamServiceIntegrationTest { - - private final Logger LOGGER = System.getLogger(getClass().getName()); + private static final int testTimeout = 2000; + private BlockAsLocalDirPathResolver pathResolverMock; @Mock private Notifier notifier; @@ -129,27 +131,22 @@ public class PbjBlockStreamServiceIntegrationTest { @Mock private ServiceInterface.RequestOptions options; - private static final String TEMP_DIR = "block-node-unit-test-dir"; + @TempDir + private Path testLiveRootPath; - private Path testPath; private BlockNodeContext blockNodeContext; - - private static final int testTimeout = 2000; + private PersistenceStorageConfig testConfig; @BeforeEach public void setUp() throws IOException { - testPath = Files.createTempDirectory(TEMP_DIR); - LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); - Map properties = new HashMap<>(); - properties.put("persistence.storage.rootPath", testPath.toString()); - + properties.put(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString()); blockNodeContext = TestConfigUtil.getTestBlockNodeContext(properties); - } + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); - @AfterEach - public void tearDown() { - TestUtils.deleteDirectory(testPath.toFile()); + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = spy(BlockAsLocalDirPathResolver.of(testLiveRootPath)); } @Test @@ -252,7 +249,7 @@ public void testFullProducerConsumerHappyPath() throws IOException { // Use a real BlockWriter to test the full integration final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); final PbjBlockStreamServiceProxy pbjBlockStreamServiceProxy = buildBlockStreamService(blockWriter); // Register 3 producers - Opening a pipeline is not enough to register a producer. @@ -515,7 +512,7 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep // Use a spy to make sure the write() method throws an IOException final BlockWriter> blockWriter = - spy(BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build()); + spy(BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock)); doThrow(IOException.class).when(blockWriter).write(blockItems); final var streamMediator = buildStreamMediator(consumers, serviceStatus); diff --git a/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java b/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java index 0f32b98df..a3476325c 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java @@ -16,9 +16,10 @@ package com.hedera.block.server.persistence; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -26,10 +27,26 @@ import com.hedera.block.server.events.BlockNodeEventHandler; import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.mediator.SubscriptionHandler; +import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.notifier.Notifier; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig.StorageType; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalFilePathResolver; +import com.hedera.block.server.persistence.storage.path.BlockPathResolver; +import com.hedera.block.server.persistence.storage.path.NoOpBlockPathResolver; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReader; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalFileReader; import com.hedera.block.server.persistence.storage.read.BlockReader; +import com.hedera.block.server.persistence.storage.read.NoOpBlockReader; +import com.hedera.block.server.persistence.storage.remove.BlockAsLocalDirRemover; +import com.hedera.block.server.persistence.storage.remove.BlockAsLocalFileRemover; +import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.block.server.persistence.storage.remove.NoOpBlockRemover; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalFileWriter; import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.persistence.storage.write.NoOpBlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.hapi.block.BlockItemUnparsed; @@ -37,85 +54,215 @@ import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.swirlds.config.api.Configuration; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.Arrays; import java.util.List; -import org.junit.jupiter.api.BeforeEach; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) class PersistenceInjectionModuleTest { + @Mock + private BlockNodeContext blockNodeContextMock; @Mock - private BlockNodeContext blockNodeContext; + private PersistenceStorageConfig persistenceStorageConfigMock; @Mock - private PersistenceStorageConfig persistenceStorageConfig; + private BlockRemover blockRemoverMock; @Mock - private SubscriptionHandler subscriptionHandler; + private BlockPathResolver blockPathResolverMock; @Mock - private Notifier notifier; + private SubscriptionHandler subscriptionHandlerMock; @Mock - private BlockWriter> blockWriter; + private Notifier notifierMock; @Mock - private ServiceStatus serviceStatus; + private BlockWriter> blockWriterMock; - @BeforeEach - void setup() throws IOException { - // Setup any necessary mocks before each test - blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); - persistenceStorageConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); - } + @Mock + private ServiceStatus serviceStatusMock; - @Test - void testProvidesBlockWriter() { + @TempDir + private Path testLiveRootPath; + + /** + * This test aims to verify that the + * {@link PersistenceInjectionModule#providesBlockWriter} method will return + * the correct {@link BlockWriter} instance based on the {@link StorageType} + * parameter. The test verifies only the result type and not what is inside + * the instance! For the purpose of this test, what is inside the instance + * is not important. We aim to test the branch that will be taken based on + * the {@link StorageType} parameter in terms of the returned instance type. + * + * @param storageType parameterized, the {@link StorageType} to test + */ + @ParameterizedTest + @MethodSource("storageTypes") + void testProvidesBlockWriter(final StorageType storageType) { + final Configuration localConfigurationMock = mock(Configuration.class); + lenient().when(blockNodeContextMock.metricsService()).thenReturn(mock(MetricsService.class)); + when(blockNodeContextMock.configuration()).thenReturn(localConfigurationMock); + when(localConfigurationMock.getConfigData(PersistenceStorageConfig.class)) + .thenReturn(persistenceStorageConfigMock); + + lenient().when(persistenceStorageConfigMock.liveRootPath()).thenReturn(testLiveRootPath.toString()); + when(persistenceStorageConfigMock.type()).thenReturn(storageType); - BlockWriter> blockWriter = - PersistenceInjectionModule.providesBlockWriter(blockNodeContext); + final BlockWriter> actual = PersistenceInjectionModule.providesBlockWriter( + blockNodeContextMock, blockRemoverMock, blockPathResolverMock); - assertNotNull(blockWriter); + final Class targetInstanceType = + switch (storageType) { + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirWriter.class; + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFileWriter.class; + case NO_OP -> NoOpBlockWriter.class; + }; + assertThat(actual).isNotNull().isExactlyInstanceOf(targetInstanceType); } + /** + * This test aims to verify that the + * {@link PersistenceInjectionModule#providesBlockWriter} throws an + * {@link java.io.UncheckedIOException} if there is a problem with the + * creation of the {@link BlockWriter} instance. + */ @Test void testProvidesBlockWriter_IOException() { - BlockNodeContext blockNodeContext = mock(BlockNodeContext.class); - PersistenceStorageConfig persistenceStorageConfig = mock(PersistenceStorageConfig.class); - when(persistenceStorageConfig.rootPath()).thenReturn("/invalid_path/:invalid_directory"); - Configuration configuration = mock(Configuration.class); - when(blockNodeContext.configuration()).thenReturn(configuration); - when(configuration.getConfigData(PersistenceStorageConfig.class)).thenReturn(persistenceStorageConfig); - - // Expect a RuntimeException due to the IOException - RuntimeException exception = assertThrows( - RuntimeException.class, () -> PersistenceInjectionModule.providesBlockWriter(blockNodeContext)); - - // Verify the exception message - assertTrue(exception.getMessage().contains("Failed to create block writer")); + final BlockNodeContext blockNodeContextMock = mock(BlockNodeContext.class); + + final PersistenceStorageConfig localPersistenceStorageConfigMock = mock(PersistenceStorageConfig.class); + when(localPersistenceStorageConfigMock.liveRootPath()).thenReturn("/invalid_path/:invalid_directory"); + when(localPersistenceStorageConfigMock.type()).thenReturn(StorageType.BLOCK_AS_LOCAL_DIRECTORY); + + final Configuration configuration = mock(Configuration.class); + when(blockNodeContextMock.configuration()).thenReturn(configuration); + when(configuration.getConfigData(PersistenceStorageConfig.class)).thenReturn(localPersistenceStorageConfigMock); + + final MetricsService metricsServiceMock = mock(MetricsService.class); + when(blockNodeContextMock.metricsService()).thenReturn(metricsServiceMock); + + // Expect an UncheckedIOException due to the IOException + assertThatExceptionOfType(UncheckedIOException.class) + .isThrownBy(() -> PersistenceInjectionModule.providesBlockWriter( + blockNodeContextMock, blockRemoverMock, blockPathResolverMock)) + .withCauseInstanceOf(IOException.class) + .withMessage("Failed to create BlockWriter"); } - @Test - void testProvidesBlockReader() { + /** + * This test aims to verify that the + * {@link PersistenceInjectionModule#providesBlockReader} method will return + * the correct {@link BlockReader} instance based on the {@link StorageType} + * parameter. The test verifies only the result type and not what is inside + * the instance! For the purpose of this test, what is inside the instance + * is not important. We aim to test the branch that will be taken based on + * the {@link StorageType} parameter in terms of the returned instance type. + * + * @param storageType parameterized, the {@link StorageType} to test + */ + @ParameterizedTest + @MethodSource("storageTypes") + void testProvidesBlockReader(final StorageType storageType) { + lenient().when(persistenceStorageConfigMock.liveRootPath()).thenReturn(testLiveRootPath.toString()); + when(persistenceStorageConfigMock.type()).thenReturn(storageType); + + final BlockReader actual = + PersistenceInjectionModule.providesBlockReader(persistenceStorageConfigMock); - BlockReader blockReader = - PersistenceInjectionModule.providesBlockReader(persistenceStorageConfig); - assertNotNull(blockReader); + final Class targetInstanceType = + switch (storageType) { + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirReader.class; + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFileReader.class; + case NO_OP -> NoOpBlockReader.class; + }; + assertThat(actual).isNotNull().isExactlyInstanceOf(targetInstanceType); + } + + /** + * This test aims to verify that the + * {@link PersistenceInjectionModule#providesBlockRemover} method will + * return the correct {@link BlockRemover} instance based on the + * {@link StorageType} parameter. The test verifies only the result type and + * not what is inside the instance! For the purpose of this test, what is + * inside the instance is not important. We aim to test the branch that will + * be taken based on the {@link StorageType} parameter in terms of the + * returned instance type. + * + * @param storageType parameterized, the {@link StorageType} to test + */ + @ParameterizedTest + @MethodSource("storageTypes") + void testProvidesBlockRemover(final StorageType storageType) { + when(persistenceStorageConfigMock.type()).thenReturn(storageType); + + final BlockRemover actual = + PersistenceInjectionModule.providesBlockRemover(persistenceStorageConfigMock, blockPathResolverMock); + + final Class targetInstanceType = + switch (storageType) { + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirRemover.class; + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFileRemover.class; + case NO_OP -> NoOpBlockRemover.class; + }; + assertThat(actual).isNotNull().isExactlyInstanceOf(targetInstanceType); + } + + /** + * This test aims to verify that the + * {@link PersistenceInjectionModule#providesPathResolver(PersistenceStorageConfig)} + * method will return the correct {@link BlockPathResolver} instance based + * on the {@link StorageType} parameter. The test verifies only the result + * type and not what is inside the instance! For the purpose of this test, + * what is inside the instance is not important. We aim to test the branch + * that will be taken based on the {@link StorageType} parameter in terms of + * the returned instance type. + * + * @param storageType parameterized, the {@link StorageType} to test + */ + @ParameterizedTest + @MethodSource("storageTypes") + void testProvidesBlockPathResolver(final StorageType storageType) { + lenient().when(persistenceStorageConfigMock.liveRootPath()).thenReturn(testLiveRootPath.toString()); + when(persistenceStorageConfigMock.type()).thenReturn(storageType); + + final BlockPathResolver actual = PersistenceInjectionModule.providesPathResolver(persistenceStorageConfigMock); + + final Class targetInstanceType = + switch (storageType) { + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirPathResolver.class; + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFilePathResolver.class; + case NO_OP -> NoOpBlockPathResolver.class; + }; + assertThat(actual).isNotNull().isExactlyInstanceOf(targetInstanceType); } @Test void testProvidesStreamValidatorBuilder() throws IOException { - - BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); // Call the method under test - BlockNodeEventHandler> streamVerifier = + final BlockNodeEventHandler> streamVerifier = new StreamPersistenceHandlerImpl( - subscriptionHandler, notifier, blockWriter, blockNodeContext, serviceStatus); - + subscriptionHandlerMock, notifierMock, blockWriterMock, blockNodeContext, serviceStatusMock); assertNotNull(streamVerifier); } + + /** + * All {@link StorageType} dynamically generated. + */ + private static Stream storageTypes() { + return Arrays.stream(StorageType.values()).map(Arguments::of); + } } diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java index dfe4d8b3e..771bd4652 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java @@ -16,74 +16,211 @@ package com.hedera.block.server.persistence.storage; -import static org.junit.jupiter.api.Assertions.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.from; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig.StorageType; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; +import java.util.Arrays; import java.util.Comparator; import java.util.stream.Stream; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +/** + * Test class that tests the functionality of the + * {@link PersistenceStorageConfig} + */ class PersistenceStorageConfigTest { + private static final Path HASHGRAPH_ROOT_ABSOLUTE_PATH = + Path.of("hashgraph/").toAbsolutePath(); + private static final Path PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH = + HASHGRAPH_ROOT_ABSOLUTE_PATH.resolve("blocknode/data/"); - final String TEMP_DIR = "block-node-unit-test-dir"; - - @Test - void testPersistenceStorageConfig_happyPath() throws IOException { - - Path testPath = Files.createTempDirectory(TEMP_DIR); + @AfterEach + void tearDown() { + if (!Files.exists(HASHGRAPH_ROOT_ABSOLUTE_PATH)) { + return; + } + try (final Stream walk = Files.walk(HASHGRAPH_ROOT_ABSOLUTE_PATH)) { + walk.sorted(Comparator.reverseOrder()).forEach(p -> { + try { + Files.delete(p); + } catch (final IOException e) { + throw new RuntimeException(e); + } + }); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } - PersistenceStorageConfig persistenceStorageConfig = - new PersistenceStorageConfig(testPath.toString(), "PRODUCTION"); - assertEquals(testPath.toString(), persistenceStorageConfig.rootPath()); + /** + * This test aims to verify that the {@link PersistenceStorageConfig} class + * correctly returns the storage type that was set in the constructor. + * + * @param storageType parameterized, the storage type to test + */ + @ParameterizedTest + @MethodSource("storageTypes") + void testPersistenceStorageConfigStorageTypes(final StorageType storageType) { + final PersistenceStorageConfig actual = new PersistenceStorageConfig("", "", storageType); + assertThat(actual).returns(storageType, from(PersistenceStorageConfig::type)); } - @Test - void testPersistenceStorageConfig_emptyRootPath() throws IOException { - final String expectedDefaultRootPath = - Paths.get("").toAbsolutePath().resolve("data_empty").toString(); - // delete if exists - deleteDirectory(Paths.get(expectedDefaultRootPath)); + /** + * This test aims to verify that the {@link PersistenceStorageConfig} class + * correctly sets the live and archive root paths. + * + * @param liveRootPathToTest parameterized, the live root path to test + * @param expectedLiveRootPathToTest parameterized, the expected live root + * @param archiveRootPathToTest parameterized, the archive root path to test + * @param expectedArchiveRootPathToTest parameterized, the expected archive + * root + */ + @ParameterizedTest + @MethodSource({"validAbsoluteDefaultRootPaths", "validAbsoluteNonDefaultRootPaths"}) + void testPersistenceStorageConfigHappyPaths( + final String liveRootPathToTest, + final String expectedLiveRootPathToTest, + final String archiveRootPathToTest, + final String expectedArchiveRootPathToTest) { + final PersistenceStorageConfig actual = new PersistenceStorageConfig( + liveRootPathToTest, archiveRootPathToTest, StorageType.BLOCK_AS_LOCAL_FILE); + assertThat(actual) + .returns(expectedLiveRootPathToTest, from(PersistenceStorageConfig::liveRootPath)) + .returns(expectedArchiveRootPathToTest, from(PersistenceStorageConfig::archiveRootPath)); + } - PersistenceStorageConfig persistenceStorageConfig = - new PersistenceStorageConfig(getAbsoluteFolder("data_empty"), "PRODUCTION"); - assertEquals(expectedDefaultRootPath, persistenceStorageConfig.rootPath()); + /** + * This test aims to verify that the {@link PersistenceStorageConfig} class + * correctly throws an {@link UncheckedIOException} when either the live or + * archive root paths are invalid. + * + * @param invalidLiveRootPathToTest parameterized, the invalid live root + * path to test + * @param invalidArchiveRootPathToTest parameterized, the invalid archive + * root path to test + */ + @ParameterizedTest + @MethodSource({"invalidRootPaths"}) + void testPersistenceStorageConfigInvalidRootPaths( + final String invalidLiveRootPathToTest, final String invalidArchiveRootPathToTest) { + assertThatExceptionOfType(UncheckedIOException.class) + .isThrownBy(() -> new PersistenceStorageConfig( + invalidLiveRootPathToTest, invalidArchiveRootPathToTest, StorageType.BLOCK_AS_LOCAL_FILE)); } - @Test - void persistenceStorageConfig_throwsExceptionForRelativePath() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, () -> new PersistenceStorageConfig("relative/path", "PRODUCTION")); - assertEquals("relative/path Root path must be absolute", exception.getMessage()); + /** + * All storage types dynamically provided. + */ + private static Stream storageTypes() { + return Arrays.stream(StorageType.values()).map(Arguments::of); } - @Test - void persistenceStorageConfig_throwsRuntimeExceptionOnIOException() { - Path invalidPath = Paths.get("/invalid/path"); + /** + * The default absolute paths. We expect these to allow the persistence + * config to be instantiated. Providing a blank string is accepted, it will + * create the config instance with it's internal defaults. + */ + @SuppressWarnings("all") + private static Stream validAbsoluteDefaultRootPaths() { + final Path defaultLiveRootAbsolutePath = + PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH.resolve("live/").toAbsolutePath(); + final Path defaultArchiveRootAbsolutePath = + PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH.resolve("archive/").toAbsolutePath(); + + // test against the default liveRootPath and archiveRootPath + final String liveToTest1 = defaultLiveRootAbsolutePath.toString(); + final String liveExpected1 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest1 = defaultArchiveRootAbsolutePath.toString(); + final String archiveExpected1 = defaultArchiveRootAbsolutePath.toString(); + + // blank liveRootPath results in the default liveRootPath to be used + final String liveToTest2 = ""; + final String liveExpected2 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest2 = defaultArchiveRootAbsolutePath.toString(); + final String archiveExpected2 = defaultArchiveRootAbsolutePath.toString(); - RuntimeException exception = assertThrows( - RuntimeException.class, () -> new PersistenceStorageConfig(invalidPath.toString(), "PRODUCTION")); - assertInstanceOf(IOException.class, exception.getCause()); + // blank archiveRootPath results in the default archiveRootPath to be used + final String liveToTest3 = defaultLiveRootAbsolutePath.toString(); + final String liveExpected3 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest3 = ""; + final String archiveExpected3 = defaultArchiveRootAbsolutePath.toString(); + + // null liveRootPath results in the default liveRootPath to be used + final String liveToTest4 = null; + final String liveExpected4 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest4 = defaultArchiveRootAbsolutePath.toString(); + final String archiveExpected4 = defaultArchiveRootAbsolutePath.toString(); + + // null archiveRootPath results in the default archiveRootPath to be used + final String liveToTest5 = defaultLiveRootAbsolutePath.toString(); + final String liveExpected5 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest5 = null; + final String archiveExpected5 = defaultArchiveRootAbsolutePath.toString(); + + // blank liveRootPath and archiveRootPath results in the default liveRootPath and archiveRootPath to be used + final String liveToTest6 = ""; + final String liveExpected6 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest6 = ""; + final String archiveExpected6 = defaultArchiveRootAbsolutePath.toString(); + + // null liveRootPath and archiveRootPath results in the default liveRootPath and archiveRootPath to be used + final String liveToTest7 = null; + final String liveExpected7 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest7 = null; + final String archiveExpected7 = defaultArchiveRootAbsolutePath.toString(); + + return Stream.of( + Arguments.of(liveToTest1, liveExpected1, archiveToTest1, archiveExpected1), + Arguments.of(liveToTest2, liveExpected2, archiveToTest2, archiveExpected2), + Arguments.of(liveToTest3, liveExpected3, archiveToTest3, archiveExpected3), + Arguments.of(liveToTest4, liveExpected4, archiveToTest4, archiveExpected4), + Arguments.of(liveToTest5, liveExpected5, archiveToTest5, archiveExpected5), + Arguments.of(liveToTest6, liveExpected6, archiveToTest6, archiveExpected6), + Arguments.of(liveToTest7, liveExpected7, archiveToTest7, archiveExpected7)); } - public static void deleteDirectory(Path path) throws IOException { - if (!Files.exists(path)) { - return; - } - try (Stream walk = Files.walk(path)) { - walk.sorted(Comparator.reverseOrder()).forEach(p -> { - try { - Files.delete(p); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } + /** + * Somve valid absolute paths that are not the default paths. We expect + * these to allow the persistence config to be instantiated. + */ + @SuppressWarnings("all") + private static Stream validAbsoluteNonDefaultRootPaths() { + final String liveToTest1 = PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH + .resolve("nondefault/live/") + .toString(); + final String archiveToTest1 = PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH + .resolve("nondefault/archive/") + .toString(); + + final String liveToTest2 = PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH + .resolve("another/nondefault/live/") + .toString(); + final String archiveToTest2 = PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH + .resolve("another/nondefault/archive/") + .toString(); + + return Stream.of( + Arguments.of(liveToTest1, liveToTest1, archiveToTest1, archiveToTest1), + Arguments.of(liveToTest2, liveToTest2, archiveToTest2, archiveToTest2)); } - private String getAbsoluteFolder(String relativePath) { - return Paths.get(relativePath).toAbsolutePath().toString(); + /** + * Supplying blank is valid, both must be valid paths in order to be able + * to create the config instance. If either liveRootPath or archiveRootPath + * is invalid, we expect to fail. There cannot be invalid paths supplied. + */ + private static Stream invalidRootPaths() { + final String invalidPath = "/invalid_path/:invalid_directory"; + return Stream.of( + Arguments.of("", invalidPath), Arguments.of(invalidPath, ""), Arguments.of(invalidPath, invalidPath)); } } diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolverTest.java new file mode 100644 index 000000000..42a70bf21 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolverTest.java @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link BlockAsLocalDirPathResolver} class. + */ +class BlockAsLocalDirPathResolverTest { + @TempDir + private static Path TEST_LIVE_ROOT_PATH; + + private BlockAsLocalDirPathResolver toTest; + + @BeforeEach + void setUp() { + toTest = BlockAsLocalDirPathResolver.of(TEST_LIVE_ROOT_PATH); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalDirPathResolver#resolvePathToBlock(long)} correctly + * resolves the path to a block by a given number. For the + * block-as-local-directory storage strategy, the path to a block is simply + * the live root path appended with the given block number. + * + * @param toResolve parameterized, valid block number + * @param expected parameterized, expected path + */ + @ParameterizedTest + @MethodSource("validBlockNumbers") + void testSuccessfulPathResolution(final long toResolve, final Path expected) { + final Path actual = toTest.resolvePathToBlock(toResolve); + assertThat(actual).isNotNull().isAbsolute().isEqualByComparingTo(expected); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalDirPathResolver#resolvePathToBlock(long)} correctly + * throws an {@link IllegalArgumentException} when an invalid block number + * is provided. A block number is invalid if it is a strictly negative number. + * + * @param toResolve parameterized, invalid block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long toResolve) { + assertThatIllegalArgumentException().isThrownBy(() -> toTest.resolvePathToBlock(toResolve)); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L, TEST_LIVE_ROOT_PATH.resolve("0")), + Arguments.of(1L, TEST_LIVE_ROOT_PATH.resolve("1")), + Arguments.of(2L, TEST_LIVE_ROOT_PATH.resolve("2")), + Arguments.of(10L, TEST_LIVE_ROOT_PATH.resolve("10")), + Arguments.of(100L, TEST_LIVE_ROOT_PATH.resolve("100")), + Arguments.of(1_000L, TEST_LIVE_ROOT_PATH.resolve("1000")), + Arguments.of(10_000L, TEST_LIVE_ROOT_PATH.resolve("10000")), + Arguments.of(100_000L, TEST_LIVE_ROOT_PATH.resolve("100000")), + Arguments.of(1_000_000L, TEST_LIVE_ROOT_PATH.resolve("1000000")), + Arguments.of(10_000_000L, TEST_LIVE_ROOT_PATH.resolve("10000000")), + Arguments.of(100_000_000L, TEST_LIVE_ROOT_PATH.resolve("100000000")), + Arguments.of(1_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("1000000000")), + Arguments.of(10_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("10000000000")), + Arguments.of(100_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("100000000000")), + Arguments.of(1_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("1000000000000")), + Arguments.of(10_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("10000000000000")), + Arguments.of(100_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("100000000000000")), + Arguments.of(1_000_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("1000000000000000")), + Arguments.of(10_000_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("10000000000000000")), + Arguments.of(100_000_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("100000000000000000")), + Arguments.of(1_000_000_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("1000000000000000000")), + Arguments.of(Long.MAX_VALUE, TEST_LIVE_ROOT_PATH.resolve(String.valueOf(Long.MAX_VALUE)))); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolverTest.java new file mode 100644 index 000000000..32d0f0c2f --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolverTest.java @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link BlockAsLocalFilePathResolver} class. + */ +class BlockAsLocalFilePathResolverTest { + @TempDir + private static Path TEST_LIVE_ROOT_PATH; + + private BlockAsLocalFilePathResolver toTest; + + @BeforeEach + void setUp() { + toTest = BlockAsLocalFilePathResolver.of(TEST_LIVE_ROOT_PATH); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFilePathResolver#resolvePathToBlock(long)} correctly + * resolves the path to a block by a given number. For the + * block-as-file storage strategy, the path to a block is a trie structure + * where each digit of the block number is a directory and the block number + * itself is the file name. + * + * @param toResolve parameterized, valid block number + * @param expected parameterized, expected path + */ + @ParameterizedTest + @MethodSource("validBlockNumbers") + void testSuccessfulPathResolution(final long toResolve, final Path expected) { + final Path actual = toTest.resolvePathToBlock(toResolve); + assertThat(actual).isNotNull().isAbsolute().isEqualByComparingTo(expected); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFilePathResolver#resolvePathToBlock(long)} correctly + * throws an {@link IllegalArgumentException} when an invalid block number + * is provided. A block number is invalid if it is a strictly negative number. + * + * @param toResolve parameterized, invalid block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long toResolve) { + assertThatIllegalArgumentException().isThrownBy(() -> toTest.resolvePathToBlock(toResolve)); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of( + 0L, TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0000000000000000000.blk")), + Arguments.of( + 1L, TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0000000000000000001.blk")), + Arguments.of( + 2L, TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0000000000000000002.blk")), + Arguments.of( + 10L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/1/0000000000000000010.blk")), + Arguments.of( + 100L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/1/0/0000000000000000100.blk")), + Arguments.of( + 1_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/1/0/0/0000000000000001000.blk")), + Arguments.of( + 10_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/0/0/0/0/1/0/0/0/0000000000000010000.blk")), + Arguments.of( + 100_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/0/0/0/1/0/0/0/0/0000000000000100000.blk")), + Arguments.of( + 1_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/0/0/1/0/0/0/0/0/0000000000001000000.blk")), + Arguments.of( + 10_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/0/1/0/0/0/0/0/0/0000000000010000000.blk")), + Arguments.of( + 100_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/0/1/0/0/0/0/0/0/0/0000000000100000000.blk")), + Arguments.of( + 1_000_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/0/1/0/0/0/0/0/0/0/0/0000000001000000000.blk")), + Arguments.of( + 10_000_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/0/1/0/0/0/0/0/0/0/0/0/0000000010000000000.blk")), + Arguments.of( + 100_000_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/0/1/0/0/0/0/0/0/0/0/0/0/0000000100000000000.blk")), + Arguments.of( + 1_000_000_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/0/1/0/0/0/0/0/0/0/0/0/0/0/0000001000000000000.blk")), + Arguments.of( + 10_000_000_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/0/1/0/0/0/0/0/0/0/0/0/0/0/0/0000010000000000000.blk")), + Arguments.of( + 100_000_000_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/0/1/0/0/0/0/0/0/0/0/0/0/0/0/0/0000100000000000000.blk")), + Arguments.of( + 1_000_000_000_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/0/1/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0001000000000000000.blk")), + Arguments.of( + 10_000_000_000_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/0/1/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0010000000000000000.blk")), + Arguments.of( + 100_000_000_000_000_000L, + TEST_LIVE_ROOT_PATH.resolve("0/1/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0100000000000000000.blk")), + Arguments.of( + 1_000_000_000_000_000_000L, + TEST_LIVE_ROOT_PATH.resolve("1/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/0/1000000000000000000.blk")), + Arguments.of( + Long.MAX_VALUE, + TEST_LIVE_ROOT_PATH.resolve("9/2/2/3/3/7/2/0/3/6/8/5/4/7/7/5/8/0/9223372036854775807.blk"))); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolverTest.java new file mode 100644 index 000000000..4db3abc68 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolverTest.java @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link NoOpBlockPathResolver} class. + */ +class NoOpBlockPathResolverTest { + private NoOpBlockPathResolver toTest; + + @BeforeEach + void setUp() { + toTest = NoOpBlockPathResolver.newInstance(); + } + + /** + * This test aims to verify that the + * {@link NoOpBlockPathResolver#resolvePathToBlock(long)} correctly resolves + * the path to a block by a given number. The no-op resolver does nothing, + * always returns a path resolved under '/tmp' based on the blockNumber and + * has no preconditions check. E.g. for blockNumber 0, the resolved path is + * '/tmp/hashgraph/blocknode/data/0.tmp.blk'. + * + * @param toResolve parameterized, block number + */ + @ParameterizedTest + @MethodSource({"validBlockNumbers", "invalidBlockNumbers"}) + void testSuccessfulPathResolution(final long toResolve, final Path expected) { + final Path actual = toTest.resolvePathToBlock(toResolve); + assertThat(actual).isNotNull().isAbsolute().isEqualByComparingTo(expected); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L, "/tmp/hashgraph/blocknode/data/0.tmp.blk"), + Arguments.of(1L, "/tmp/hashgraph/blocknode/data/1.tmp.blk"), + Arguments.of(2L, "/tmp/hashgraph/blocknode/data/2.tmp.blk"), + Arguments.of(10L, "/tmp/hashgraph/blocknode/data/10.tmp.blk"), + Arguments.of(100L, "/tmp/hashgraph/blocknode/data/100.tmp.blk"), + Arguments.of(1_000L, "/tmp/hashgraph/blocknode/data/1000.tmp.blk"), + Arguments.of(10_000L, "/tmp/hashgraph/blocknode/data/10000.tmp.blk"), + Arguments.of(100_000L, "/tmp/hashgraph/blocknode/data/100000.tmp.blk"), + Arguments.of(1_000_000L, "/tmp/hashgraph/blocknode/data/1000000.tmp.blk"), + Arguments.of(10_000_000L, "/tmp/hashgraph/blocknode/data/10000000.tmp.blk"), + Arguments.of(100_000_000L, "/tmp/hashgraph/blocknode/data/100000000.tmp.blk"), + Arguments.of(1_000_000_000L, "/tmp/hashgraph/blocknode/data/1000000000.tmp.blk"), + Arguments.of(10_000_000_000L, "/tmp/hashgraph/blocknode/data/10000000000.tmp.blk"), + Arguments.of(100_000_000_000L, "/tmp/hashgraph/blocknode/data/100000000000.tmp.blk"), + Arguments.of(1_000_000_000_000L, "/tmp/hashgraph/blocknode/data/1000000000000.tmp.blk"), + Arguments.of(10_000_000_000_000L, "/tmp/hashgraph/blocknode/data/10000000000000.tmp.blk"), + Arguments.of(100_000_000_000_000L, "/tmp/hashgraph/blocknode/data/100000000000000.tmp.blk"), + Arguments.of(1_000_000_000_000_000L, "/tmp/hashgraph/blocknode/data/1000000000000000.tmp.blk"), + Arguments.of(10_000_000_000_000_000L, "/tmp/hashgraph/blocknode/data/10000000000000000.tmp.blk"), + Arguments.of(100_000_000_000_000_000L, "/tmp/hashgraph/blocknode/data/100000000000000000.tmp.blk"), + Arguments.of(1_000_000_000_000_000_000L, "/tmp/hashgraph/blocknode/data/1000000000000000000.tmp.blk"), + Arguments.of(Long.MAX_VALUE, "/tmp/hashgraph/blocknode/data/9223372036854775807.tmp.blk")); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L, "/tmp/hashgraph/blocknode/data/-1.tmp.blk"), + Arguments.of(-2L, "/tmp/hashgraph/blocknode/data/-2.tmp.blk"), + Arguments.of(-10L, "/tmp/hashgraph/blocknode/data/-10.tmp.blk"), + Arguments.of(-100L, "/tmp/hashgraph/blocknode/data/-100.tmp.blk"), + Arguments.of(-1_000L, "/tmp/hashgraph/blocknode/data/-1000.tmp.blk"), + Arguments.of(-10_000L, "/tmp/hashgraph/blocknode/data/-10000.tmp.blk"), + Arguments.of(-100_000L, "/tmp/hashgraph/blocknode/data/-100000.tmp.blk"), + Arguments.of(-1_000_000L, "/tmp/hashgraph/blocknode/data/-1000000.tmp.blk"), + Arguments.of(-10_000_000L, "/tmp/hashgraph/blocknode/data/-10000000.tmp.blk"), + Arguments.of(-100_000_000L, "/tmp/hashgraph/blocknode/data/-100000000.tmp.blk"), + Arguments.of(-1_000_000_000L, "/tmp/hashgraph/blocknode/data/-1000000000.tmp.blk"), + Arguments.of(-10_000_000_000L, "/tmp/hashgraph/blocknode/data/-10000000000.tmp.blk"), + Arguments.of(-100_000_000_000L, "/tmp/hashgraph/blocknode/data/-100000000000.tmp.blk"), + Arguments.of(-1_000_000_000_000L, "/tmp/hashgraph/blocknode/data/-1000000000000.tmp.blk"), + Arguments.of(-10_000_000_000_000L, "/tmp/hashgraph/blocknode/data/-10000000000000.tmp.blk"), + Arguments.of(-100_000_000_000_000L, "/tmp/hashgraph/blocknode/data/-100000000000000.tmp.blk"), + Arguments.of(-1_000_000_000_000_000L, "/tmp/hashgraph/blocknode/data/-1000000000000000.tmp.blk"), + Arguments.of(-10_000_000_000_000_000L, "/tmp/hashgraph/blocknode/data/-10000000000000000.tmp.blk"), + Arguments.of(-100_000_000_000_000_000L, "/tmp/hashgraph/blocknode/data/-100000000000000000.tmp.blk"), + Arguments.of(-1_000_000_000_000_000_000L, "/tmp/hashgraph/blocknode/data/-1000000000000000000.tmp.blk"), + Arguments.of(Long.MIN_VALUE, "/tmp/hashgraph/blocknode/data/-9223372036854775808.tmp.blk")); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReaderTest.java similarity index 61% rename from server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java rename to server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReaderTest.java index 6a86cc045..e43dffddc 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReaderTest.java @@ -17,20 +17,26 @@ package com.hedera.block.server.persistence.storage.read; import static com.hedera.block.server.Constants.BLOCK_FILE_EXTENSION; +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; import static com.hedera.block.server.util.PersistTestUtils.reverseByteArray; import static com.hedera.block.server.util.PersistTestUtils.writeBlockItemToPath; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; @@ -48,31 +54,38 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; -public class BlockAsDirReaderTest { - +public class BlockAsLocalDirReaderTest { @TempDir - private Path testPath; + private Path testLiveRootPath; + private BlockAsLocalDirPathResolver pathResolverMock; private BlockNodeContext blockNodeContext; - private PersistenceStorageConfig config; + private PersistenceStorageConfig testConfig; private List blockItems; @BeforeEach public void setUp() throws IOException { - blockNodeContext = - TestConfigUtil.getTestBlockNodeContext(Map.of("persistence.storage.rootPath", testPath.toString())); - config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + blockNodeContext = TestConfigUtil.getTestBlockNodeContext( + Map.of(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString())); + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); blockItems = generateBlockItemsUnparsed(1); + + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = spy(BlockAsLocalDirPathResolver.of(testLiveRootPath)); } @Test public void testReadBlockDoesNotExist() throws IOException, ParseException { - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); final Optional blockOpt = blockReader.read(10000); assertTrue(blockOpt.isEmpty()); } @@ -80,64 +93,43 @@ public void testReadBlockDoesNotExist() throws IOException, ParseException { @Test public void testReadPermsRepairSucceeded() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (BlockItemUnparsed blockItem : blockItems) { + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); + for (final BlockItemUnparsed blockItem : blockItems) { blockWriter.write(List.of(blockItem)); } // Make the block unreadable - removeBlockReadPerms(1, config); + removeBlockReadPerms(1, testConfig); // The default BlockReader will attempt to repair the permissions and should succeed - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); final Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); assertEquals(10, blockOpt.get().blockItems().size()); } - @Test - public void testRemoveBlockReadPermsRepairFailed() throws IOException, ParseException { - final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - blockWriter.write(blockItems); - - // Make the block unreadable - removeBlockReadPerms(1, config); - - // For this test, build the Reader with ineffective repair permissions to - // simulate a failed repair (root changed the perms, etc.) - final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config) - .folderPermissions(TestUtils.getNoPerms()) - .build(); - final Optional blockOpt = blockReader.read(1); - assertTrue(blockOpt.isEmpty()); - } - @Test public void testRemoveBlockItemReadPerms() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); blockWriter.write(blockItems); - removeBlockItemReadPerms(1, 1, config); + removeBlockItemReadPerms(1, 1, testConfig); - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); assertThrows(IOException.class, () -> blockReader.read(1)); } @Test public void testPathIsNotDirectory() throws IOException, ParseException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(testConfig.liveRootPath()); // Write a file named "1" where a directory should be writeBlockItemToPath(blockNodeRootPath.resolve(Path.of("1")), blockItems.getFirst()); // Should return empty because the path is not a directory - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); final Optional blockOpt = blockReader.read(1); assertTrue(blockOpt.isEmpty()); } @@ -145,15 +137,15 @@ public void testPathIsNotDirectory() throws IOException, ParseException { @Test public void testRepairReadPermsFails() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); blockWriter.write(blockItems); - removeBlockReadPerms(1, config); + removeBlockReadPerms(1, testConfig); // Use a spy on a subclass of the BlockAsDirReader to proxy calls // to the actual methods but to also throw an IOException when // the setPerm method is called. - final TestBlockAsDirReader blockReader = spy(new TestBlockAsDirReader(config)); + final TestBlockAsLocalDirReader blockReader = spy(new TestBlockAsLocalDirReader(testConfig)); doThrow(IOException.class).when(blockReader).setPerm(any(), any()); final Optional blockOpt = blockReader.read(1); @@ -164,12 +156,12 @@ public void testRepairReadPermsFails() throws IOException, ParseException { public void testBlockNodePathReadFails() throws IOException, ParseException { // Remove read perm on the root path - removePathReadPerms(Path.of(config.rootPath())); + removePathReadPerms(Path.of(testConfig.liveRootPath())); // Use a spy on a subclass of the BlockAsDirReader to proxy calls // to the actual methods but to also throw an IOException when // the setPerm method is called. - final TestBlockAsDirReader blockReader = spy(new TestBlockAsDirReader(config)); + final TestBlockAsLocalDirReader blockReader = spy(new TestBlockAsLocalDirReader(testConfig)); doThrow(IOException.class).when(blockReader).setPerm(any(), any()); final Optional blockOpt = blockReader.read(1); @@ -179,18 +171,17 @@ public void testBlockNodePathReadFails() throws IOException, ParseException { @Test public void testParseExceptionHandling() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); blockWriter.write(blockItems); // Read the block back and confirm it's read successfully - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); final Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); final PersistenceStorageConfig persistenceStorageConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); - final Path blockNodeRootPath = Path.of(persistenceStorageConfig.rootPath()); + final Path blockNodeRootPath = Path.of(persistenceStorageConfig.liveRootPath()); Path blockPath = blockNodeRootPath.resolve(String.valueOf(1)); byte[] bytes; @@ -211,8 +202,23 @@ public void testParseExceptionHandling() throws IOException, ParseException { assertThrows(ParseException.class, () -> blockReader.read(1)); } + /** + * This test aims to verify that the + * {@link BlockAsLocalDirReader#read(long)} correctly throws an + * {@link IllegalArgumentException} when an invalid block number is + * provided. A block number is invalid if it is a strictly negative number. + * + * @param blockNumber parameterized, block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long blockNumber) { + final BlockReader toTest = BlockAsLocalDirReader.of(testConfig); + assertThatIllegalArgumentException().isThrownBy(() -> toTest.read(blockNumber)); + } + public static void removeBlockReadPerms(int blockNumber, final PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(config.liveRootPath()); final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); removePathReadPerms(blockPath); } @@ -223,7 +229,7 @@ static void removePathReadPerms(final Path path) throws IOException { private void removeBlockItemReadPerms(int blockNumber, int blockItem, PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(config.liveRootPath()); final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); final Path blockItemPath = blockPath.resolve(blockItem + BLOCK_FILE_EXTENSION); Files.setPosixFilePermissions(blockItemPath, TestUtils.getNoRead().value()); @@ -231,9 +237,9 @@ private void removeBlockItemReadPerms(int blockNumber, int blockItem, Persistenc // TestBlockAsDirReader overrides the setPerm() method to allow a test spy to simulate an // IOException while allowing the real setPerm() method to remain protected. - private static final class TestBlockAsDirReader extends BlockAsDirReader { - public TestBlockAsDirReader(PersistenceStorageConfig config) { - super(config, null); + private static final class TestBlockAsLocalDirReader extends BlockAsLocalDirReader { + public TestBlockAsLocalDirReader(PersistenceStorageConfig config) { + super(config); } @Override @@ -242,4 +248,34 @@ public void setPerm(@NonNull final Path path, @NonNull final Set invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } } diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReaderTest.java new file mode 100644 index 000000000..495e3bac0 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReaderTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.read; + +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; + +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link BlockAsLocalFileReader} class. + */ +class BlockAsLocalFileReaderTest { + private BlockAsLocalFileReader toTest; + + @BeforeEach + void setUp() { + toTest = BlockAsLocalFileReader.newInstance(); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFileReader#read(long)} correctly reads a block with + * a given block number. + * + * @param toRead parameterized, block number + */ + @ParameterizedTest + @MethodSource("validBlockNumbers") + void testSuccessfulBlockRead(final long toRead) { + // todo currently throws UnsupportedOperationException + assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> toTest.read(toRead)); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFileReader#read(long)} correctly throws an + * {@link IllegalArgumentException} when an invalid block number is + * provided. A block number is invalid if it is a strictly negative number. + * + * @param toRead parameterized, block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long toRead) { + assertThatIllegalArgumentException().isThrownBy(() -> toTest.read(toRead)); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L), + Arguments.of(1L), + Arguments.of(2L), + Arguments.of(10L), + Arguments.of(100L), + Arguments.of(1_000L), + Arguments.of(10_000L), + Arguments.of(100_000L), + Arguments.of(1_000_000L), + Arguments.of(10_000_000L), + Arguments.of(100_000_000L), + Arguments.of(1_000_000_000L), + Arguments.of(10_000_000_000L), + Arguments.of(100_000_000_000L), + Arguments.of(1_000_000_000_000L), + Arguments.of(10_000_000_000_000L), + Arguments.of(100_000_000_000_000L), + Arguments.of(1_000_000_000_000_000L), + Arguments.of(10_000_000_000_000_000L), + Arguments.of(100_000_000_000_000_000L), + Arguments.of(1_000_000_000_000_000_000L), + Arguments.of(Long.MAX_VALUE)); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReaderTest.java new file mode 100644 index 000000000..403a3f42b --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReaderTest.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.read; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.pbj.runtime.ParseException; +import java.io.IOException; +import java.util.Optional; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link NoOpBlockReader} class. + */ +class NoOpBlockReaderTest { + private NoOpBlockReader toTest; + + @BeforeEach + void setUp() { + toTest = NoOpBlockReader.newInstance(); + } + + /** + * This test aims to verify that the + * {@link NoOpBlockReader#read(long)} does nothing and does not throw any + * exceptions. The no-op reader has no preconditions check as well. The + * method always returns an empty {@link Optional}. + * + * @param toRead parameterized, block number + */ + @ParameterizedTest + @MethodSource({"validBlockNumbers", "invalidBlockNumbers"}) + void testSuccessfulBlockReading(final long toRead) throws IOException, ParseException { + final Optional actual = toTest.read(toRead); + assertThat(actual).isNotNull().isEmpty(); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L), + Arguments.of(1L), + Arguments.of(2L), + Arguments.of(10L), + Arguments.of(100L), + Arguments.of(1_000L), + Arguments.of(10_000L), + Arguments.of(100_000L), + Arguments.of(1_000_000L), + Arguments.of(10_000_000L), + Arguments.of(100_000_000L), + Arguments.of(1_000_000_000L), + Arguments.of(10_000_000_000L), + Arguments.of(100_000_000_000L), + Arguments.of(1_000_000_000_000L), + Arguments.of(10_000_000_000_000L), + Arguments.of(100_000_000_000_000L), + Arguments.of(1_000_000_000_000_000L), + Arguments.of(10_000_000_000_000_000L), + Arguments.of(100_000_000_000_000_000L), + Arguments.of(1_000_000_000_000_000_000L), + Arguments.of(Long.MAX_VALUE)); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java deleted file mode 100644 index fe7921a53..000000000 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2024 Hedera Hashgraph, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.hedera.block.server.persistence.storage.remove; - -import static java.lang.System.Logger; -import static java.lang.System.Logger.Level.INFO; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; -import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; -import com.hedera.block.server.persistence.storage.write.BlockWriter; -import com.hedera.block.server.util.PersistTestUtils; -import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.hapi.block.BlockItemUnparsed; -import com.hedera.hapi.block.BlockUnparsed; -import com.hedera.pbj.runtime.ParseException; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class BlockAsDirRemoverTest { - - private final Logger LOGGER = System.getLogger(getClass().getName()); - - private static final String TEMP_DIR = "block-node-unit-test-dir"; - - private Path testPath; - private BlockNodeContext blockNodeContext; - private PersistenceStorageConfig testConfig; - - @BeforeEach - public void setUp() throws IOException { - testPath = Files.createTempDirectory(TEMP_DIR); - LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); - - blockNodeContext = - TestConfigUtil.getTestBlockNodeContext(Map.of("persistence.storage.rootPath", testPath.toString())); - testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); - } - - @Test - public void testRemoveNonExistentBlock() throws IOException, ParseException { - - // Write a block - final var blockItems = PersistTestUtils.generateBlockItemsUnparsed(1); - - final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (BlockItemUnparsed blockItem : blockItems) { - blockWriter.write(List.of(blockItem)); - } - - // Remove a block that does not exist - final BlockRemover blockRemover = new BlockAsDirRemover(testPath); - blockRemover.remove(2); - - // Verify the block was not removed - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); - Optional blockOpt = blockReader.read(1); - assert (blockOpt.isPresent()); - assertEquals( - blockItems.getFirst().blockHeader(), - blockOpt.get().blockItems().getFirst().blockHeader()); - - // Now remove the block - blockRemover.remove(1); - - // Verify the block is removed - blockOpt = blockReader.read(1); - assert (blockOpt.isEmpty()); - } -} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemoverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemoverTest.java new file mode 100644 index 000000000..c4fe51fad --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemoverTest.java @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.from; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReader; +import com.hedera.block.server.persistence.storage.read.BlockReader; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; +import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.util.PersistTestUtils; +import com.hedera.block.server.util.TestConfigUtil; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.pbj.runtime.ParseException; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class BlockAsLocalDirRemoverTest { + private BlockNodeContext blockNodeContext; + private PersistenceStorageConfig testConfig; + private BlockAsLocalDirPathResolver pathResolverMock; + private BlockAsLocalDirRemover toTest; + + @TempDir + private Path testLiveRootPath; + + @BeforeEach + public void setUp() throws IOException { + blockNodeContext = TestConfigUtil.getTestBlockNodeContext( + Map.of(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString())); + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = spy(BlockAsLocalDirPathResolver.of(testLiveRootPath)); + toTest = BlockAsLocalDirRemover.of(pathResolverMock); + } + + @Test + public void testRemoveNonExistentBlock() throws IOException, ParseException { + // Write a block + final List blockItems = PersistTestUtils.generateBlockItemsUnparsed(1); + + final BlockWriter> blockWriter = + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); + for (final BlockItemUnparsed blockItem : blockItems) { + blockWriter.write(List.of(blockItem)); + } + + // Remove a block that does not exist + toTest.remove(2); + + // Verify the block was not removed + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); + final Optional before = blockReader.read(1); + assertThat(before) + .isNotNull() + .isPresent() + .get() + .returns( + blockItems.getFirst().blockHeader(), + from(block -> block.blockItems().getFirst().blockHeader())); + + // Now remove the block + toTest.remove(1); + + // Verify the block is removed + final Optional after = blockReader.read(1); + assertThat(after).isNotNull().isEmpty(); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalDirRemover#remove(long)} correctly throws an + * {@link IllegalArgumentException} when an invalid block number is + * provided. A block number is invalid if it is a strictly negative number. + * + * @param toRemove parameterized, block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long toRemove) { + assertThatIllegalArgumentException().isThrownBy(() -> toTest.remove(toRemove)); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemoverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemoverTest.java new file mode 100644 index 000000000..4a03fa8e3 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemoverTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; + +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Test class for {@link BlockAsLocalFileRemover}. + */ +class BlockAsLocalFileRemoverTest { + private BlockAsLocalFileRemover toTest; + + @BeforeEach + void setUp() { + toTest = BlockAsLocalFileRemover.newInstance(); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFileRemover#remove(long)} correctly deletes a block + * with the given block number. + * + * @param toRemove parameterized, block number + */ + @ParameterizedTest + @MethodSource("validBlockNumbers") + void testSuccessfulBlockDeletion(final long toRemove) { + // todo currently throws UnsupportedOperationException + assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> toTest.remove(toRemove)); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFileRemover#remove(long)} correctly throws an + * {@link IllegalArgumentException} when an invalid block number is + * provided. A block number is invalid if it is a strictly negative number. + * + * @param toRemove parameterized, block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long toRemove) { + assertThatIllegalArgumentException().isThrownBy(() -> toTest.remove(toRemove)); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L), + Arguments.of(1L), + Arguments.of(2L), + Arguments.of(10L), + Arguments.of(100L), + Arguments.of(1_000L), + Arguments.of(10_000L), + Arguments.of(100_000L), + Arguments.of(1_000_000L), + Arguments.of(10_000_000L), + Arguments.of(100_000_000L), + Arguments.of(1_000_000_000L), + Arguments.of(10_000_000_000L), + Arguments.of(100_000_000_000L), + Arguments.of(1_000_000_000_000L), + Arguments.of(10_000_000_000_000L), + Arguments.of(100_000_000_000_000L), + Arguments.of(1_000_000_000_000_000L), + Arguments.of(10_000_000_000_000_000L), + Arguments.of(100_000_000_000_000_000L), + Arguments.of(1_000_000_000_000_000_000L), + Arguments.of(Long.MAX_VALUE)); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemoverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemoverTest.java new file mode 100644 index 000000000..8e90fbacb --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemoverTest.java @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Test class for {@link NoOpBlockRemover}. + */ +class NoOpBlockRemoverTest { + private NoOpBlockRemover toTest; + + @BeforeEach + void setUp() { + toTest = NoOpBlockRemover.newInstance(); + } + + /** + * This test aims to verify that the + * {@link NoOpBlockRemover#remove(long)} does nothing and does not throw any + * exceptions. The no-op remover has no preconditions check as well. + * + * @param toRemove parameterized, block number + */ + @ParameterizedTest + @MethodSource({"validBlockNumbers", "invalidBlockNumbers"}) + void testSuccessfulBlockDeletion(final long toRemove) { + assertThatNoException().isThrownBy(() -> toTest.remove(toRemove)); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L), + Arguments.of(1L), + Arguments.of(2L), + Arguments.of(10L), + Arguments.of(100L), + Arguments.of(1_000L), + Arguments.of(10_000L), + Arguments.of(100_000L), + Arguments.of(1_000_000L), + Arguments.of(10_000_000L), + Arguments.of(100_000_000L), + Arguments.of(1_000_000_000L), + Arguments.of(10_000_000_000L), + Arguments.of(100_000_000_000L), + Arguments.of(1_000_000_000_000L), + Arguments.of(10_000_000_000_000L), + Arguments.of(100_000_000_000_000L), + Arguments.of(1_000_000_000_000_000L), + Arguments.of(10_000_000_000_000_000L), + Arguments.of(100_000_000_000_000_000L), + Arguments.of(1_000_000_000_000_000_000L), + Arguments.of(Long.MAX_VALUE)); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriterTest.java similarity index 60% rename from server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java rename to server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriterTest.java index e5583f9d4..7e7b15984 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriterTest.java @@ -16,28 +16,28 @@ package com.hedera.block.server.persistence.storage.write; -import static com.hedera.block.server.persistence.storage.read.BlockAsDirReaderTest.removeBlockReadPerms; +import static com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReaderTest.removeBlockReadPerms; +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; -import static com.hedera.block.server.util.TestConfigUtil.DEFAULT_TEST_FOLDER_PERMISSIONS; -import static java.lang.System.Logger; -import static java.lang.System.Logger.Level.ERROR; -import static java.lang.System.Logger.Level.INFO; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIOException; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReader; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.remove.BlockAsDirRemover; +import com.hedera.block.server.persistence.storage.remove.BlockAsLocalDirRemover; import com.hedera.block.server.persistence.storage.remove.BlockRemover; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; @@ -45,74 +45,62 @@ import com.hedera.hapi.block.BlockUnparsed; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.pbj.runtime.ParseException; -import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; -import org.junit.jupiter.api.AfterEach; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; -public class BlockAsDirWriterTest { - - private final Logger LOGGER = System.getLogger(getClass().getName()); - - private static final String TEMP_DIR = "block-node-unit-test-dir"; - private static final String PERSISTENCE_STORAGE_ROOT_PATH_KEY = "persistence.storage.rootPath"; - - private Path testPath; +public class BlockAsLocalDirWriterTest { private BlockNodeContext blockNodeContext; private PersistenceStorageConfig testConfig; - + private BlockAsLocalDirPathResolver pathResolverMock; private List blockItems; + @TempDir + private Path testLiveRootPath; + @BeforeEach public void setUp() throws IOException { - testPath = Files.createTempDirectory(TEMP_DIR); - LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); - - blockNodeContext = - TestConfigUtil.getTestBlockNodeContext(Map.of(PERSISTENCE_STORAGE_ROOT_PATH_KEY, testPath.toString())); + blockNodeContext = TestConfigUtil.getTestBlockNodeContext( + Map.of(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString())); testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); blockItems = generateBlockItemsUnparsed(1); - } - @AfterEach - public void tearDown() { - if (!TestUtils.deleteDirectory(testPath.toFile())) { - LOGGER.log(ERROR, "Failed to delete temp directory: " + testPath.toString()); - } + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = spy(BlockAsLocalDirPathResolver.of(testLiveRootPath)); } @Test public void testWriterAndReaderHappyPath() throws IOException, ParseException { - final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext) - .folderPermissions(DEFAULT_TEST_FOLDER_PERMISSIONS) - .build(); + final BlockWriter> blockWriter = + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); for (int i = 0; i < 10; i++) { + final Optional> result = blockWriter.write(List.of(blockItems.get(i))); if (i == 9) { - Optional> result = blockWriter.write(List.of(blockItems.get(i))); if (result.isPresent()) { - assertEquals(blockItems.get(i), result.get().get(0)); + assertEquals(blockItems.get(i), result.get().getFirst()); } else { fail("The optional should contain the last block proof block item"); } } else { - Optional> result = blockWriter.write(List.of(blockItems.get(i))); assertTrue(result.isEmpty()); } } // Confirm the block - BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); + BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); @@ -142,7 +130,7 @@ public void testWriterAndReaderHappyPath() throws IOException, ParseException { public void testRemoveBlockWritePerms() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); // Change the permissions on the block node root directory removeRootWritePerms(testConfig); @@ -153,8 +141,7 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { assertFalse(result.isPresent()); // Confirm we're able to read 1 block item - BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); + BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); assertEquals(1, blockOpt.get().blockItems().size()); @@ -186,14 +173,9 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { @Test public void testUnrecoverableIOExceptionOnWrite() throws IOException, ParseException { - - final BlockRemover blockRemover = new BlockAsDirRemover(Path.of(testConfig.rootPath())); - // Use a spy to simulate an IOException when the first block item is written final BlockWriter> blockWriter = - spy(BlockAsDirWriterBuilder.newBuilder(blockNodeContext) - .blockRemover(blockRemover) - .build()); + spy(BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock)); doThrow(IOException.class).when(blockWriter).write(blockItems); assertThrows(IOException.class, () -> blockWriter.write(blockItems)); } @@ -202,7 +184,7 @@ public void testUnrecoverableIOExceptionOnWrite() throws IOException, ParseExcep public void testRemoveRootDirReadPerm() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); // Write the first block item to create the block // directory @@ -220,7 +202,7 @@ public void testRemoveRootDirReadPerm() throws IOException, ParseException { if (i == 9) { result = blockWriter.write(List.of(blockItems.get(i))); if (result.isPresent()) { - assertEquals(blockItems.get(i), result.get().get(0)); + assertEquals(blockItems.get(i), result.get().getFirst()); } else { fail("The optional should contain the last block proof block item"); } @@ -230,8 +212,7 @@ public void testRemoveRootDirReadPerm() throws IOException, ParseException { } } - BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); + BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); assertEquals(10, blockOpt.get().blockItems().size()); @@ -239,52 +220,56 @@ public void testRemoveRootDirReadPerm() throws IOException, ParseException { @Test public void testPartialBlockRemoval() throws IOException, ParseException { + final int expectedItemsPerBlock = 10; final List blockItems = generateBlockItemsUnparsed(3); - final BlockRemover blockRemover = new BlockAsDirRemover(Path.of(testConfig.rootPath())); - - // Use a spy of TestBlockAsDirWriter to proxy block items to the real writer - // for the first 22 block items. Then simulate an IOException on the 23rd block item - // thrown from a protected write method in the real class. This should trigger the - // blockRemover instance to remove the partially written block. - final TestBlockAsDirWriter blockWriter = spy(new TestBlockAsDirWriter(blockRemover, null, blockNodeContext)); - - for (int i = 0; i < 23; i++) { - // Prepare the block writer to call the actual write method - // for 23 block items - doCallRealMethod().when(blockWriter).write(same(List.of(blockItems.get(i)))); - } - - // Simulate an IOException when writing the 24th block item - // from an overridden write method in sub-class. - doThrow(IOException.class).when(blockWriter).write(any(), same(blockItems.get(23))); + final BlockRemover blockRemover = BlockAsLocalDirRemover.of(pathResolverMock); + final BlockAsLocalDirWriter toTest = BlockAsLocalDirWriter.of(blockNodeContext, blockRemover, pathResolverMock); // Now make the calls for (int i = 0; i < 23; i++) { - Optional> result = blockWriter.write(List.of(blockItems.get(i))); + final Optional> result = toTest.write(List.of(blockItems.get(i))); if (i == 9 || i == 19) { // The last block item in each block is the block proof // and should be returned by the write method assertTrue(result.isPresent()); - assertEquals(blockItems.get(i), result.get().get(0)); + assertEquals(blockItems.get(i), result.get().getFirst()); } else { // The write method should return an empty optional assertTrue(result.isEmpty()); } } - // Verify the IOException was thrown on the 23rd block item - assertThrows(IOException.class, () -> blockWriter.write(List.of(blockItems.get(23)))); + // simulate an IOException on the 23rd block item + final AtomicInteger callCount = new AtomicInteger(0); + doAnswer(invocation -> { + if (callCount.incrementAndGet() == 1) { + return Path.of("/invalid_path/:invalid_directory"); + } + return invocation.callRealMethod(); + }) + .when(pathResolverMock) + .resolvePathToBlock(3); + + assertThatIOException().isThrownBy(() -> toTest.write(List.of(blockItems.get(23)))); // Verify the partially written block was removed - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); Optional blockOpt = blockReader.read(3); assertTrue(blockOpt.isEmpty()); + final Path targetPathForBlock1 = pathResolverMock.resolvePathToBlock(1); + assertThat(targetPathForBlock1).isNotNull().exists().isDirectory(); + + final Path targetPathForBlock2 = pathResolverMock.resolvePathToBlock(2); + assertThat(targetPathForBlock2).isNotNull().exists().isDirectory(); + + final Path targetPathForBlock3 = pathResolverMock.resolvePathToBlock(3); + assertThat(targetPathForBlock3).isNotNull().doesNotExist(); + // Confirm blocks 1 and 2 still exist blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); - assertEquals(10, blockOpt.get().blockItems().size()); + assertEquals(expectedItemsPerBlock, blockOpt.get().blockItems().size()); assertEquals( 1, BlockHeader.PROTOBUF @@ -293,7 +278,7 @@ public void testPartialBlockRemoval() throws IOException, ParseException { blockOpt = blockReader.read(2); assertFalse(blockOpt.isEmpty()); - assertEquals(10, blockOpt.get().blockItems().size()); + assertEquals(expectedItemsPerBlock, blockOpt.get().blockItems().size()); assertEquals( 2, BlockHeader.PROTOBUF @@ -301,37 +286,72 @@ public void testPartialBlockRemoval() throws IOException, ParseException { .number()); } + /** + * This test aims to verify that the + * {@link BlockAsLocalDirWriter#write(List)} correctly throws an + * {@link IllegalArgumentException} when an invalid block number is + * provided. A block number is invalid if it is a strictly negative number. + * + * @param blockNumber parameterized, block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long blockNumber) throws IOException { + final BlockAsLocalDirWriter toTest = + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); + + final BlockHeader blockHeader = + BlockHeader.newBuilder().number(blockNumber).build(); + final BlockItemUnparsed blockItem = BlockItemUnparsed.newBuilder() + .blockHeader(BlockHeader.PROTOBUF.toBytes(blockHeader)) + .build(); + + assertThatIllegalArgumentException().isThrownBy(() -> toTest.write(List.of(blockItem))); + } + private void removeRootWritePerms(final PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(config.liveRootPath()); Files.setPosixFilePermissions(blockNodeRootPath, TestUtils.getNoWrite().value()); } private void removeRootReadPerms(final PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(config.liveRootPath()); Files.setPosixFilePermissions(blockNodeRootPath, TestUtils.getNoRead().value()); } private void removeBlockAllPerms(final int blockNumber, final PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(config.liveRootPath()); final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); Files.setPosixFilePermissions(blockPath, TestUtils.getNoPerms().value()); } - // TestBlockAsDirWriter overrides the write() method to allow a test spy to simulate an - // IOException while allowing the real write() method to remain protected. - private static final class TestBlockAsDirWriter extends BlockAsDirWriter { - public TestBlockAsDirWriter( - final BlockRemover blockRemover, - final FileAttribute> filePerms, - final BlockNodeContext blockNodeContext) - throws IOException { - super(blockRemover, filePerms, blockNodeContext); - } - - @Override - public void write(@NonNull final Path blockItemFilePath, @NonNull final BlockItemUnparsed blockItem) - throws IOException { - super.write(blockItemFilePath, blockItem); - } + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); } } diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriterTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriterTest.java new file mode 100644 index 000000000..042b0bde3 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriterTest.java @@ -0,0 +1,236 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.write; + +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.mockito.Mockito.spy; + +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalFilePathResolver; +import com.hedera.block.server.util.TestConfigUtil; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.hapi.block.stream.BlockProof; +import com.hedera.hapi.block.stream.output.BlockHeader; +import com.hedera.pbj.runtime.ParseException; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Test class for {@link BlockAsLocalFileWriter}. + */ +class BlockAsLocalFileWriterTest { + private BlockNodeContext blockNodeContext; + private PersistenceStorageConfig testConfig; + private BlockAsLocalFilePathResolver pathResolverMock; + + @TempDir + private Path testLiveRootPath; + + private BlockAsLocalFileWriter toTest; + + @BeforeEach + void setUp() throws IOException { + blockNodeContext = TestConfigUtil.getTestBlockNodeContext( + Map.of(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString())); + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = spy(BlockAsLocalFilePathResolver.of(testLiveRootPath)); + + toTest = BlockAsLocalFileWriter.of(blockNodeContext, pathResolverMock); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFileWriter#write(List)} writes correctly a given Block + * to the file system. The Block is written correctly if the input items to + * write begin with a {@link BlockHeader}, end with a {@link BlockProof}, + * the Block file is written to the correct path and the input items are + * returned. + */ + @Test + void testSuccessfulBlockWrite() throws IOException, ParseException { + final int expectedBlockItems = 10; + final List toWrite = generateBlockItemsUnparsed(1); + + assertThat(toWrite).isNotNull().isNotEmpty().satisfies(blockItemUnparsed -> { + assertThat(blockItemUnparsed).isNotNull().isNotEmpty().hasSize(10); + assertThat(blockItemUnparsed.getFirst().blockHeader()).isNotNull(); + assertThat(blockItemUnparsed.getLast().blockProof()).isNotNull(); + }); + + final BlockHeader targetHeader = + BlockHeader.PROTOBUF.parse(toWrite.getFirst().blockHeader()); + + final Path expectedPath = pathResolverMock.resolvePathToBlock(targetHeader.number()); + assertThat(expectedPath).isNotNull().doesNotExist(); + + final BlockUnparsed expectedBlockToWrite = + BlockUnparsed.newBuilder().blockItems(toWrite).build(); + final Optional> actual = toTest.write(toWrite); + assertThat(actual) + .isNotNull() + .isNotEmpty() + .get(InstanceOfAssertFactories.list(BlockItemUnparsed.class)) + .isNotNull() + .isNotEmpty() + .hasSize(expectedBlockItems) + .containsExactlyElementsOf(toWrite); + assertThat(expectedPath) + .isNotNull() + .exists() + .isNotEmptyFile() + .isReadable() + .hasBinaryContent( + BlockUnparsed.PROTOBUF.toBytes(expectedBlockToWrite).toByteArray()); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFileWriter#write(List)} writes correctly a given Block + * to the file system. The Block is written correctly if the input items to + * write begin with a {@link BlockHeader}, end with a {@link BlockProof}, + * the Block file is written to the correct path and the input items are + * returned. Items to can start with a block header and can end in a block + * proof. If they start with a block header, but not end with block proof, + * we expect nothing to be written and an empty optional to be returned. + * If they do not start with a block header and do not end with a block + * proof, we expect nothing to be written and an empty optional to be + * returned. If they do not start with a block header, but end with a block + * proof, we expect the block to be written and the input items to be + * returned. After writing, all block items must be returned. We kind of + * buffer the block items until we have a block proof in memory and then we + * persist them to the fs. + */ + @Test + void testSuccessfulBlockWritePartial() throws IOException, ParseException { + final int expectedBlockItems = 10; + final List toWrite = generateBlockItemsUnparsed(1); + + assertThat(toWrite).isNotNull().isNotEmpty().satisfies(blockItemUnparsed -> { + assertThat(blockItemUnparsed).isNotNull().isNotEmpty().hasSize(expectedBlockItems); + assertThat(blockItemUnparsed.getFirst().blockHeader()).isNotNull(); + assertThat(blockItemUnparsed.getLast().blockProof()).isNotNull(); + }); + + final List firstHalfToWrite = toWrite.subList(0, 5); + final List secondHalfToWrite = toWrite.subList(5, toWrite.size()); + + final BlockHeader targetHeader = + BlockHeader.PROTOBUF.parse(toWrite.getFirst().blockHeader()); + + final Path expectedPath = pathResolverMock.resolvePathToBlock(targetHeader.number()); + assertThat(expectedPath).isNotNull().doesNotExist(); + + final BlockUnparsed expectedBlockToWrite = + BlockUnparsed.newBuilder().blockItems(toWrite).build(); + + final Optional> actualOnFirstWrite = toTest.write(firstHalfToWrite); + assertThat(actualOnFirstWrite).isNotNull().isEmpty(); + assertThat(expectedPath).isNotNull().doesNotExist(); + + final Optional> actualOnSecondWrite = toTest.write(secondHalfToWrite); + assertThat(actualOnSecondWrite) + .isNotNull() + .isNotEmpty() + .get(InstanceOfAssertFactories.list(BlockItemUnparsed.class)) + .isNotNull() + .isNotEmpty() + .hasSize(expectedBlockItems) + .containsExactlyElementsOf(toWrite); + assertThat(expectedPath) + .isNotNull() + .exists() + .isNotEmptyFile() + .isReadable() + .hasBinaryContent( + BlockUnparsed.PROTOBUF.toBytes(expectedBlockToWrite).toByteArray()); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalDirWriter#write(List)} correctly throws an + * {@link IllegalArgumentException} when an invalid block number is + * provided. A block number is invalid if it is a strictly negative number. + * + * @param blockNumber parameterized, block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long blockNumber) throws IOException { + final BlockHeader blockHeader = + BlockHeader.newBuilder().number(blockNumber).build(); + final BlockItemUnparsed blockHeaderUnparsed = BlockItemUnparsed.newBuilder() + .blockHeader(BlockHeader.PROTOBUF.toBytes(blockHeader)) + .build(); + + final BlockProof blockProof = BlockProof.newBuilder().build(); + final BlockItemUnparsed blockProofUnparsed = BlockItemUnparsed.newBuilder() + .blockProof(BlockProof.PROTOBUF.toBytes(blockProof)) + .build(); + + assertThatIllegalArgumentException() + .isThrownBy(() -> toTest.write(List.of(blockHeaderUnparsed, blockProofUnparsed))); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriterTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriterTest.java new file mode 100644 index 000000000..a06085edc --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriterTest.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.write; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.stream.BlockProof; +import com.hedera.hapi.block.stream.output.BlockHeader; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Test class for {@link NoOpBlockWriter}. + */ +class NoOpBlockWriterTest { + private NoOpBlockWriter toTest; + + @BeforeEach + void setUp() { + toTest = NoOpBlockWriter.newInstance(); + } + + /** + * This test aims to verify that the {@link NoOpBlockWriter#write(List)} + * does nothing and does not throw any exceptions. The no-op writer has no + * preconditions check as well. The method will return the input list of + * block items unparsed if the list ends with a block proof. + */ + @Test + void testSuccessfulBlockWrite() throws IOException { + final BlockProof blockProof = BlockProof.newBuilder().build(); + final Bytes blockProofAsBytes = BlockProof.PROTOBUF.toBytes(blockProof); + final BlockItemUnparsed blockProofUnparsed = + BlockItemUnparsed.newBuilder().blockProof(blockProofAsBytes).build(); + final List expected = List.of(blockProofUnparsed); + + final Optional> actual = toTest.write(expected); + assertThat(actual).isNotNull().isNotEmpty().containsSame(expected); + } + + /** + * This test aims to verify that the {@link NoOpBlockWriter#write(List)} + * does nothing and does not throw any exceptions. The no-op writer has no + * preconditions check as well. The method will return an empty optional if + * the list does not end with a block proof. + */ + @Test + void testSuccessfulBlockWriteNoProof() throws IOException { + final BlockHeader blockHeader = BlockHeader.newBuilder().build(); + final Bytes blockHeaderAsBytes = BlockHeader.PROTOBUF.toBytes(blockHeader); + final BlockItemUnparsed blockHeaderUnparsed = + BlockItemUnparsed.newBuilder().blockHeader(blockHeaderAsBytes).build(); + final List expected = List.of(blockHeaderUnparsed); + + final Optional> actual = toTest.write(expected); + assertThat(actual).isNotNull().isEmpty(); + } +} diff --git a/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java b/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java index eac41ae0f..aabf544ac 100644 --- a/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java +++ b/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java @@ -33,8 +33,8 @@ import java.util.List; public final class PersistTestUtils { - private static final Logger LOGGER = System.getLogger(PersistTestUtils.class.getName()); + public static final String PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY = "persistence.storage.liveRootPath"; private PersistTestUtils() {} diff --git a/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java b/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java index 5253f9da2..b42e487d4 100644 --- a/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java +++ b/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java @@ -28,26 +28,12 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; import java.util.Collections; import java.util.Map; -import java.util.Set; public class TestConfigUtil { public static final String CONSUMER_TIMEOUT_THRESHOLD_KEY = "consumer.timeoutThresholdMillis"; public static final String MEDIATOR_RING_BUFFER_SIZE_KEY = "mediator.ringBufferSize"; - public static final FileAttribute> DEFAULT_TEST_FOLDER_PERMISSIONS = - PosixFilePermissions.asFileAttribute(Set.of( - PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, - PosixFilePermission.OWNER_EXECUTE, - PosixFilePermission.GROUP_READ, - PosixFilePermission.GROUP_EXECUTE, - PosixFilePermission.OTHERS_READ, - PosixFilePermission.OTHERS_EXECUTE)); - private static final String TEST_APP_PROPERTIES_FILE = "app.properties"; private TestConfigUtil() {} diff --git a/server/src/test/resources/app.properties b/server/src/test/resources/app.properties index f1ea37129..01e942648 100644 --- a/server/src/test/resources/app.properties +++ b/server/src/test/resources/app.properties @@ -2,3 +2,5 @@ prometheus.endpointEnabled=false mediator.ringBufferSize=1024 notifier.ringBufferSize=1024 +# @todo(#372) - default persistence type should be BLOCK_AS_LOCAL_FILE +persistence.storage.type=BLOCK_AS_LOCAL_DIRECTORY diff --git a/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java b/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java index 26bf66fe5..324916d20 100644 --- a/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java +++ b/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java @@ -36,9 +36,8 @@ */ @DisplayName("Positive Data Persistence Tests") public class PositiveDataPersistenceTests extends BaseSuite { - private final String[] GET_BLOCKS_COMMAND = new String[] {"ls", "data", "-1"}; - - private BlockStreamSimulatorApp blockStreamSimulatorApp; + // @todo(#371) - the default life/archive root path must be absolute starting from /opt + private final String[] GET_BLOCKS_COMMAND = new String[] {"ls", "hashgraph/blocknode/data/live", "-1"}; private Future simulatorThread; @@ -66,7 +65,7 @@ public void verifyBlockDataSavedInCorrectDirectory() throws InterruptedException String savedBlocksFolderBefore = getContainerCommandResult(GET_BLOCKS_COMMAND); int savedBlocksCountBefore = getSavedBlocksCount(savedBlocksFolderBefore); - blockStreamSimulatorApp = createBlockSimulator(); + final BlockStreamSimulatorApp blockStreamSimulatorApp = createBlockSimulator(); simulatorThread = startSimulatorInThread(blockStreamSimulatorApp); Thread.sleep(5000); blockStreamSimulatorApp.stop();