diff --git a/common/src/main/java/com/hedera/block/common/utils/Preconditions.java b/common/src/main/java/com/hedera/block/common/utils/Preconditions.java index 08be1e517..2289c3924 100644 --- a/common/src/main/java/com/hedera/block/common/utils/Preconditions.java +++ b/common/src/main/java/com/hedera/block/common/utils/Preconditions.java @@ -75,6 +75,65 @@ public static int requirePositive(final int toCheck) { return requirePositive(toCheck, null); } + /** + * This method asserts a given integer is a positive. An integer is positive + * if it is NOT equal to zero and is greater than zero. + * + * @param toCheck the integer to check if it is a positive power of two + * @param errorMessage the error message to be used in the exception if the + * input integer to check is not positive, if null, a default message will + * be used + * @return the number to check if it is positive + * @throws IllegalArgumentException if the input number to check is not + * positive + */ + public static int requirePositive(final int toCheck, final String errorMessage) { + if (0 >= toCheck) { + final String message = Objects.isNull(errorMessage) + ? "The input integer [%d] is required be positive.".formatted(toCheck) + : errorMessage; + throw new IllegalArgumentException(message); + } else { + return toCheck; + } + } + + /** + * This method asserts a given long is a positive. A long is positive + * if it is NOT equal to zero and is greater than zero. + * + * @param toCheck the long to check if it is a positive power of two + * @return the long to check if it is positive + * @throws IllegalArgumentException if the input long to check is not + * positive + */ + public static long requirePositive(final long toCheck) { + return requirePositive(toCheck, null); + } + + /** + * This method asserts a given long is a positive. A long is positive + * if it is NOT equal to zero and is greater than zero. + * + * @param toCheck the long to check if it is a positive power of two + * @param errorMessage the error message to be used in the exception if the + * input long to check is not positive, if null, a default message will + * be used + * @return the long to check if it is positive + * @throws IllegalArgumentException if the input long to check is not + * positive + */ + public static long requirePositive(final long toCheck, final String errorMessage) { + if (0L >= toCheck) { + final String message = Objects.isNull(errorMessage) + ? "The input long [%d] is required be positive.".formatted(toCheck) + : errorMessage; + throw new IllegalArgumentException(message); + } else { + return toCheck; + } + } + /** * This method asserts a given long is a whole number. A long is whole * if it is greater or equal to zero. @@ -103,34 +162,11 @@ public static long requireWhole(final long toCheck) { public static long requireWhole(final long toCheck, final String errorMessage) { if (toCheck >= 0) { return toCheck; - } - - final String message = Objects.isNull(errorMessage) - ? "The input integer [%d] is required be whole.".formatted(toCheck) - : errorMessage; - throw new IllegalArgumentException(message); - } - - /** - * This method asserts a given integer is a positive. An integer is positive - * if it is NOT equal to zero and is greater than zero. - * - * @param toCheck the integer to check if it is a positive power of two - * @param errorMessage the error message to be used in the exception if the - * input integer to check is not positive, if null, a default message will - * be used - * @return the number to check if it is positive - * @throws IllegalArgumentException if the input number to check is not - * positive - */ - public static int requirePositive(final int toCheck, final String errorMessage) { - if (0 >= toCheck) { + } else { final String message = Objects.isNull(errorMessage) - ? "The input integer [%d] is required be positive.".formatted(toCheck) + ? "The input integer [%d] is required be whole.".formatted(toCheck) : errorMessage; throw new IllegalArgumentException(message); - } else { - return toCheck; } } diff --git a/common/src/test/java/com/hedera/block/common/utils/PreconditionsTest.java b/common/src/test/java/com/hedera/block/common/utils/PreconditionsTest.java index b6c2dd8e2..31b6f609e 100644 --- a/common/src/test/java/com/hedera/block/common/utils/PreconditionsTest.java +++ b/common/src/test/java/com/hedera/block/common/utils/PreconditionsTest.java @@ -145,6 +145,44 @@ void testRequirePositiveFail(final int toTest) { .withMessage(testErrorMessage); } + /** + * This test aims to verify that the + * {@link Preconditions#requirePositive(long)} will return the input 'toTest' + * parameter if the positive check passes. Test includes overloads. + * + * @param toTest parameterized, the number to test + */ + @ParameterizedTest + @MethodSource("com.hedera.block.common.CommonsTestUtility#positiveIntegers") + void testRequirePositiveLongPass(final long toTest) { + final Consumer asserts = actual -> assertThat(actual).isPositive().isEqualTo(toTest); + + final long actual = Preconditions.requirePositive(toTest); + assertThat(actual).satisfies(asserts); + + final long actualOverload = Preconditions.requirePositive(toTest, "test error message"); + assertThat(actualOverload).satisfies(asserts); + } + + /** + * This test aims to verify that the + * {@link Preconditions#requirePositive(long)} will throw an + * {@link IllegalArgumentException} if the positive check fails. Test + * includes overloads. + * + * @param toTest parameterized, the number to test + */ + @ParameterizedTest + @MethodSource("com.hedera.block.common.CommonsTestUtility#zeroAndNegativeIntegers") + void testRequirePositiveLongFail(final long toTest) { + assertThatIllegalArgumentException().isThrownBy(() -> Preconditions.requirePositive(toTest)); + + final String testErrorMessage = "test error message"; + assertThatIllegalArgumentException() + .isThrownBy(() -> Preconditions.requirePositive(toTest, testErrorMessage)) + .withMessage(testErrorMessage); + } + /** * This test aims to verify that the * {@link Preconditions#requirePowerOfTwo(int)} will return the input diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 24aad29a3..25592c617 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -36,8 +36,10 @@ mainModuleInfo { testModuleInfo { annotationProcessor("dagger.compiler") requires("org.junit.jupiter.api") + requires("org.junit.jupiter.params") requires("org.mockito") requires("org.mockito.junit.jupiter") + requires("org.assertj.core") requiresStatic("com.github.spotbugs.annotations") } diff --git a/server/docker/logging.properties b/server/docker/logging.properties index 60bb91bd6..c703a1553 100644 --- a/server/docker/logging.properties +++ b/server/docker/logging.properties @@ -24,7 +24,7 @@ io.helidon.common.level=INFO # Configure specific Block Node loggers #com.hedera.block.server.producer.ProducerBlockItemObserver.level=FINE #com.hedera.block.server.mediator.LiveStreamMediatorImpl.level=FINE -#com.hedera.block.server.persistence.storage.write.BlockAsDirWriter.level=FINE +#com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter.level=FINE #com.hedera.block.server.consumer.ConsumerStreamResponseObserver.level=FINE #com.hedera.block.server.pbj.PbjBlockStreamServiceProxy.level=FINE diff --git a/server/docker/update-env.sh b/server/docker/update-env.sh index 29a593980..0fa641586 100755 --- a/server/docker/update-env.sh +++ b/server/docker/update-env.sh @@ -28,6 +28,8 @@ fi if [ true = "$is_smoke_test" ]; then # add smoke test variables + # @todo(#372) - default persistence type should be BLOCK_AS_LOCAL_FILE + echo "PERSISTENCE_STORAGE_TYPE=BLOCK_AS_LOCAL_DIRECTORY" >> .env echo "MEDIATOR_RING_BUFFER_SIZE=1024" >> .env echo "NOTIFIER_RING_BUFFER_SIZE=1024" >> .env echo "JAVA_OPTS='-Xms4G -Xmx4G'" >> .env diff --git a/server/docs/configuration.md b/server/docs/configuration.md index cda8487c1..5dc5f08e6 100644 --- a/server/docs/configuration.md +++ b/server/docs/configuration.md @@ -9,12 +9,14 @@ The default configuration allows users to quickly get up and running without hav ease of use at the trade-off of some insecure default configuration. Most configuration settings have appropriate defaults and can be left unchanged. It is recommended to browse the properties below and adjust to your needs. -| Environment Variable | Description | Default Value | -|:----------------------------------|:--------------------------------------------------------------------------------------------------------------|:--------------| -| PERSISTENCE_STORAGE_ROOT_PATH | The root path for the storage, if not provided will attempt to create a `data` on the working dir of the app. | | -| CONSUMER_TIMEOUT_THRESHOLD_MILLIS | Time to wait for subscribers before disconnecting in milliseconds | 1500 | -| SERVICE_DELAY_MILLIS | Service shutdown delay in milliseconds | 500 | -| MEDIATOR_RING_BUFFER_SIZE | Size of the ring buffer used by the mediator (must be a power of 2) | 67108864 | -| NOTIFIER_RING_BUFFER_SIZE | Size of the ring buffer used by the notifier (must be a power of 2) | 2048 | -| SERVER_PORT | The port the server will listen on | 8080 | -| SERVER_MAX_MESSAGE_SIZE_BYTES | The maximum size of a message frame in bytes | 1048576 | +| Environment Variable | Description | Default Value | +|:---|:---|---:| +| PERSISTENCE_STORAGE_LIVE_ROOT_PATH | The root path for the live storage. The provided path must be absolute! | | +| PERSISTENCE_STORAGE_ARCHIVE_ROOT_PATH | The root path for the archive storage. The provided path must be absolute! | | +| PERSISTENCE_STORAGE_TYPE | Type of the persistence storage | BLOCK_AS_LOCAL_FILE | +| CONSUMER_TIMEOUT_THRESHOLD_MILLIS | Time to wait for subscribers before disconnecting in milliseconds | 1500 | +| SERVICE_DELAY_MILLIS | Service shutdown delay in milliseconds | 500 | +| MEDIATOR_RING_BUFFER_SIZE | Size of the ring buffer used by the mediator (must be a power of 2) | 67108864 | +| NOTIFIER_RING_BUFFER_SIZE | Size of the ring buffer used by the notifier (must be a power of 2) | 2048 | +| SERVER_PORT | The port the server will listen on | 8080 | +| SERVER_MAX_MESSAGE_SIZE_BYTES | The maximum size of a message frame in bytes | 1048576 | diff --git a/server/src/main/java/com/hedera/block/server/Constants.java b/server/src/main/java/com/hedera/block/server/Constants.java index 047243a4d..1870db7ef 100644 --- a/server/src/main/java/com/hedera/block/server/Constants.java +++ b/server/src/main/java/com/hedera/block/server/Constants.java @@ -18,8 +18,11 @@ /** Constants used in the BlockNode service. */ public final class Constants { - /** Constant mapped to the semantic name of the Block Node root directory */ - public static final String BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME = "Block Node Root Directory"; + /** Constant mapped to the semantic name of the Block Node live root directory */ + public static final String BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME = "Block Node Live Root Directory"; + + /** Constant mapped to the semantic name of the Block Node archive root directory */ + public static final String BLOCK_NODE_ARCHIVE_ROOT_DIRECTORY_SEMANTIC_NAME = "Block Node Archive Root Directory"; /** Constant mapped to PbjProtocolProvider.CONFIG_NAME in the PBJ Helidon Plugin */ public static final String PBJ_PROTOCOL_PROVIDER_CONFIG_NAME = "pbj"; diff --git a/server/src/main/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializer.java b/server/src/main/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializer.java index 67b2d8aa0..1b441de09 100644 --- a/server/src/main/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializer.java +++ b/server/src/main/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializer.java @@ -28,7 +28,9 @@ public final class ServerMappedConfigSourceInitializer { private static final List MAPPINGS = List.of( new ConfigMapping("consumer.timeoutThresholdMillis", "CONSUMER_TIMEOUT_THRESHOLD_MILLIS"), - new ConfigMapping("persistence.storage.rootPath", "PERSISTENCE_STORAGE_ROOT_PATH"), + new ConfigMapping("persistence.storage.liveRootPath", "PERSISTENCE_STORAGE_LIVE_ROOT_PATH"), + new ConfigMapping("persistence.storage.archiveRootPath", "PERSISTENCE_STORAGE_ARCHIVE_ROOT_PATH"), + new ConfigMapping("persistence.storage.type", "PERSISTENCE_STORAGE_TYPE"), new ConfigMapping("service.delayMillis", "SERVICE_DELAY_MILLIS"), new ConfigMapping("mediator.ringBufferSize", "MEDIATOR_RING_BUFFER_SIZE"), new ConfigMapping("notifier.ringBufferSize", "NOTIFIER_RING_BUFFER_SIZE"), diff --git a/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java b/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java index 205c17e1a..59dea3097 100644 --- a/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java +++ b/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java @@ -28,7 +28,7 @@ public interface MetricsService { * @param key to get a specific counter * @return the counter */ - Counter get(@NonNull BlockNodeMetricTypes.Counter key); + Counter get(@NonNull final BlockNodeMetricTypes.Counter key); /** * Use this method to get a specific gauge for the given metric type. @@ -36,5 +36,5 @@ public interface MetricsService { * @param key to get a specific gauge * @return the gauge */ - LongGauge get(@NonNull BlockNodeMetricTypes.Gauge key); + LongGauge get(@NonNull final BlockNodeMetricTypes.Gauge key); } diff --git a/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java b/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java index 776318333..0a38cd5ee 100644 --- a/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java +++ b/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java @@ -21,6 +21,7 @@ import com.swirlds.metrics.api.Metrics; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.EnumMap; +import java.util.Objects; import javax.inject.Inject; /** @@ -29,10 +30,8 @@ *

Metrics are updated by calling the appropriate method on the metric object instance. For * example, to increment a counter, call {@link Counter#increment()}. */ -public class MetricsServiceImpl implements MetricsService { - +public final class MetricsServiceImpl implements MetricsService { private static final String CATEGORY = "hedera_block_node"; - private final EnumMap counters = new EnumMap<>(BlockNodeMetricTypes.Counter.class); private final EnumMap gauges = @@ -45,22 +44,21 @@ public class MetricsServiceImpl implements MetricsService { */ @Inject public MetricsServiceImpl(@NonNull final Metrics metrics) { + Objects.requireNonNull(metrics); // Initialize the counters - for (BlockNodeMetricTypes.Counter counter : BlockNodeMetricTypes.Counter.values()) { + for (final BlockNodeMetricTypes.Counter counter : BlockNodeMetricTypes.Counter.values()) { counters.put( counter, - metrics.getOrCreate( - new Counter.Config(CATEGORY, counter.grafanaLabel()) - .withDescription(counter.description()))); + metrics.getOrCreate(new Counter.Config(CATEGORY, counter.grafanaLabel()) + .withDescription(counter.description()))); } // Initialize the gauges - for (BlockNodeMetricTypes.Gauge gauge : BlockNodeMetricTypes.Gauge.values()) { + for (final BlockNodeMetricTypes.Gauge gauge : BlockNodeMetricTypes.Gauge.values()) { gauges.put( gauge, metrics.getOrCreate( - new LongGauge.Config(CATEGORY, gauge.grafanaLabel()) - .withDescription(gauge.description()))); + new LongGauge.Config(CATEGORY, gauge.grafanaLabel()).withDescription(gauge.description()))); } } @@ -72,8 +70,8 @@ public MetricsServiceImpl(@NonNull final Metrics metrics) { */ @NonNull @Override - public Counter get(@NonNull BlockNodeMetricTypes.Counter key) { - return counters.get(key); + public Counter get(@NonNull final BlockNodeMetricTypes.Counter key) { + return counters.get(Objects.requireNonNull(key)); } /** @@ -84,7 +82,7 @@ public Counter get(@NonNull BlockNodeMetricTypes.Counter key) { */ @NonNull @Override - public LongGauge get(@NonNull BlockNodeMetricTypes.Gauge key) { - return gauges.get(key); + public LongGauge get(@NonNull final BlockNodeMetricTypes.Gauge key) { + return gauges.get(Objects.requireNonNull(key)); } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java b/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java index 518033e8c..5a0879ca2 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java @@ -20,9 +20,21 @@ import com.hedera.block.server.events.BlockNodeEventHandler; import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig.StorageType; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalFilePathResolver; +import com.hedera.block.server.persistence.storage.path.BlockPathResolver; +import com.hedera.block.server.persistence.storage.path.NoOpBlockPathResolver; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReader; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalFileReader; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; +import com.hedera.block.server.persistence.storage.read.NoOpBlockReader; +import com.hedera.block.server.persistence.storage.remove.BlockAsLocalDirRemover; +import com.hedera.block.server.persistence.storage.remove.BlockAsLocalFileRemover; +import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.block.server.persistence.storage.remove.NoOpBlockRemover; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalFileWriter; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.persistence.storage.write.NoOpBlockWriter; import com.hedera.hapi.block.BlockItemUnparsed; @@ -31,47 +43,107 @@ import dagger.Binds; import dagger.Module; import dagger.Provides; +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; import java.util.List; +import java.util.Objects; import javax.inject.Singleton; /** A Dagger module for providing dependencies for Persistence Module. */ @Module public interface PersistenceInjectionModule { - /** * Provides a block writer singleton using the block node context. * * @param blockNodeContext the application context + * @param blockRemover the block remover + * @param blockPathResolver the block path resolver * @return a block writer singleton */ @Provides @Singleton - static BlockWriter> providesBlockWriter(BlockNodeContext blockNodeContext) { - final String persistenceType = blockNodeContext + static BlockWriter> providesBlockWriter( + @NonNull final BlockNodeContext blockNodeContext, + @NonNull final BlockRemover blockRemover, + @NonNull final BlockPathResolver blockPathResolver) { + Objects.requireNonNull(blockRemover); + Objects.requireNonNull(blockPathResolver); + final StorageType persistenceType = blockNodeContext .configuration() .getConfigData(PersistenceStorageConfig.class) .type(); - if ("NOOP".equalsIgnoreCase(persistenceType)) { - return new NoOpBlockWriter(blockNodeContext); - } try { - return BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - } catch (IOException e) { - throw new RuntimeException("Failed to create block writer", e); + return switch (persistenceType) { + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFileWriter.of( + blockNodeContext, blockRemover, blockPathResolver); + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirWriter.of( + blockNodeContext, blockRemover, blockPathResolver); + case NO_OP -> NoOpBlockWriter.newInstance(); + }; + } catch (final IOException e) { + // we cannot have checked exceptions with dagger @Provides + throw new UncheckedIOException("Failed to create BlockWriter", e); } } /** * Provides a block reader singleton using the persistence storage config. * - * @param config the persistence storage configuration needed to build the block reader + * @param config the persistence storage configuration needed to build the + * block reader * @return a block reader singleton */ @Provides @Singleton - static BlockReader providesBlockReader(PersistenceStorageConfig config) { - return BlockAsDirReaderBuilder.newBuilder(config).build(); + static BlockReader providesBlockReader(@NonNull final PersistenceStorageConfig config) { + final StorageType persistenceType = config.type(); + return switch (persistenceType) { + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFileReader.newInstance(); + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirReader.of(config); + case NO_OP -> NoOpBlockReader.newInstance(); + }; + } + + /** + * Provides a block remover singleton using the persistence storage config. + * + * @param config the persistence storage configuration needed to build the + * block remover + * @param blockPathResolver the block path resolver + * @return a block remover singleton + */ + @Provides + @Singleton + static BlockRemover providesBlockRemover( + @NonNull final PersistenceStorageConfig config, @NonNull final BlockPathResolver blockPathResolver) { + Objects.requireNonNull(blockPathResolver); + final StorageType persistenceType = config.type(); + return switch (persistenceType) { + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFileRemover.newInstance(); + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirRemover.of(blockPathResolver); + case NO_OP -> NoOpBlockRemover.newInstance(); + }; + } + + /** + * Provides a path resolver singleton using the persistence storage config. + * + * @param config the persistence storage configuration needed to build the + * path resolver + * @return a path resolver singleton + */ + @Provides + @Singleton + static BlockPathResolver providesPathResolver(@NonNull final PersistenceStorageConfig config) { + final StorageType persistenceType = config.type(); + final Path blockStorageRoot = Path.of(config.liveRootPath()); + return switch (persistenceType) { + case BLOCK_AS_LOCAL_FILE -> BlockAsLocalFilePathResolver.of(blockStorageRoot); + case BLOCK_AS_LOCAL_DIRECTORY -> BlockAsLocalDirPathResolver.of(blockStorageRoot); + case NO_OP -> NoOpBlockPathResolver.newInstance(); + }; } /** @@ -83,5 +155,5 @@ static BlockReader providesBlockReader(PersistenceStorageConfig c @Binds @Singleton BlockNodeEventHandler> bindBlockNodeEventHandler( - StreamPersistenceHandlerImpl streamPersistenceHandler); + @NonNull final StreamPersistenceHandlerImpl streamPersistenceHandler); } diff --git a/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java b/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java index 76d3e96ac..f70682d9f 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java +++ b/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java @@ -102,7 +102,6 @@ public StreamPersistenceHandlerImpl( public void onEvent(ObjectEvent event, long l, boolean b) { try { if (serviceStatus.isRunning()) { - final SubscribeStreamResponseUnparsed subscribeStreamResponse = event.get(); final OneOf oneOfTypeOneOf = subscribeStreamResponse.response(); @@ -137,9 +136,13 @@ public void onEvent(ObjectEvent event, long l, } else { LOGGER.log(ERROR, "Service is not running. Block item will not be processed further."); } - - } catch (BlockStreamProtocolException | IOException | ParseException e) { - + } catch (final BlockStreamProtocolException | IOException | ParseException e) { + // todo maybe we should catch here Exception so we do not miss anything and we can + // cleanup properly after every kind of exception + // also it would be a good idea to have a cleanup method to call instead of having this + // cleanup logic here, cause we could also use the cleanup method in another place if needed + // and in that way cleanup will not be error prone and will be managed only in one place + // also most of this seems to be duplicated in notifier.notifyUnrecoverableError() LOGGER.log(ERROR, "Failed to persist BlockItems: ", e); metricsService.get(StreamPersistenceHandlerError).increment(); diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java b/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java index 2ae5711a9..728568d2f 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java @@ -16,56 +16,157 @@ package com.hedera.block.server.persistence.storage; -import static com.hedera.block.server.Constants.BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME; -import static java.lang.System.Logger.Level.ERROR; +import static com.hedera.block.server.Constants.BLOCK_NODE_ARCHIVE_ROOT_DIRECTORY_SEMANTIC_NAME; +import static com.hedera.block.server.Constants.BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME; import static java.lang.System.Logger.Level.INFO; -import com.hedera.block.common.utils.FileUtilities; +import com.hedera.block.common.utils.StringUtilities; import com.swirlds.config.api.ConfigData; import com.swirlds.config.api.ConfigProperty; +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; +import java.util.Objects; /** - * Use this configuration across the persistent storage package + * Use this configuration across the persistence storage package. * - * @param rootPath provides the root path for saving block data, if you want to override it need to - * set it as persistence.storage.rootPath + * @param liveRootPath provides the root path for saving blocks live + * @param archiveRootPath provides the root path for archived blocks * @param type use a predefined type string to replace the persistence component implementation. - * Non-PRODUCTION values should only be used for troubleshooting and development purposes. + * Non-PRODUCTION values should only be used for troubleshooting and development purposes. */ @ConfigData("persistence.storage") public record PersistenceStorageConfig( - @ConfigProperty(defaultValue = "") String rootPath, @ConfigProperty(defaultValue = "PRODUCTION") String type) { + // @todo(#371) - the default life/archive root path must be absolute starting from /opt + @ConfigProperty(defaultValue = "") String liveRootPath, + // @todo(#371) - the default life/archive root path must be absolute starting from /opt + @ConfigProperty(defaultValue = "") String archiveRootPath, + @ConfigProperty(defaultValue = "BLOCK_AS_LOCAL_FILE") StorageType type) { private static final System.Logger LOGGER = System.getLogger(PersistenceStorageConfig.class.getName()); + // @todo(#371) - the default life/archive root path must be absolute starting from /opt + private static final String LIVE_ROOT_PATH = + Path.of("hashgraph/blocknode/data/live/").toAbsolutePath().toString(); + // @todo(#371) - the default life/archive root path must be absolute starting from /opt + private static final String ARCHIVE_ROOT_PATH = + Path.of("hashgraph/blocknode/data/archive/").toAbsolutePath().toString(); /** * Constructor to set the default root path if not provided, it will be set to the data * directory in the current working directory */ public PersistenceStorageConfig { - // verify rootPath prop - Path path = Path.of(rootPath); - if (rootPath.isEmpty()) { - path = Paths.get(rootPath).toAbsolutePath().resolve("data"); + Objects.requireNonNull(type); + liveRootPath = resolvePath(liveRootPath, LIVE_ROOT_PATH, BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME); + archiveRootPath = + resolvePath(archiveRootPath, ARCHIVE_ROOT_PATH, BLOCK_NODE_ARCHIVE_ROOT_DIRECTORY_SEMANTIC_NAME); + LOGGER.log(INFO, "Persistence Storage Configuration: persistence.storage.type=" + type); + LOGGER.log(INFO, "Persistence Storage Configuration: persistence.storage.liveRootPath=" + liveRootPath); + LOGGER.log(INFO, "Persistence Storage Configuration: persistence.storage.archiveRootPath=" + archiveRootPath); + } + + /** + * This method attempts to resolve a given configured path. If the input + * path is blank, a default path is used. The resolved path must be + * absolute! If the path resolution is successful, at attempt is made to + * create the directory path. If the directory path cannot be created, an + * {@link UncheckedIOException} is thrown. + * + * @param pathToResolve the path to resolve + * @param defaultIfBlank the default path if the path to resolve is blank + * @param semanticPathName the semantic name of the path used for logging + * @return the resolved path + * @throws IllegalArgumentException if the resolved path is not absolute + * @throws UncheckedIOException if the resolved path cannot be created + */ + @NonNull + private String resolvePath( + final String pathToResolve, @NonNull final String defaultIfBlank, @NonNull final String semanticPathName) { + final Path normalized = getNormalizedPath(pathToResolve, defaultIfBlank, semanticPathName); + createDirectoryPath(normalized, semanticPathName); + return normalized.toString(); + } + + /** + * This method normalizes a given path. If the path to normalize is blank, + * a default path is used. The normalized path must be absolute! If the + * normalized path is not absolute, an {@link IllegalArgumentException} is + * thrown. + * + * @param pathToNormalize the path to normalize + * @param defaultIfBlank the default path if the path to normalize is blank + * @param semanticPathName the semantic name of the path used for logging + * @return the normalized path + * @throws IllegalArgumentException if the path to normalize is not absolute + */ + @NonNull + private Path getNormalizedPath( + final String pathToNormalize, + @NonNull final String defaultIfBlank, + @NonNull final String semanticPathName) { + final Path result; + if (StringUtilities.isBlank(pathToNormalize)) { + result = Path.of(defaultIfBlank).toAbsolutePath(); + } else { + result = Path.of(pathToNormalize); } - // Check if absolute - if (!path.isAbsolute()) { - throw new IllegalArgumentException(rootPath + " Root path must be absolute"); + + if (!result.isAbsolute()) { + throw new IllegalArgumentException("Path provided for [%s] must be absolute!".formatted(semanticPathName)); + } else { + return result; } - // Create Directory if it does not exist + } + + /** + * This method creates a directory path at the given target path. If the + * directory path cannot be created, an {@link UncheckedIOException} is + * thrown. + * + * @param targetPath the target path to create the directory path + * @param semanticPathName the semantic name of the path used for logging + * @throws UncheckedIOException if the directory path cannot be created + */ + private void createDirectoryPath(@NonNull final Path targetPath, @NonNull final String semanticPathName) { try { - FileUtilities.createFolderPathIfNotExists(path, ERROR, BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME); + Files.createDirectories(targetPath); } catch (final IOException e) { - final String message = - "Unable to instantiate [%s]! Unable to create the root directory for the block storage [%s]" - .formatted(this.getClass().getName(), path); - throw new RuntimeException(message, e); + final String classname = this.getClass().getName(); + final String message = "Unable to instantiate [%s]! Unable to create the [%s] path that was provided!" + .formatted(classname, semanticPathName); + throw new UncheckedIOException(message, e); } + } - LOGGER.log(INFO, "Persistence Storage configuration persistence.storage.rootPath: " + path); - rootPath = path.toString(); - LOGGER.log(INFO, "Persistence configuration persistence.storage.type: " + type); + /** + * An enum that reflects the type of Block Storage Persistence that is + * currently used within the given server instance. During runtime one + * should only query for the storage type that was configured by calling + * {@link PersistenceStorageConfig#type()} on an instance of the persistence + * storage config that was only constructed via + * {@link com.swirlds.config.api.Configuration#getConfigData(Class)}! + */ + public enum StorageType { + /** + * This type of storage stores Blocks as individual files with the Block + * number as a unique file name and persisted in a trie structure with + * digit-per-folder + * (see #125). + * This is also the default setting for the server if it is not + * explicitly specified via an environment variable or app.properties. + */ + BLOCK_AS_LOCAL_FILE, + /** + * This type of storage stores Blocks as directories with the Block + * number being the directory number. Block Items are stored as files + * within a given Block directory. Used primarily for testing purposes. + */ + BLOCK_AS_LOCAL_DIRECTORY, + /** + * This type of storage does nothing. + */ + NO_OP } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolver.java b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolver.java new file mode 100644 index 000000000..10f87522e --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolver.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import com.hedera.block.common.utils.Preconditions; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.file.Path; +import java.util.Objects; + +/** + * A Block path resolver for block-as-dir. + */ +public final class BlockAsLocalDirPathResolver implements BlockPathResolver { + private final Path liveRootPath; + + /** + * Constructor. + * + * @param liveRootPath valid, {@code non-null} instance of {@link Path} + * that points to the live root of the block storage + */ + private BlockAsLocalDirPathResolver(@NonNull final Path liveRootPath) { + this.liveRootPath = Objects.requireNonNull(liveRootPath); + } + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalDirPathResolver}. + * + * @param liveRootPath valid, {@code non-null} instance of {@link Path} + * that points to the live root of the block storage + * @return a new, fully initialized instance of + * {@link BlockAsLocalDirPathResolver} + */ + public static BlockAsLocalDirPathResolver of(@NonNull final Path liveRootPath) { + return new BlockAsLocalDirPathResolver(liveRootPath); + } + + /** + * {@inheritDoc} + */ + @NonNull + @Override + public Path resolvePathToBlock(final long blockNumber) { + Preconditions.requireWhole(blockNumber); + return liveRootPath.resolve(String.valueOf(blockNumber)); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolver.java b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolver.java new file mode 100644 index 000000000..6b2235cbc --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolver.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import com.hedera.block.common.utils.Preconditions; +import com.hedera.block.server.Constants; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.file.Path; +import java.util.Objects; + +/** + * A Block path resolver for block-as-file. + */ +public final class BlockAsLocalFilePathResolver implements BlockPathResolver { + private static final int MAX_LONG_DIGITS = 19; + private final Path liveRootPath; + + /** + * Constructor. + * + * @param liveRootPath valid, {@code non-null} instance of {@link Path} + * that points to the live root of the block storage + */ + private BlockAsLocalFilePathResolver(@NonNull final Path liveRootPath) { + this.liveRootPath = Objects.requireNonNull(liveRootPath); + } + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalFilePathResolver}. + * + * @param liveRootPath valid, {@code non-null} instance of {@link Path} + * that points to the live root of the block storage + * @return a new, fully initialized instance of + * {@link BlockAsLocalFilePathResolver} + */ + public static BlockAsLocalFilePathResolver of(@NonNull final Path liveRootPath) { + return new BlockAsLocalFilePathResolver(liveRootPath); + } + + /** + * {@inheritDoc} + */ + @NonNull + @Override + public Path resolvePathToBlock(final long blockNumber) { + Preconditions.requireWhole(blockNumber); + final String rawBlockNumber = String.format("%0" + MAX_LONG_DIGITS + "d", blockNumber); + final String[] blockPath = rawBlockNumber.split(""); + final String blockFileName = rawBlockNumber.concat(Constants.BLOCK_FILE_EXTENSION); + blockPath[blockPath.length - 1] = blockFileName; + return Path.of(liveRootPath.toString(), blockPath); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockPathResolver.java b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockPathResolver.java new file mode 100644 index 000000000..d2de3654e --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/path/BlockPathResolver.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import java.nio.file.Path; + +/** + * A Block path resolver. Used to resolve path to a given Block and all the + * supporting related operations. + */ +public interface BlockPathResolver { + /** + * This method resolves the fs {@link Path} to a Block by a given input + * number. This method does not guarantee that the returned {@link Path} + * exists! + * + * @param blockNumber to be resolved the path for + * @return the resolved path to the given Block by a number + * @throws IllegalArgumentException if the blockNumber IS NOT a whole number + */ + Path resolvePathToBlock(final long blockNumber); +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolver.java b/server/src/main/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolver.java new file mode 100644 index 000000000..718b82077 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolver.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import java.nio.file.Path; + +/** + * A no-op path resolver. + */ +public final class NoOpBlockPathResolver implements BlockPathResolver { + /** + * Constructor. + */ + private NoOpBlockPathResolver() {} + + /** + * This method creates and returns a new instance of + * {@link NoOpBlockPathResolver}. + * + * @return a new, fully initialized instance of + * {@link NoOpBlockPathResolver} + */ + public static NoOpBlockPathResolver newInstance() { + return new NoOpBlockPathResolver(); + } + + /** + * No-op resolver. Does nothing and always returns null. No preconditions + * check also. + */ + @Override + public Path resolvePathToBlock(final long blockNumber) { + return null; + // todo should we return some path here so it is not null? + // reason would be to not have null pointer somewhere in our code + // if for whatever reason this no op impl needs to be used? + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java deleted file mode 100644 index 8cb90cbee..000000000 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (C) 2024 Hedera Hashgraph, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.hedera.block.server.persistence.storage.read; - -import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.hapi.block.BlockUnparsed; -import edu.umd.cs.findbugs.annotations.NonNull; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.util.Objects; -import java.util.Set; - -/** - * Use builder methods to create a {@link BlockReader} to read blocks from storage. - * - *

When a block reader is created, it will provide access to read blocks from storage. - */ -public final class BlockAsDirReaderBuilder { - private final PersistenceStorageConfig config; - private FileAttribute> folderPermissions; - - private BlockAsDirReaderBuilder(@NonNull final PersistenceStorageConfig config) { - this.config = Objects.requireNonNull(config); - } - - /** - * Creates a new block reader builder using the minimum required parameters. - * - * @param config is required to supply pertinent configuration info for the block reader to - * access storage. - * @return a block reader builder configured with required parameters. - */ - @NonNull - public static BlockAsDirReaderBuilder newBuilder(@NonNull final PersistenceStorageConfig config) { - return new BlockAsDirReaderBuilder(config); - } - - /** - * Optionally, provide file permissions for the block reader to use when managing block files - * and directories. - * - * @param folderPermissions the folder permissions to use when managing block files as directories. - * @return a block reader builder configured with required parameters. - */ - @NonNull - public BlockAsDirReaderBuilder folderPermissions( - @NonNull final FileAttribute> folderPermissions) { - this.folderPermissions = Objects.requireNonNull(folderPermissions); - return this; - } - - /** - * Use the build method to construct a block reader to read blocks from storage. - * - * @return a new block reader configured with the parameters provided to the builder. - */ - @NonNull - public BlockReader build() { - return new BlockAsDirReader(config, folderPermissions); - } -} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReader.java similarity index 81% rename from server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java rename to server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReader.java index 3a082c4b0..dc5b19645 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReader.java @@ -22,6 +22,7 @@ import static java.lang.System.Logger.Level.ERROR; import static java.lang.System.Logger.Level.INFO; +import com.hedera.block.common.utils.Preconditions; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.BlockUnparsed; @@ -39,7 +40,6 @@ import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -47,35 +47,10 @@ * The BlockAsDirReader class reads a block from the file system. The block is stored as a directory * containing block items. The block items are stored as files within the block directory. */ -class BlockAsDirReader implements BlockReader { - private final Logger LOGGER = System.getLogger(getClass().getName()); - private final Path blockNodeRootPath; - private final FileAttribute> folderPermissions; - - /** - * Constructor for the BlockAsDirReader class. It initializes the BlockAsDirReader with the - * given parameters. - * - * @param config the configuration to retrieve the block node root path - * @param folderPermissions the folder permissions to set on the block node root path, default will be used if null provided - */ - BlockAsDirReader( - @NonNull final PersistenceStorageConfig config, - final FileAttribute> folderPermissions) { - LOGGER.log(INFO, "Initializing FileSystemBlockReader"); - - final Path blockNodeRootPath = Path.of(config.rootPath()); - - LOGGER.log(INFO, config.toString()); - LOGGER.log(INFO, "Block Node Root Path: " + blockNodeRootPath); - - this.blockNodeRootPath = blockNodeRootPath; - - if (Objects.nonNull(folderPermissions)) { - this.folderPermissions = folderPermissions; - } else { - // default permissions for folders - this.folderPermissions = PosixFilePermissions.asFileAttribute(Set.of( +public class BlockAsLocalDirReader implements LocalBlockReader { + private static final Logger LOGGER = System.getLogger(BlockAsLocalDirReader.class.getName()); + private static final FileAttribute> DEFAULT_FOLDER_PERMISSIONS = + PosixFilePermissions.asFileAttribute(Set.of( PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, @@ -83,7 +58,30 @@ class BlockAsDirReader implements BlockReader { PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_EXECUTE)); - } + private final Path liveRootPath; + + /** + * Constructor. + * + * @param config the configuration to retrieve the root paths + */ + protected BlockAsLocalDirReader(@NonNull final PersistenceStorageConfig config) { + LOGGER.log(INFO, "Initializing %s...".formatted(getClass().getName())); + this.liveRootPath = Path.of(config.liveRootPath()); + } + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalDirReader}. + * + * @param config valid, {@code non-null} instance of + * {@link PersistenceStorageConfig} used to retrieve the block node root + * path + * @return a new, fully initialized instance of + * {@link BlockAsLocalDirReader} + */ + public static BlockAsLocalDirReader of(@NonNull final PersistenceStorageConfig config) { + return new BlockAsLocalDirReader(config); } /** @@ -97,15 +95,15 @@ class BlockAsDirReader implements BlockReader { @NonNull @Override public Optional read(final long blockNumber) throws IOException, ParseException { - - // Verify path attributes of the block node root path - if (isPathDisqualified(blockNodeRootPath)) { + Preconditions.requireWhole(blockNumber); + // Verify path attributes of the block node live root path + if (isPathDisqualified(liveRootPath)) { return Optional.empty(); } // Verify path attributes of the block directory within the - // block node root path - final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); + // block node live root path + final Path blockPath = liveRootPath.resolve(String.valueOf(blockNumber)); if (isPathDisqualified(blockPath)) { return Optional.empty(); } @@ -130,7 +128,6 @@ public Optional read(final long blockNumber) throws IOException, blockItems.add(blockItemOpt.get()); continue; } - break; } @@ -138,7 +135,7 @@ public Optional read(final long blockNumber) throws IOException, // Return the Block return Optional.of(builder.build()); - } catch (IOException io) { + } catch (final IOException io) { LOGGER.log(ERROR, "Error reading block: " + blockPath, io); throw io; } @@ -187,7 +184,7 @@ private boolean isPathDisqualified(@NonNull final Path path) { try { // If resetting the permissions fails or // if the path is still unreadable, return true. - setPerm(path, folderPermissions.value()); + setPerm(path, DEFAULT_FOLDER_PERMISSIONS.value()); if (!path.toFile().canRead()) { return true; } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReader.java new file mode 100644 index 000000000..52d20161c --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReader.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.read; + +import com.hedera.block.common.utils.Preconditions; +import com.hedera.hapi.block.BlockUnparsed; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.Optional; + +/** + * A Block reader that reads block-as-file. + */ +public final class BlockAsLocalFileReader implements LocalBlockReader { + /** + * Constructor. + */ + private BlockAsLocalFileReader() {} + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalFileReader}. + * + * @return a new, fully initialized instance of + * {@link BlockAsLocalFileReader} + */ + public static BlockAsLocalFileReader newInstance() { + return new BlockAsLocalFileReader(); + } + + /** + * {@inheritDoc} + */ + @NonNull + @Override + public Optional read(final long blockNumber) { + Preconditions.requireWhole(blockNumber); + throw new UnsupportedOperationException("Not implemented yet"); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockReader.java index 1a24bccdb..2c7a42bc8 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockReader.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockReader.java @@ -24,10 +24,9 @@ /** * The BlockReader interface defines the contract for reading a block from storage. * - * @param the type of the block to read + * @param the type to be returned after reading the block */ -public interface BlockReader { - +public interface BlockReader { /** * Reads the block with the given block number. * @@ -37,7 +36,8 @@ public interface BlockReader { * @throws ParseException if the PBJ codec encounters a problem caused by I/O issues, malformed * input data, or any other reason that prevents the parse() method from completing the * operation when fetching the block. + * @throws IllegalArgumentException if the blockNumber IS NOT a whole number */ @NonNull - Optional read(final long blockNumber) throws IOException, ParseException; + Optional read(final long blockNumber) throws IOException, ParseException; } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/LocalBlockReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/LocalBlockReader.java new file mode 100644 index 000000000..095d26b9f --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/LocalBlockReader.java @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.read; + +/** + * A marker interface that groups all writers that operate on a local file + * system. + * + * @param the type that will be returned after reading the block + */ +public interface LocalBlockReader extends BlockReader {} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReader.java new file mode 100644 index 000000000..03bd7a341 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReader.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.read; + +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.pbj.runtime.ParseException; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.IOException; +import java.util.Optional; + +/** + * A no-op Block reader. + */ +public final class NoOpBlockReader implements BlockReader { + /** + * Constructor. + */ + private NoOpBlockReader() {} + + /** + * This method creates and returns a new instance of + * {@link NoOpBlockReader}. + * + * @return a new, fully initialized instance of + * {@link NoOpBlockReader} + */ + public static NoOpBlockReader newInstance() { + return new NoOpBlockReader(); + } + + /** + * {@inheritDoc} + */ + @NonNull + @Override + public Optional read(final long blockNumber) throws IOException, ParseException { + return Optional.empty(); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemover.java b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemover.java similarity index 51% rename from server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemover.java rename to server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemover.java index 5fbc4a161..85aa5cbce 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemover.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemover.java @@ -19,6 +19,8 @@ import static java.lang.System.Logger; import static java.lang.System.Logger.Level.ERROR; +import com.hedera.block.common.utils.Preconditions; +import com.hedera.block.server.persistence.storage.path.BlockPathResolver; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.File; import java.io.IOException; @@ -30,37 +32,51 @@ * The BlockAsDirRemover class removes a block from the file system. The block is stored as a * directory containing block items. The block items are stored as files within the block directory. */ -public class BlockAsDirRemover implements BlockRemover { +public final class BlockAsLocalDirRemover implements LocalBlockRemover { private final Logger LOGGER = System.getLogger(getClass().getName()); - private final Path blockNodeRootPath; + private final BlockPathResolver blockPathResolver; /** - * Create a block remover to manage removing blocks from storage. + * Constructor. * - * @param blockNodeRootPath the root path where blocks are stored. + * @param blockPathResolver valid, {@code non-null} instance of + * {@link BlockPathResolver} to be used internally to resolve paths to Block */ - public BlockAsDirRemover(@NonNull final Path blockNodeRootPath) { - this.blockNodeRootPath = Objects.requireNonNull(blockNodeRootPath); + private BlockAsLocalDirRemover(@NonNull final BlockPathResolver blockPathResolver) { + this.blockPathResolver = Objects.requireNonNull(blockPathResolver); + } + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalDirRemover}. + * + * @param blockPathResolver valid, {@code non-null} instance of + * {@link BlockPathResolver} to be used internally to resolve paths to Block + * @return a new, fully initialized instance of + * {@link BlockAsLocalDirRemover} + */ + public static BlockAsLocalDirRemover of(@NonNull final BlockPathResolver blockPathResolver) { + return new BlockAsLocalDirRemover(blockPathResolver); } /** * Removes a block from the file system. * - * @param id the id of the block to remove + * @param blockNumber the id of the block to remove * @throws IOException if an I/O error occurs + * @throws IllegalArgumentException if the blockNumber IS NOT a whole number */ @Override - public void remove(final long id) throws IOException { - // Calculate the block path and proactively set the permissions - // for removal - final Path blockPath = blockNodeRootPath.resolve(String.valueOf(id)); + public void remove(final long blockNumber) throws IOException { + Preconditions.requireWhole(blockNumber); + final Path blockPath = blockPathResolver.resolvePathToBlock(blockNumber); if (Files.notExists(blockPath)) { - LOGGER.log(ERROR, "Block does not exist: {0}", id); - return; - } - // Best effort to delete the block - if (!delete(blockPath.toFile())) { - LOGGER.log(ERROR, "Failed to delete block: {0}", id); + LOGGER.log(ERROR, "Block cannot be deleted as it does not exist: {0}", blockNumber); + } else { + final boolean deleted = delete(blockPath.toFile()); + if (!deleted) { + LOGGER.log(ERROR, "Failed to delete block: {0}", blockNumber); + } } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemover.java b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemover.java new file mode 100644 index 000000000..c6065489f --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemover.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +import com.hedera.block.common.utils.Preconditions; +import java.io.IOException; + +/** + * A Block remover that handles block-as-local-file. + */ +public final class BlockAsLocalFileRemover implements LocalBlockRemover { + /** + * Constructor. + */ + private BlockAsLocalFileRemover() {} + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalFileRemover}. + * + * @return a new fully initialized instance of + * {@link BlockAsLocalFileRemover} + */ + public static BlockAsLocalFileRemover newInstance() { + return new BlockAsLocalFileRemover(); + } + + /** + * Remove a block with the given block number. + * + * @param blockNumber the block number of the block to remove + * @throws IOException when failing to remove a block + * @throws IllegalArgumentException if the blockNumber IS NOT a whole number + */ + @Override + public void remove(final long blockNumber) throws IOException { + Preconditions.requireWhole(blockNumber); + throw new UnsupportedOperationException("Not implemented yet"); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockRemover.java b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockRemover.java index bee39bed8..9ece1c3fa 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockRemover.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/BlockRemover.java @@ -20,12 +20,12 @@ /** The BlockRemover interface defines the contract for removing a block from storage. */ public interface BlockRemover { - /** * Remove a block with the given block number. * - * @param blockNumber the block number of the block to remove. - * @throws IOException when failing to remove a block. + * @param blockNumber the block number of the block to remove + * @throws IOException when failing to remove a block + * @throws IllegalArgumentException if the blockNumber IS NOT a whole number */ void remove(final long blockNumber) throws IOException; } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/LocalBlockRemover.java b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/LocalBlockRemover.java new file mode 100644 index 000000000..f996f1d81 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/LocalBlockRemover.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +/** + * A marker interface that groups all removers that operate on a local file + * system. + */ +public interface LocalBlockRemover extends BlockRemover {} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemover.java b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemover.java new file mode 100644 index 000000000..45313b651 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemover.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +/** + * A no-op Block remover. + */ +public final class NoOpBlockRemover implements BlockRemover { + /** + * Constructor. + */ + private NoOpBlockRemover() {} + + /** + * This method creates and returns a new instance of {@link NoOpBlockRemover}. + * + * @return a new, fully initialized instance of {@link NoOpBlockRemover} + */ + public static NoOpBlockRemover newInstance() { + return new NoOpBlockRemover(); + } + + /** + * No-op remover. Does nothing and returns immediately. No preconditions + * check also. + */ + @Override + public void remove(final long blockNumber) { + // do nothing + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java deleted file mode 100644 index b6e826807..000000000 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (C) 2024 Hedera Hashgraph, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.hedera.block.server.persistence.storage.write; - -import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.remove.BlockAsDirRemover; -import com.hedera.block.server.persistence.storage.remove.BlockRemover; -import com.hedera.hapi.block.BlockItemUnparsed; -import edu.umd.cs.findbugs.annotations.NonNull; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.util.List; -import java.util.Objects; -import java.util.Set; - -/** - * Use builder methods to create a {@link BlockWriter} to write blocks to storage. - * - *

When a block writer is created, it will provide access to write blocks to storage. - */ -public final class BlockAsDirWriterBuilder { - private final BlockNodeContext blockNodeContext; - private FileAttribute> folderPermissions; - private BlockRemover blockRemover; - - private BlockAsDirWriterBuilder(@NonNull final BlockNodeContext blockNodeContext) { - this.blockNodeContext = Objects.requireNonNull(blockNodeContext); - final PersistenceStorageConfig config = - blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); - this.blockRemover = new BlockAsDirRemover(Path.of(config.rootPath())); - } - - /** - * Creates a new block writer builder using the minimum required parameters. - * - * @param blockNodeContext is required to provide metrics reporting mechanisms . - * @return a block writer builder configured with required parameters. - */ - @NonNull - public static BlockAsDirWriterBuilder newBuilder(@NonNull final BlockNodeContext blockNodeContext) { - return new BlockAsDirWriterBuilder(blockNodeContext); - } - - /** - * Optionally, provide file permissions for the block writer to use when managing block files - * and directories. - * - * @param folderPermissions the folder permissions to use when managing block files as directories. - * @return a block writer builder configured with required parameters. - */ - @NonNull - public BlockAsDirWriterBuilder folderPermissions( - @NonNull final FileAttribute> folderPermissions) { - this.folderPermissions = Objects.requireNonNull(folderPermissions); - return this; - } - - /** - * Optionally, provide a block remover to remove blocks from storage. - * - *

By default, the block writer will use the block remover defined in {@link - * BlockAsDirRemover}. This method is primarily used for testing purposes. Default values should - * be sufficient for production use. - * - * @param blockRemover the block remover to use when removing blocks from storage. - * @return a block writer builder configured with required parameters. - */ - @NonNull - public BlockAsDirWriterBuilder blockRemover(@NonNull final BlockRemover blockRemover) { - this.blockRemover = Objects.requireNonNull(blockRemover); - return this; - } - - /** - * Use the build method to construct a block writer to write blocks to storage. - * - * @return a new block writer configured with the parameters provided to the builder. - * @throws IOException when an error occurs while persisting block items to storage. - */ - @NonNull - public BlockWriter> build() throws IOException { - return new BlockAsDirWriter(blockRemover, folderPermissions, blockNodeContext); - } -} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriter.java similarity index 65% rename from server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java rename to server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriter.java index 461c74911..99a98e7ca 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriter.java @@ -17,7 +17,7 @@ package com.hedera.block.server.persistence.storage.write; import static com.hedera.block.server.Constants.BLOCK_FILE_EXTENSION; -import static com.hedera.block.server.Constants.BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME; +import static com.hedera.block.server.Constants.BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.BlocksPersisted; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; @@ -28,6 +28,7 @@ import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.persistence.storage.path.BlockPathResolver; import com.hedera.block.server.persistence.storage.remove.BlockRemover; import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.stream.output.BlockHeader; @@ -55,106 +56,121 @@ * to remove the current, incomplete block (directory) before re-throwing the exception to the * caller. */ -class BlockAsDirWriter implements BlockWriter> { - private final Logger LOGGER = System.getLogger(getClass().getName()); - private final Path blockNodeRootPath; - private final FileAttribute> folderPermissions; - private final BlockRemover blockRemover; +public class BlockAsLocalDirWriter implements LocalBlockWriter> { + private static final Logger LOGGER = System.getLogger(BlockAsLocalDirWriter.class.getName()); + private static final FileAttribute> DEFAULT_FOLDER_PERMISSIONS = + PosixFilePermissions.asFileAttribute(Set.of( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE, + PosixFilePermission.GROUP_READ, + PosixFilePermission.GROUP_EXECUTE, + PosixFilePermission.OTHERS_READ, + PosixFilePermission.OTHERS_EXECUTE)); + private final Path liveRootPath; private final MetricsService metricsService; + private final BlockRemover blockRemover; + private final BlockPathResolver blockPathResolver; private long blockNodeFileNameIndex; - private Path currentBlockDir; + private long currentBlockNumber; /** * Use the corresponding builder to construct a new BlockAsDirWriter with the given parameters. * - * @param blockRemover the block remover to use for removing blocks - * @param folderPermissions the folder permissions to use for writing blocks, if null provided then defaults will be used * @param blockNodeContext the block node context to use for writing blocks + * @param blockRemover the block remover to use for removing blocks + * @param blockPathResolver used internally to resolve paths + * * @throws IOException if an error occurs while initializing the BlockAsDirWriter */ - BlockAsDirWriter( + protected BlockAsLocalDirWriter( + @NonNull final BlockNodeContext blockNodeContext, @NonNull final BlockRemover blockRemover, - final FileAttribute> folderPermissions, - @NonNull final BlockNodeContext blockNodeContext) + @NonNull final BlockPathResolver blockPathResolver) throws IOException { - LOGGER.log(INFO, "Initializing FileSystemBlockStorage"); + LOGGER.log(INFO, "Initializing %s...".formatted(getClass().getName())); + + this.metricsService = Objects.requireNonNull(blockNodeContext.metricsService()); + this.blockRemover = Objects.requireNonNull(blockRemover); + this.blockPathResolver = Objects.requireNonNull(blockPathResolver); final PersistenceStorageConfig config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); - - final Path blockNodeRootPath = Path.of(config.rootPath()); - LOGGER.log(INFO, "Block Node Root Path: " + blockNodeRootPath); - - this.blockNodeRootPath = blockNodeRootPath; - this.blockRemover = blockRemover; - this.metricsService = blockNodeContext.metricsService(); - - if (Objects.nonNull(folderPermissions)) { - this.folderPermissions = folderPermissions; - } else { - // default permissions for folders - this.folderPermissions = PosixFilePermissions.asFileAttribute(Set.of( - PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, - PosixFilePermission.OWNER_EXECUTE, - PosixFilePermission.GROUP_READ, - PosixFilePermission.GROUP_EXECUTE, - PosixFilePermission.OTHERS_READ, - PosixFilePermission.OTHERS_EXECUTE)); - } + this.liveRootPath = Path.of(config.liveRootPath()); // Initialize the block node root directory if it does not exist FileUtilities.createFolderPathIfNotExists( - blockNodeRootPath, INFO, this.folderPermissions, BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME); + liveRootPath, INFO, DEFAULT_FOLDER_PERMISSIONS, BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME); + } + + /** + * This method creates and returns a new instance of + * {@link BlockAsLocalDirWriter}. + * + * @param blockNodeContext valid, {@code non-null} instance of + * {@link BlockNodeContext} used to get the {@link MetricsService} + * @param blockRemover valid, {@code non-null} instance of + * {@link BlockRemover} used to remove blocks in case of cleanup + * @param blockPathResolver valid, {@code non-null} instance of + * {@link BlockPathResolver} used to resolve paths to Blocks + * @return a new, fully initialized instance of {@link BlockAsLocalDirWriter} + * @throws IOException if an error occurs while initializing the BlockAsDirWriter + */ + public static BlockAsLocalDirWriter of( + @NonNull final BlockNodeContext blockNodeContext, + @NonNull final BlockRemover blockRemover, + @NonNull final BlockPathResolver blockPathResolver) + throws IOException { + return new BlockAsLocalDirWriter(blockNodeContext, blockRemover, blockPathResolver); } /** * Writes the given block item to the filesystem. * - * @param blockItems the block item to write + * @param valueToWrite the block item to write * @throws IOException if an error occurs while writing the block item */ + @NonNull @Override - public Optional> write(@NonNull final List blockItems) + public Optional> write(@NonNull final List valueToWrite) throws IOException, ParseException { - - final Bytes unparsedBlockHeader = blockItems.getFirst().blockHeader(); + final Bytes unparsedBlockHeader = valueToWrite.getFirst().blockHeader(); if (unparsedBlockHeader != null) { resetState(BlockHeader.PROTOBUF.parse(unparsedBlockHeader)); } - for (BlockItemUnparsed blockItemUnparsed : blockItems) { + for (final BlockItemUnparsed blockItemUnparsed : valueToWrite) { final Path blockItemFilePath = calculateBlockItemPath(); for (int retries = 0; ; retries++) { try { write(blockItemFilePath, blockItemUnparsed); break; - } catch (IOException e) { + } catch (final IOException e) { LOGGER.log(ERROR, "Error writing the BlockItem protobuf to a file: ", e); // Remove the block if repairing the permissions fails if (retries > 0) { // Attempt to remove the block - blockRemover.remove(Long.parseLong(currentBlockDir.toString())); + blockRemover.remove(currentBlockNumber); throw e; } else { // Attempt to repair the permissions on the block path // and the blockItem path - repairPermissions(blockNodeRootPath); - repairPermissions(calculateBlockPath()); + repairPermissions(liveRootPath); + repairPermissions(blockPathResolver.resolvePathToBlock(currentBlockNumber)); LOGGER.log(INFO, "Retrying to write the BlockItem protobuf to a file"); } } } } - if (blockItems.getLast().hasBlockProof()) { + if (valueToWrite.getLast().hasBlockProof()) { metricsService.get(BlocksPersisted).increment(); - return Optional.of(blockItems); + return Optional.of(valueToWrite); + } else { + return Optional.empty(); } - - return Optional.empty(); } /** @@ -170,7 +186,8 @@ protected void write(@NonNull final Path blockItemFilePath, @NonNull final Block // Write the Bytes directly BlockItemUnparsed.PROTOBUF.toBytes(blockItem).writeTo(fos); LOGGER.log(DEBUG, "Successfully wrote the block item file: {0}", blockItemFilePath); - } catch (IOException e) { + } catch (final IOException e) { + // fixme writeTo throws UncheckedIOException, we should handle other cases as well LOGGER.log(ERROR, "Error writing the BlockItem protobuf to a file: ", e); throw e; } @@ -179,40 +196,39 @@ protected void write(@NonNull final Path blockItemFilePath, @NonNull final Block private void resetState(@NonNull final BlockHeader blockHeader) throws IOException { // Here a "block" is represented as a directory of BlockItems. // Create the "block" directory based on the block_number - currentBlockDir = Path.of(String.valueOf(blockHeader.number())); + currentBlockNumber = blockHeader.number(); // Check the blockNodeRootPath permissions and // attempt to repair them if possible - repairPermissions(blockNodeRootPath); + repairPermissions(liveRootPath); // Construct the path to the block directory FileUtilities.createFolderPathIfNotExists( - calculateBlockPath(), DEBUG, folderPermissions, BLOCK_NODE_ROOT_DIRECTORY_SEMANTIC_NAME); + blockPathResolver.resolvePathToBlock(currentBlockNumber), + DEBUG, + DEFAULT_FOLDER_PERMISSIONS, + BLOCK_NODE_LIVE_ROOT_DIRECTORY_SEMANTIC_NAME); // Reset blockNodeFileNameIndex = 0; } + // todo do we need this method at all? private void repairPermissions(@NonNull final Path path) throws IOException { final boolean isWritable = Files.isWritable(path); if (!isWritable) { LOGGER.log(ERROR, "Block node root directory is not writable. Attempting to change the" + " permissions."); // Attempt to restore the permissions on the block node root directory - Files.setPosixFilePermissions(path, folderPermissions.value()); + Files.setPosixFilePermissions(path, DEFAULT_FOLDER_PERMISSIONS.value()); } } @NonNull private Path calculateBlockItemPath() { // Build the path to a .blk file - final Path blockPath = calculateBlockPath(); + final Path blockPath = blockPathResolver.resolvePathToBlock(currentBlockNumber); blockNodeFileNameIndex++; return blockPath.resolve(blockNodeFileNameIndex + BLOCK_FILE_EXTENSION); } - - @NonNull - private Path calculateBlockPath() { - return blockNodeRootPath.resolve(currentBlockDir); - } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriter.java new file mode 100644 index 000000000..4a4f09822 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalFileWriter.java @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.write; + +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.BlocksPersisted; + +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.metrics.MetricsService; +import com.hedera.block.server.persistence.storage.path.BlockPathResolver; +import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.hapi.block.stream.output.BlockHeader; +import com.hedera.pbj.runtime.ParseException; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * A Block writer that handles writing of block-as-file. + */ +public final class BlockAsLocalFileWriter implements LocalBlockWriter> { + private final MetricsService metricsService; + private final BlockRemover blockRemover; // todo do I need here? + private final BlockPathResolver blockPathResolver; + private BlockUnparsed + currentBlockUnparsed; // fixme this is temporary just to explore the workflow and make proof of concept + + /** + * Constructor. + * + * @param blockNodeContext valid, {@code non-null} instance of {@link BlockNodeContext} used to + * get the {@link MetricsService} + * @param blockRemover valid, {@code non-null} instance of {@link BlockRemover} used to remove + * blocks in case of cleanup + * @param blockPathResolver valid, {@code non-null} instance of {@link BlockPathResolver} used + * to resolve paths to Blocks + */ + private BlockAsLocalFileWriter( + @NonNull final BlockNodeContext blockNodeContext, + @NonNull final BlockRemover blockRemover, + @NonNull final BlockPathResolver blockPathResolver) { + this.metricsService = Objects.requireNonNull(blockNodeContext.metricsService()); + this.blockRemover = Objects.requireNonNull(blockRemover); + this.blockPathResolver = Objects.requireNonNull(blockPathResolver); + } + + /** + * This method creates and returns a new instance of {@link BlockAsLocalFileWriter}. + * + * @param blockNodeContext valid, {@code non-null} instance of + * {@link BlockNodeContext} used to get the {@link MetricsService} + * @param blockRemover valid, {@code non-null} instance of + * {@link BlockRemover} used to remove blocks in case of cleanup + * @param blockPathResolver valid, {@code non-null} instance of + * {@link BlockPathResolver} used to resolve paths to Blocks + * @return a new, fully initialized instance of {@link BlockAsLocalFileWriter} + */ + public static BlockAsLocalFileWriter of( + @NonNull final BlockNodeContext blockNodeContext, + @NonNull final BlockRemover blockRemover, + @NonNull final BlockPathResolver blockPathResolver) { + return new BlockAsLocalFileWriter(blockNodeContext, blockRemover, blockPathResolver); + } + + /** + * {@inheritDoc} + */ + @NonNull + @Override + public Optional> write(@NonNull final List valueToWrite) + throws IOException, ParseException { + if (valueToWrite.getFirst().hasBlockHeader()) { + currentBlockUnparsed = + BlockUnparsed.newBuilder().blockItems(valueToWrite).build(); + } else { + final List currentItems = currentBlockUnparsed.blockItems(); + currentItems.addAll(valueToWrite); + currentBlockUnparsed = + BlockUnparsed.newBuilder().blockItems(currentItems).build(); + } + + if (valueToWrite.getLast().hasBlockProof()) { + metricsService.get(BlocksPersisted).increment(); + return Optional.of(writeToFs(currentBlockUnparsed)); + } else { + return Optional.empty(); + } + } + + // todo we could recursively retry if exception occurs, then after a few attempts + // if we cannot persist, we must throw the initial exception + private List writeToFs(final BlockUnparsed blockToWrite) throws IOException, ParseException { + final long number = BlockHeader.PROTOBUF + .parse(blockToWrite.blockItems().getFirst().blockHeader()) // fixme could be null, handle! + .number(); // todo should we check if whole? If so, what handle we should have? + + // todo we should handle cases where the block path exists, we do not expect it to + // exist at this stage, if it does, there is something wrong here + final Path blockToWritePathResolved = blockPathResolver.resolvePathToBlock(number); + Files.createDirectories(blockToWritePathResolved.getParent()); + Files.createFile(blockToWritePathResolved); + + // todo maybe this is not the place to handle the exceptions, but maybe if we do a retry mechanism we + // must catch here. also should we repair permissions and use the remover to remove as a cleanup? + // do we have such cases? + try (final FileOutputStream fos = new FileOutputStream(blockToWritePathResolved.toFile())) { + BlockUnparsed.PROTOBUF.toBytes(blockToWrite).writeTo(fos); + // todo what should be fallback logic if something goes wrong here? we attempt to resolve the path + // with proper perms (is that necessary)? we must clean up and retry? + } catch (final IOException ioe) { + // todo handle properly + throw new UncheckedIOException(ioe); + } catch (final UncheckedIOException uioe) { + // todo handle properly + throw uioe; + } catch (final Exception e) { + // todo handle properly + throw new RuntimeException(e); + } + currentBlockUnparsed = null; + return blockToWrite.blockItems(); + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java index eccbe4291..8fe4ec697 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java @@ -24,18 +24,19 @@ /** * BlockWriter defines the contract for writing block items to storage. * - * @param the type of the block item to write + * @param the type of the block item to write */ -public interface BlockWriter { +public interface BlockWriter { // todo maybe we can remove the generics? /** * Write the block item to storage. * - * @param blockItem the block item to write to storage. - * @return an optional containing the block item written to storage if the block item was a - * block proof signaling the end of the block, an empty optional otherwise. - * @throws IOException when failing to write the block item to storage. + * @param valueToWrite to storage. + * @return an optional containing the item written to storage if the item + * was a block proof signaling the end of the block, an empty optional otherwise. + * @throws IOException when failing to write the item to storage. * @throws ParseException when failing to parse a block item. */ - Optional write(@NonNull final V blockItem) throws IOException, ParseException; + @NonNull + Optional write(@NonNull final T valueToWrite) throws IOException, ParseException; } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/LocalBlockWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/LocalBlockWriter.java new file mode 100644 index 000000000..d6c6536df --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/LocalBlockWriter.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.write; + +/** + * A marker interface that groups all writers that operate on a local file + * system. + */ +interface LocalBlockWriter extends BlockWriter {} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java index a805c1fd6..2fe0c328b 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java @@ -16,9 +16,6 @@ package com.hedera.block.server.persistence.storage.write; -import static java.lang.System.Logger.Level.INFO; - -import com.hedera.block.server.config.BlockNodeContext; import com.hedera.hapi.block.BlockItemUnparsed; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; @@ -30,29 +27,36 @@ * designed to isolate the Producer and Mediator components from storage implementation during testing while still * providing metrics and logging for troubleshooting. */ -public class NoOpBlockWriter implements BlockWriter> { +public final class NoOpBlockWriter implements BlockWriter> { + /** + * Constructor. + */ + private NoOpBlockWriter() {} /** - * Creates a new NoOpBlockWriter instance for testing and troubleshooting only. + * This method creates and returns a new instance of + * {@link NoOpBlockWriter}. * - * @param blockNodeContext the block node context + * @return a new, fully initialized instance of {@link NoOpBlockWriter} */ - public NoOpBlockWriter(BlockNodeContext blockNodeContext) { - System.getLogger(getClass().getName()).log(INFO, "Using " + getClass().getSimpleName()); + public static NoOpBlockWriter newInstance() { + return new NoOpBlockWriter(); } /** * {@inheritDoc} */ + @NonNull @Override - public Optional> write(@NonNull List blockItems) throws IOException { - if (blockItems.getLast().hasBlockProof()) { + public Optional> write(@NonNull final List valueToWrite) + throws IOException { + if (valueToWrite.getLast().hasBlockProof()) { // Returning the BlockItems triggers a // PublishStreamResponse to be sent to the // upstream producer. - return Optional.of(blockItems); + return Optional.of(valueToWrite); + } else { + return Optional.empty(); } - - return Optional.empty(); } } diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 642467866..d00c922e4 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -6,6 +6,7 @@ exports com.hedera.block.server.consumer; exports com.hedera.block.server.exception; exports com.hedera.block.server.persistence.storage; + exports com.hedera.block.server.persistence.storage.path; exports com.hedera.block.server.persistence.storage.write; exports com.hedera.block.server.persistence.storage.read; exports com.hedera.block.server.persistence.storage.remove; diff --git a/server/src/main/resources/app.properties b/server/src/main/resources/app.properties index ce2fe915a..fcdc53254 100644 --- a/server/src/main/resources/app.properties +++ b/server/src/main/resources/app.properties @@ -8,3 +8,9 @@ prometheus.endpointPortNumber=9999 # Timeout for consumers to wait for block item before timing out. # Default is 1500 milliseconds. #consumer.timeoutThresholdMillis=1500 + +#Persistence Storage Config +#persistence.storage.liveRootPath= +#persistence.storage.archiveRootPath= +# @todo(#372) - default persistence type should be BLOCK_AS_LOCAL_FILE +persistence.storage.type=BLOCK_AS_LOCAL_DIRECTORY diff --git a/server/src/test/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializerTest.java b/server/src/test/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializerTest.java index 5c2f03dde..47b86af65 100644 --- a/server/src/test/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializerTest.java +++ b/server/src/test/java/com/hedera/block/server/config/ServerMappedConfigSourceInitializerTest.java @@ -30,7 +30,9 @@ class ServerMappedConfigSourceInitializerTest { private static final ConfigMapping[] SUPPORTED_MAPPINGS = { new ConfigMapping("consumer.timeoutThresholdMillis", "CONSUMER_TIMEOUT_THRESHOLD_MILLIS"), - new ConfigMapping("persistence.storage.rootPath", "PERSISTENCE_STORAGE_ROOT_PATH"), + new ConfigMapping("persistence.storage.liveRootPath", "PERSISTENCE_STORAGE_LIVE_ROOT_PATH"), + new ConfigMapping("persistence.storage.archiveRootPath", "PERSISTENCE_STORAGE_ARCHIVE_ROOT_PATH"), + new ConfigMapping("persistence.storage.type", "PERSISTENCE_STORAGE_TYPE"), new ConfigMapping("service.delayMillis", "SERVICE_DELAY_MILLIS"), new ConfigMapping("mediator.ringBufferSize", "MEDIATOR_RING_BUFFER_SIZE"), new ConfigMapping("notifier.ringBufferSize", "NOTIFIER_RING_BUFFER_SIZE"), diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java index 98094390f..b3d6246f9 100644 --- a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java @@ -18,9 +18,12 @@ import static com.hedera.block.server.Constants.FULL_SERVICE_NAME_BLOCK_ACCESS; import static com.hedera.block.server.Constants.SERVICE_NAME_BLOCK_ACCESS; +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -29,11 +32,14 @@ import com.hedera.block.server.pbj.PbjBlockAccessService; import com.hedera.block.server.pbj.PbjBlockAccessServiceProxy; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReader; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; +import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; +import com.hedera.block.server.util.PersistTestUtils; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.BlockUnparsed; @@ -57,6 +63,7 @@ @ExtendWith(MockitoExtension.class) class BlockAccessServiceTest { + private BlockAsLocalDirPathResolver pathResolverMock; @Mock private Pipeline responseObserver; @@ -64,29 +71,28 @@ class BlockAccessServiceTest { @Mock private BlockReader blockReader; - @Mock - private BlockWriter> blockWriter; - @Mock private ServiceStatus serviceStatus; - private static final int testTimeout = 1000; - @TempDir - private Path testPath; + private Path testLiveRootPath; private BlockNodeContext blockNodeContext; - private PersistenceStorageConfig config; + private PersistenceStorageConfig testConfig; private PbjBlockAccessService blockAccessService; @BeforeEach public void setUp() throws IOException { - blockNodeContext = - TestConfigUtil.getTestBlockNodeContext(Map.of("persistence.storage.rootPath", testPath.toString())); - config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + blockNodeContext = TestConfigUtil.getTestBlockNodeContext( + Map.of(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString())); + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); blockAccessService = new PbjBlockAccessServiceProxy(serviceStatus, blockReader, blockNodeContext); + + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = mock(BlockAsLocalDirPathResolver.class); } @Test @@ -106,9 +112,7 @@ public void testMethods() { @Test void testSingleBlockHappyPath() throws IOException, ParseException { - - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); final PbjBlockAccessService blockAccessService = new PbjBlockAccessServiceProxy(serviceStatus, blockReader, blockNodeContext); @@ -116,9 +120,12 @@ void testSingleBlockHappyPath() throws IOException, ParseException { // Enable the serviceStatus when(serviceStatus.isRunning()).thenReturn(true); + final BlockAsLocalDirPathResolver trainedPathResolver = + PersistTestUtils.trainAndReturnBlockAsLocalDirPathResolver( + pathResolverMock, Path.of(testConfig.liveRootPath())); // Generate and persist a block final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), trainedPathResolver); final List blockItems = generateBlockItemsUnparsed(1); blockWriter.write(blockItems); diff --git a/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java b/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java index 13feee431..2806244f8 100644 --- a/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java +++ b/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java @@ -19,13 +19,14 @@ import static com.hedera.block.server.util.PbjProtoTestUtils.buildAck; import static com.hedera.block.server.util.PbjProtoTestUtils.buildEmptyPublishStreamRequest; import static com.hedera.block.server.util.PbjProtoTestUtils.buildEmptySubscribeStreamRequest; +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; -import static java.lang.System.Logger; -import static java.lang.System.Logger.Level.INFO; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; @@ -39,13 +40,16 @@ import com.hedera.block.server.notifier.Notifier; import com.hedera.block.server.notifier.NotifierImpl; import com.hedera.block.server.persistence.StreamPersistenceHandlerImpl; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; +import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.service.ServiceStatusImpl; +import com.hedera.block.server.util.PersistTestUtils; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.Acknowledgement; import com.hedera.hapi.block.BlockItemSetUnparsed; import com.hedera.hapi.block.BlockItemUnparsed; @@ -66,7 +70,6 @@ import com.lmax.disruptor.BatchEventProcessor; import io.helidon.webserver.WebServer; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.security.NoSuchAlgorithmException; import java.util.HashMap; @@ -75,17 +78,17 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) public class PbjBlockStreamServiceIntegrationTest { - - private final Logger LOGGER = System.getLogger(getClass().getName()); + private static final int testTimeout = 2000; + private BlockAsLocalDirPathResolver pathResolverMock; @Mock private Notifier notifier; @@ -129,27 +132,24 @@ public class PbjBlockStreamServiceIntegrationTest { @Mock private ServiceInterface.RequestOptions options; - private static final String TEMP_DIR = "block-node-unit-test-dir"; + @TempDir + private Path testLiveRootPath; - private Path testPath; private BlockNodeContext blockNodeContext; - - private static final int testTimeout = 2000; + private PersistenceStorageConfig testConfig; @BeforeEach public void setUp() throws IOException { - testPath = Files.createTempDirectory(TEMP_DIR); - LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); - Map properties = new HashMap<>(); - properties.put("persistence.storage.rootPath", testPath.toString()); - + properties.put(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString()); blockNodeContext = TestConfigUtil.getTestBlockNodeContext(properties); - } + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + + pathResolverMock = mock(BlockAsLocalDirPathResolver.class); - @AfterEach - public void tearDown() { - TestUtils.deleteDirectory(testPath.toFile()); + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = mock(BlockAsLocalDirPathResolver.class); } @Test @@ -250,9 +250,12 @@ public void testPublishBlockStreamRegistrationAndExecution() public void testFullProducerConsumerHappyPath() throws IOException { int numberOfBlocks = 100; + final BlockAsLocalDirPathResolver trainedPathResolver = + PersistTestUtils.trainAndReturnBlockAsLocalDirPathResolver( + pathResolverMock, Path.of(testConfig.liveRootPath())); // Use a real BlockWriter to test the full integration final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), trainedPathResolver); final PbjBlockStreamServiceProxy pbjBlockStreamServiceProxy = buildBlockStreamService(blockWriter); // Register 3 producers - Opening a pipeline is not enough to register a producer. @@ -515,7 +518,7 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep // Use a spy to make sure the write() method throws an IOException final BlockWriter> blockWriter = - spy(BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build()); + spy(BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock)); doThrow(IOException.class).when(blockWriter).write(blockItems); final var streamMediator = buildStreamMediator(consumers, serviceStatus); diff --git a/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java b/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java index 0f32b98df..16f4322d0 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java @@ -16,9 +16,8 @@ package com.hedera.block.server.persistence; +import static org.assertj.core.api.Assertions.assertThatRuntimeException; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -26,9 +25,13 @@ import com.hedera.block.server.events.BlockNodeEventHandler; import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.mediator.SubscriptionHandler; +import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.notifier.Notifier; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig.StorageType; +import com.hedera.block.server.persistence.storage.path.NoOpBlockPathResolver; import com.hedera.block.server.persistence.storage.read.BlockReader; +import com.hedera.block.server.persistence.storage.remove.NoOpBlockRemover; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; @@ -75,27 +78,33 @@ void setup() throws IOException { @Test void testProvidesBlockWriter() { - BlockWriter> blockWriter = - PersistenceInjectionModule.providesBlockWriter(blockNodeContext); + BlockWriter> blockWriter = PersistenceInjectionModule.providesBlockWriter( + blockNodeContext, NoOpBlockRemover.newInstance(), NoOpBlockPathResolver.newInstance()); assertNotNull(blockWriter); } @Test void testProvidesBlockWriter_IOException() { - BlockNodeContext blockNodeContext = mock(BlockNodeContext.class); - PersistenceStorageConfig persistenceStorageConfig = mock(PersistenceStorageConfig.class); - when(persistenceStorageConfig.rootPath()).thenReturn("/invalid_path/:invalid_directory"); - Configuration configuration = mock(Configuration.class); + final BlockNodeContext blockNodeContext = mock(BlockNodeContext.class); + + final PersistenceStorageConfig persistenceStorageConfig = mock(PersistenceStorageConfig.class); + when(persistenceStorageConfig.liveRootPath()).thenReturn("/invalid_path/:invalid_directory"); + when(persistenceStorageConfig.type()).thenReturn(StorageType.BLOCK_AS_LOCAL_DIRECTORY); + + final Configuration configuration = mock(Configuration.class); when(blockNodeContext.configuration()).thenReturn(configuration); when(configuration.getConfigData(PersistenceStorageConfig.class)).thenReturn(persistenceStorageConfig); - // Expect a RuntimeException due to the IOException - RuntimeException exception = assertThrows( - RuntimeException.class, () -> PersistenceInjectionModule.providesBlockWriter(blockNodeContext)); + final MetricsService metricsServiceMock = mock(MetricsService.class); + when(blockNodeContext.metricsService()).thenReturn(metricsServiceMock); - // Verify the exception message - assertTrue(exception.getMessage().contains("Failed to create block writer")); + // Expect a RuntimeException due to the IOException + assertThatRuntimeException() + .isThrownBy(() -> PersistenceInjectionModule.providesBlockWriter( + blockNodeContext, NoOpBlockRemover.newInstance(), NoOpBlockPathResolver.newInstance())) + .withCauseInstanceOf(IOException.class) + .withMessage("Failed to create BlockWriter"); } @Test diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java index dfe4d8b3e..3db2ded21 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfigTest.java @@ -16,74 +16,245 @@ package com.hedera.block.server.persistence.storage; -import static org.junit.jupiter.api.Assertions.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.from; +import com.hedera.block.server.persistence.storage.PersistenceStorageConfig.StorageType; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; +import java.util.Arrays; import java.util.Comparator; import java.util.stream.Stream; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +/** + * Test class that tests the functionality of the + * {@link PersistenceStorageConfig} + */ class PersistenceStorageConfigTest { + private static final Path HASHGRAPH_ROOT_ABSOLUTE_PATH = + Path.of("hashgraph/").toAbsolutePath(); + private static final Path PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH = + HASHGRAPH_ROOT_ABSOLUTE_PATH.resolve("blocknode/data/"); - final String TEMP_DIR = "block-node-unit-test-dir"; + @AfterEach + void tearDown() { + if (!Files.exists(HASHGRAPH_ROOT_ABSOLUTE_PATH)) { + return; + } + try (final Stream walk = Files.walk(HASHGRAPH_ROOT_ABSOLUTE_PATH)) { + walk.sorted(Comparator.reverseOrder()).forEach(p -> { + try { + Files.delete(p); + } catch (final IOException e) { + throw new RuntimeException(e); + } + }); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } - @Test - void testPersistenceStorageConfig_happyPath() throws IOException { + /** + * This test aims to verify that the {@link PersistenceStorageConfig} class + * correctly returns the storage type that was set in the constructor. + * + * @param storageType parameterized, the storage type to test + */ + @ParameterizedTest + @MethodSource("storageTypes") + void testPersistenceStorageConfigStorageTypes(final StorageType storageType) { + final PersistenceStorageConfig actual = new PersistenceStorageConfig("", "", storageType); + assertThat(actual).returns(storageType, from(PersistenceStorageConfig::type)); + } - Path testPath = Files.createTempDirectory(TEMP_DIR); + /** + * This test aims to verify that the {@link PersistenceStorageConfig} class + * correctly sets the live and archive root paths. + * + * @param liveRootPathToTest parameterized, the live root path to test + * @param expectedLiveRootPathToTest parameterized, the expected live root + * @param archiveRootPathToTest parameterized, the archive root path to test + * @param expectedArchiveRootPathToTest parameterized, the expected archive + * root + */ + @ParameterizedTest + @MethodSource({"validAbsoluteDefaultRootPaths", "validAbsoluteNonDefaultRootPaths"}) + void testPersistenceStorageConfigHappyPaths( + final String liveRootPathToTest, + final String expectedLiveRootPathToTest, + final String archiveRootPathToTest, + final String expectedArchiveRootPathToTest) { + final PersistenceStorageConfig actual = new PersistenceStorageConfig( + liveRootPathToTest, archiveRootPathToTest, StorageType.BLOCK_AS_LOCAL_FILE); + assertThat(actual) + .returns(expectedLiveRootPathToTest, from(PersistenceStorageConfig::liveRootPath)) + .returns(expectedArchiveRootPathToTest, from(PersistenceStorageConfig::archiveRootPath)); + } - PersistenceStorageConfig persistenceStorageConfig = - new PersistenceStorageConfig(testPath.toString(), "PRODUCTION"); - assertEquals(testPath.toString(), persistenceStorageConfig.rootPath()); + /** + * This test aims to verify that the {@link PersistenceStorageConfig} class + * correctly throws an {@link UncheckedIOException} when either the live or + * archive root paths are invalid. + * + * @param invalidLiveRootPathToTest parameterized, the invalid live root + * path to test + * @param invalidArchiveRootPathToTest parameterized, the invalid archive + * root path to test + */ + @ParameterizedTest + @MethodSource({"invalidRootPaths"}) + void testPersistenceStorageConfigInvalidRootPaths( + final String invalidLiveRootPathToTest, final String invalidArchiveRootPathToTest) { + assertThatExceptionOfType(UncheckedIOException.class) + .isThrownBy(() -> new PersistenceStorageConfig( + invalidLiveRootPathToTest, invalidArchiveRootPathToTest, StorageType.BLOCK_AS_LOCAL_FILE)); } - @Test - void testPersistenceStorageConfig_emptyRootPath() throws IOException { - final String expectedDefaultRootPath = - Paths.get("").toAbsolutePath().resolve("data_empty").toString(); - // delete if exists - deleteDirectory(Paths.get(expectedDefaultRootPath)); + /** + * This test aims to verify that the {@link PersistenceStorageConfig} class + * correctly throws an {@link IllegalArgumentException} when either the live + * or archive root paths are relative. + * + * @param invalidLiveRootPathToTest parameterized, the invalid live root + * path to test + * @param invalidArchiveRootPathToTest parameterized, the invalid archive + * root path to test + */ + @ParameterizedTest + @MethodSource({"validNonAbsolutePaths"}) + void testPersistenceStorageConfigRelativeRootPaths( + final String invalidLiveRootPathToTest, final String invalidArchiveRootPathToTest) { + assertThatIllegalArgumentException() + .isThrownBy(() -> new PersistenceStorageConfig( + invalidLiveRootPathToTest, invalidArchiveRootPathToTest, StorageType.BLOCK_AS_LOCAL_FILE)); + } - PersistenceStorageConfig persistenceStorageConfig = - new PersistenceStorageConfig(getAbsoluteFolder("data_empty"), "PRODUCTION"); - assertEquals(expectedDefaultRootPath, persistenceStorageConfig.rootPath()); + /** + * All storage types dynamically provided. + */ + private static Stream storageTypes() { + return Arrays.stream(StorageType.values()).map(Arguments::of); } - @Test - void persistenceStorageConfig_throwsExceptionForRelativePath() { - IllegalArgumentException exception = assertThrows( - IllegalArgumentException.class, () -> new PersistenceStorageConfig("relative/path", "PRODUCTION")); - assertEquals("relative/path Root path must be absolute", exception.getMessage()); + /** + * The default absolute paths. We expect these to allow the persistence + * config to be instantiated. Providing a blank string is accepted, it will + * create the config instance with it's internal defaults. + */ + @SuppressWarnings("all") + private static Stream validAbsoluteDefaultRootPaths() { + final Path defaultLiveRootAbsolutePath = + PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH.resolve("live/").toAbsolutePath(); + final Path defaultArchiveRootAbsolutePath = + PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH.resolve("archive/").toAbsolutePath(); + + // test against the default liveRootPath and archiveRootPath + final String liveToTest1 = defaultLiveRootAbsolutePath.toString(); + final String liveExpected1 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest1 = defaultArchiveRootAbsolutePath.toString(); + final String archiveExpected1 = defaultArchiveRootAbsolutePath.toString(); + + // blank liveRootPath results in the default liveRootPath to be used + final String liveToTest2 = ""; + final String liveExpected2 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest2 = defaultArchiveRootAbsolutePath.toString(); + final String archiveExpected2 = defaultArchiveRootAbsolutePath.toString(); + + // blank archiveRootPath results in the default archiveRootPath to be used + final String liveToTest3 = defaultLiveRootAbsolutePath.toString(); + final String liveExpected3 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest3 = ""; + final String archiveExpected3 = defaultArchiveRootAbsolutePath.toString(); + + // null liveRootPath results in the default liveRootPath to be used + final String liveToTest4 = null; + final String liveExpected4 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest4 = defaultArchiveRootAbsolutePath.toString(); + final String archiveExpected4 = defaultArchiveRootAbsolutePath.toString(); + + // null archiveRootPath results in the default archiveRootPath to be used + final String liveToTest5 = defaultLiveRootAbsolutePath.toString(); + final String liveExpected5 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest5 = null; + final String archiveExpected5 = defaultArchiveRootAbsolutePath.toString(); + + // blank liveRootPath and archiveRootPath results in the default liveRootPath and archiveRootPath to be used + final String liveToTest6 = ""; + final String liveExpected6 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest6 = ""; + final String archiveExpected6 = defaultArchiveRootAbsolutePath.toString(); + + // null liveRootPath and archiveRootPath results in the default liveRootPath and archiveRootPath to be used + final String liveToTest7 = null; + final String liveExpected7 = defaultLiveRootAbsolutePath.toString(); + final String archiveToTest7 = null; + final String archiveExpected7 = defaultArchiveRootAbsolutePath.toString(); + + return Stream.of( + Arguments.of(liveToTest1, liveExpected1, archiveToTest1, archiveExpected1), + Arguments.of(liveToTest2, liveExpected2, archiveToTest2, archiveExpected2), + Arguments.of(liveToTest3, liveExpected3, archiveToTest3, archiveExpected3), + Arguments.of(liveToTest4, liveExpected4, archiveToTest4, archiveExpected4), + Arguments.of(liveToTest5, liveExpected5, archiveToTest5, archiveExpected5), + Arguments.of(liveToTest6, liveExpected6, archiveToTest6, archiveExpected6), + Arguments.of(liveToTest7, liveExpected7, archiveToTest7, archiveExpected7)); } - @Test - void persistenceStorageConfig_throwsRuntimeExceptionOnIOException() { - Path invalidPath = Paths.get("/invalid/path"); + /** + * Somve valid absolute paths that are not the default paths. We expect + * these to allow the persistence config to be instantiated. + */ + @SuppressWarnings("all") + private static Stream validAbsoluteNonDefaultRootPaths() { + final String liveToTest1 = PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH + .resolve("nondefault/live/") + .toString(); + final String archiveToTest1 = PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH + .resolve("nondefault/archive/") + .toString(); - RuntimeException exception = assertThrows( - RuntimeException.class, () -> new PersistenceStorageConfig(invalidPath.toString(), "PRODUCTION")); - assertInstanceOf(IOException.class, exception.getCause()); + final String liveToTest2 = PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH + .resolve("another/nondefault/live/") + .toString(); + final String archiveToTest2 = PERSISTENCE_STORAGE_ROOT_ABSOLUTE_PATH + .resolve("another/nondefault/archive/") + .toString(); + + return Stream.of( + Arguments.of(liveToTest1, liveToTest1, archiveToTest1, archiveToTest1), + Arguments.of(liveToTest2, liveToTest2, archiveToTest2, archiveToTest2)); } - public static void deleteDirectory(Path path) throws IOException { - if (!Files.exists(path)) { - return; - } - try (Stream walk = Files.walk(path)) { - walk.sorted(Comparator.reverseOrder()).forEach(p -> { - try { - Files.delete(p); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } + /** + * Supplying blank is valid, both must be valid paths in order to be able + * to create the config instance. If either liveRootPath or archiveRootPath + * is invalid, we expect to fail. If we provide external paths, they must be + * absolute. + */ + private static Stream validNonAbsolutePaths() { + // these must fail, if we provide external paths, they must be absolute + final String liveToTest = "hashgraph/blocknode/data/relative/live/"; + final String archiveToTest = "hashgraph/blocknode/data/relative/archive/"; + return Stream.of( + Arguments.of("", archiveToTest), Arguments.of(liveToTest, ""), Arguments.of(liveToTest, archiveToTest)); } - private String getAbsoluteFolder(String relativePath) { - return Paths.get(relativePath).toAbsolutePath().toString(); + /** + * Supplying blank is valid, both must be valid paths in order to be able + * to create the config instance. If either liveRootPath or archiveRootPath + * is invalid, we expect to fail. There cannot be invalid paths supplied. + */ + private static Stream invalidRootPaths() { + final String invalidPath = "/invalid_path/:invalid_directory"; + return Stream.of( + Arguments.of("", invalidPath), Arguments.of(invalidPath, ""), Arguments.of(invalidPath, invalidPath)); } } diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolverTest.java new file mode 100644 index 000000000..7b1fe6cab --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalDirPathResolverTest.java @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Path; +import java.util.stream.Stream; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link BlockAsLocalDirPathResolver} class. + */ +class BlockAsLocalDirPathResolverTest { + @TempDir + private static Path TEST_LIVE_ROOT_PATH; + + private BlockAsLocalDirPathResolver toTest; + + @BeforeEach + void setUp() { + toTest = BlockAsLocalDirPathResolver.of(TEST_LIVE_ROOT_PATH); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalDirPathResolver#resolvePathToBlock(long)} correctly + * resolves the path to a block by a given number. For the + * block-as-local-directory storage strategy, the path to a block is simply + * the live root path appended with the given block number. + * + * @param toResolve parameterized, valid block number + * @param expected parameterized, expected path + */ + @ParameterizedTest + @MethodSource("validBlockNumbers") + void testSuccessfulPathResolution(final long toResolve, final Path expected) { + final Path actual = toTest.resolvePathToBlock(toResolve); + assertThat(actual).isNotNull().isEqualTo(expected); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalDirPathResolver#resolvePathToBlock(long)} correctly + * throws an {@link IllegalArgumentException} when an invalid block number + * is provided. A block number is invalid if it is a strictly negative number. + * + * @param toResolve parameterized, invalid block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long toResolve) { + Assertions.assertThatIllegalArgumentException().isThrownBy(() -> toTest.resolvePathToBlock(toResolve)); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L, TEST_LIVE_ROOT_PATH.resolve("0")), + Arguments.of(1L, TEST_LIVE_ROOT_PATH.resolve("1")), + Arguments.of(2L, TEST_LIVE_ROOT_PATH.resolve("2")), + Arguments.of(10L, TEST_LIVE_ROOT_PATH.resolve("10")), + Arguments.of(100L, TEST_LIVE_ROOT_PATH.resolve("100")), + Arguments.of(1_000L, TEST_LIVE_ROOT_PATH.resolve("1000")), + Arguments.of(10_000L, TEST_LIVE_ROOT_PATH.resolve("10000")), + Arguments.of(100_000L, TEST_LIVE_ROOT_PATH.resolve("100000")), + Arguments.of(1_000_000L, TEST_LIVE_ROOT_PATH.resolve("1000000")), + Arguments.of(10_000_000L, TEST_LIVE_ROOT_PATH.resolve("10000000")), + Arguments.of(100_000_000L, TEST_LIVE_ROOT_PATH.resolve("100000000")), + Arguments.of(1_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("1000000000")), + Arguments.of(10_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("10000000000")), + Arguments.of(100_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("100000000000")), + Arguments.of(1_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("1000000000000")), + Arguments.of(10_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("10000000000000")), + Arguments.of(100_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("100000000000000")), + Arguments.of(1_000_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("1000000000000000")), + Arguments.of(10_000_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("10000000000000000")), + Arguments.of(100_000_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("100000000000000000")), + Arguments.of(1_000_000_000_000_000_000L, TEST_LIVE_ROOT_PATH.resolve("1000000000000000000")), + Arguments.of(Long.MAX_VALUE, TEST_LIVE_ROOT_PATH.resolve(String.valueOf(Long.MAX_VALUE)))); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolverTest.java new file mode 100644 index 000000000..396f1e369 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/path/BlockAsLocalFilePathResolverTest.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +import com.hedera.block.server.Constants; +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link BlockAsLocalFilePathResolver} class. + */ +class BlockAsLocalFilePathResolverTest { + private static final long MAX_LONG_DIGITS = 19L; + + @TempDir + private static Path TEST_LIVE_ROOT_PATH; + + private BlockAsLocalFilePathResolver toTest; + + @BeforeEach + void setUp() { + toTest = BlockAsLocalFilePathResolver.of(TEST_LIVE_ROOT_PATH); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFilePathResolver#resolvePathToBlock(long)} correctly + * resolves the path to a block by a given number. For the + * block-as-file storage strategy, the path to a block is a trie structure + * where each digit of the block number is a directory and the block number + * itself is the file name. + * + * @param toResolve parameterized, valid block number + * @param expected parameterized, expected path + */ + @ParameterizedTest + @MethodSource("validBlockNumbers") + void testSuccessfulPathResolution(final long toResolve, final Path expected) { + final Path actual = toTest.resolvePathToBlock(toResolve); + assertThat(actual).isNotNull().isEqualTo(expected); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFilePathResolver#resolvePathToBlock(long)} correctly + * throws an {@link IllegalArgumentException} when an invalid block number + * is provided. A block number is invalid if it is a strictly negative number. + * + * @param toResolve parameterized, invalid block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long toResolve) { + assertThatIllegalArgumentException().isThrownBy(() -> toTest.resolvePathToBlock(toResolve)); + } + + private static Path expectedResolvedBlockPath(final long blockNumber) { + final String rawBlockNumber = String.format("%0" + MAX_LONG_DIGITS + "d", blockNumber); + final String[] blockPath = rawBlockNumber.split(""); + final String blockFileName = rawBlockNumber.concat(Constants.BLOCK_FILE_EXTENSION); + blockPath[blockPath.length - 1] = blockFileName; + return Path.of(TEST_LIVE_ROOT_PATH.toString(), blockPath); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L, expectedResolvedBlockPath(0L)), + Arguments.of(1L, expectedResolvedBlockPath(1L)), + Arguments.of(2L, expectedResolvedBlockPath(2L)), + Arguments.of(10L, expectedResolvedBlockPath(10L)), + Arguments.of(100L, expectedResolvedBlockPath(100L)), + Arguments.of(1_000L, expectedResolvedBlockPath(1_000L)), + Arguments.of(10_000L, expectedResolvedBlockPath(10_000L)), + Arguments.of(100_000L, expectedResolvedBlockPath(100_000L)), + Arguments.of(1_000_000L, expectedResolvedBlockPath(1_000_000L)), + Arguments.of(10_000_000L, expectedResolvedBlockPath(10_000_000L)), + Arguments.of(100_000_000L, expectedResolvedBlockPath(100_000_000L)), + Arguments.of(1_000_000_000L, expectedResolvedBlockPath(1_000_000_000L)), + Arguments.of(10_000_000_000L, expectedResolvedBlockPath(10_000_000_000L)), + Arguments.of(100_000_000_000L, expectedResolvedBlockPath(100_000_000_000L)), + Arguments.of(1_000_000_000_000L, expectedResolvedBlockPath(1_000_000_000_000L)), + Arguments.of(10_000_000_000_000L, expectedResolvedBlockPath(10_000_000_000_000L)), + Arguments.of(100_000_000_000_000L, expectedResolvedBlockPath(100_000_000_000_000L)), + Arguments.of(1_000_000_000_000_000L, expectedResolvedBlockPath(1_000_000_000_000_000L)), + Arguments.of(10_000_000_000_000_000L, expectedResolvedBlockPath(10_000_000_000_000_000L)), + Arguments.of(100_000_000_000_000_000L, expectedResolvedBlockPath(100_000_000_000_000_000L)), + Arguments.of(1_000_000_000_000_000_000L, expectedResolvedBlockPath(1_000_000_000_000_000_000L)), + Arguments.of(Long.MAX_VALUE, expectedResolvedBlockPath(Long.MAX_VALUE))); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolverTest.java new file mode 100644 index 000000000..fc6895724 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/path/NoOpBlockPathResolverTest.java @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.path; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link NoOpBlockPathResolver} class. + */ +class NoOpBlockPathResolverTest { + private NoOpBlockPathResolver toTest; + + @BeforeEach + void setUp() { + toTest = NoOpBlockPathResolver.newInstance(); + } + + /** + * This test aims to verify that the + * {@link NoOpBlockPathResolver#resolvePathToBlock(long)} correctly resolves + * the path to a block by a given number. The no-op resolver does nothing, + * always returns null and has no preconditions check. + * + * @param toResolve parameterized, block number + */ + @ParameterizedTest + @MethodSource({"validBlockNumbers", "invalidBlockNumbers"}) + void testSuccessfulPathResolution(final long toResolve) { + final Path actual = toTest.resolvePathToBlock(toResolve); + // we always expect null, no preconditions as well + assertThat(actual).isNull(); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L), + Arguments.of(1L), + Arguments.of(2L), + Arguments.of(10L), + Arguments.of(100L), + Arguments.of(1_000L), + Arguments.of(10_000L), + Arguments.of(100_000L), + Arguments.of(1_000_000L), + Arguments.of(10_000_000L), + Arguments.of(100_000_000L), + Arguments.of(1_000_000_000L), + Arguments.of(10_000_000_000L), + Arguments.of(100_000_000_000L), + Arguments.of(1_000_000_000_000L), + Arguments.of(10_000_000_000_000L), + Arguments.of(100_000_000_000_000L), + Arguments.of(1_000_000_000_000_000L), + Arguments.of(10_000_000_000_000_000L), + Arguments.of(100_000_000_000_000_000L), + Arguments.of(1_000_000_000_000_000_000L), + Arguments.of(Long.MAX_VALUE)); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReaderTest.java similarity index 72% rename from server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java rename to server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReaderTest.java index 6a86cc045..6e9c55ee7 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalDirReaderTest.java @@ -17,21 +17,27 @@ package com.hedera.block.server.persistence.storage.read; import static com.hedera.block.server.Constants.BLOCK_FILE_EXTENSION; +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; import static com.hedera.block.server.util.PersistTestUtils.reverseByteArray; import static com.hedera.block.server.util.PersistTestUtils.writeBlockItemToPath; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.util.PersistTestUtils; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.BlockItemUnparsed; @@ -52,27 +58,30 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -public class BlockAsDirReaderTest { - +public class BlockAsLocalDirReaderTest { @TempDir - private Path testPath; + private Path testLiveRootPath; + private BlockAsLocalDirPathResolver pathResolverMock; private BlockNodeContext blockNodeContext; - private PersistenceStorageConfig config; + private PersistenceStorageConfig testConfig; private List blockItems; @BeforeEach public void setUp() throws IOException { - blockNodeContext = - TestConfigUtil.getTestBlockNodeContext(Map.of("persistence.storage.rootPath", testPath.toString())); - config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + blockNodeContext = TestConfigUtil.getTestBlockNodeContext( + Map.of(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString())); + testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); blockItems = generateBlockItemsUnparsed(1); + + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = PersistTestUtils.getTrainedBlockAsLocalDirPathResolver(Path.of(testConfigLiveRootPath)); } @Test public void testReadBlockDoesNotExist() throws IOException, ParseException { - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); final Optional blockOpt = blockReader.read(10000); assertTrue(blockOpt.isEmpty()); } @@ -80,64 +89,43 @@ public void testReadBlockDoesNotExist() throws IOException, ParseException { @Test public void testReadPermsRepairSucceeded() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (BlockItemUnparsed blockItem : blockItems) { + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); + for (final BlockItemUnparsed blockItem : blockItems) { blockWriter.write(List.of(blockItem)); } // Make the block unreadable - removeBlockReadPerms(1, config); + removeBlockReadPerms(1, testConfig); // The default BlockReader will attempt to repair the permissions and should succeed - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); final Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); assertEquals(10, blockOpt.get().blockItems().size()); } - @Test - public void testRemoveBlockReadPermsRepairFailed() throws IOException, ParseException { - final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - blockWriter.write(blockItems); - - // Make the block unreadable - removeBlockReadPerms(1, config); - - // For this test, build the Reader with ineffective repair permissions to - // simulate a failed repair (root changed the perms, etc.) - final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config) - .folderPermissions(TestUtils.getNoPerms()) - .build(); - final Optional blockOpt = blockReader.read(1); - assertTrue(blockOpt.isEmpty()); - } - @Test public void testRemoveBlockItemReadPerms() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); blockWriter.write(blockItems); - removeBlockItemReadPerms(1, 1, config); + removeBlockItemReadPerms(1, 1, testConfig); - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); assertThrows(IOException.class, () -> blockReader.read(1)); } @Test public void testPathIsNotDirectory() throws IOException, ParseException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(testConfig.liveRootPath()); // Write a file named "1" where a directory should be writeBlockItemToPath(blockNodeRootPath.resolve(Path.of("1")), blockItems.getFirst()); // Should return empty because the path is not a directory - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); final Optional blockOpt = blockReader.read(1); assertTrue(blockOpt.isEmpty()); } @@ -145,15 +133,15 @@ public void testPathIsNotDirectory() throws IOException, ParseException { @Test public void testRepairReadPermsFails() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); blockWriter.write(blockItems); - removeBlockReadPerms(1, config); + removeBlockReadPerms(1, testConfig); // Use a spy on a subclass of the BlockAsDirReader to proxy calls // to the actual methods but to also throw an IOException when // the setPerm method is called. - final TestBlockAsDirReader blockReader = spy(new TestBlockAsDirReader(config)); + final TestBlockAsLocalDirReader blockReader = spy(new TestBlockAsLocalDirReader(testConfig)); doThrow(IOException.class).when(blockReader).setPerm(any(), any()); final Optional blockOpt = blockReader.read(1); @@ -164,12 +152,12 @@ public void testRepairReadPermsFails() throws IOException, ParseException { public void testBlockNodePathReadFails() throws IOException, ParseException { // Remove read perm on the root path - removePathReadPerms(Path.of(config.rootPath())); + removePathReadPerms(Path.of(testConfig.liveRootPath())); // Use a spy on a subclass of the BlockAsDirReader to proxy calls // to the actual methods but to also throw an IOException when // the setPerm method is called. - final TestBlockAsDirReader blockReader = spy(new TestBlockAsDirReader(config)); + final TestBlockAsLocalDirReader blockReader = spy(new TestBlockAsLocalDirReader(testConfig)); doThrow(IOException.class).when(blockReader).setPerm(any(), any()); final Optional blockOpt = blockReader.read(1); @@ -179,18 +167,17 @@ public void testBlockNodePathReadFails() throws IOException, ParseException { @Test public void testParseExceptionHandling() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); blockWriter.write(blockItems); // Read the block back and confirm it's read successfully - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(config).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); final Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); final PersistenceStorageConfig persistenceStorageConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); - final Path blockNodeRootPath = Path.of(persistenceStorageConfig.rootPath()); + final Path blockNodeRootPath = Path.of(persistenceStorageConfig.liveRootPath()); Path blockPath = blockNodeRootPath.resolve(String.valueOf(1)); byte[] bytes; @@ -212,7 +199,7 @@ public void testParseExceptionHandling() throws IOException, ParseException { } public static void removeBlockReadPerms(int blockNumber, final PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(config.liveRootPath()); final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); removePathReadPerms(blockPath); } @@ -223,7 +210,7 @@ static void removePathReadPerms(final Path path) throws IOException { private void removeBlockItemReadPerms(int blockNumber, int blockItem, PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(config.liveRootPath()); final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); final Path blockItemPath = blockPath.resolve(blockItem + BLOCK_FILE_EXTENSION); Files.setPosixFilePermissions(blockItemPath, TestUtils.getNoRead().value()); @@ -231,9 +218,9 @@ private void removeBlockItemReadPerms(int blockNumber, int blockItem, Persistenc // TestBlockAsDirReader overrides the setPerm() method to allow a test spy to simulate an // IOException while allowing the real setPerm() method to remain protected. - private static final class TestBlockAsDirReader extends BlockAsDirReader { - public TestBlockAsDirReader(PersistenceStorageConfig config) { - super(config, null); + private static final class TestBlockAsLocalDirReader extends BlockAsLocalDirReader { + public TestBlockAsLocalDirReader(PersistenceStorageConfig config) { + super(config); } @Override diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReaderTest.java new file mode 100644 index 000000000..495e3bac0 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsLocalFileReaderTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.read; + +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; + +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link BlockAsLocalFileReader} class. + */ +class BlockAsLocalFileReaderTest { + private BlockAsLocalFileReader toTest; + + @BeforeEach + void setUp() { + toTest = BlockAsLocalFileReader.newInstance(); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFileReader#read(long)} correctly reads a block with + * a given block number. + * + * @param toRead parameterized, block number + */ + @ParameterizedTest + @MethodSource("validBlockNumbers") + void testSuccessfulBlockRead(final long toRead) { + // todo currently throws UnsupportedOperationException + assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> toTest.read(toRead)); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFileReader#read(long)} correctly throws an + * {@link IllegalArgumentException} when an invalid block number is + * provided. A block number is invalid if it is a strictly negative number. + * + * @param toRead parameterized, block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long toRead) { + assertThatIllegalArgumentException().isThrownBy(() -> toTest.read(toRead)); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L), + Arguments.of(1L), + Arguments.of(2L), + Arguments.of(10L), + Arguments.of(100L), + Arguments.of(1_000L), + Arguments.of(10_000L), + Arguments.of(100_000L), + Arguments.of(1_000_000L), + Arguments.of(10_000_000L), + Arguments.of(100_000_000L), + Arguments.of(1_000_000_000L), + Arguments.of(10_000_000_000L), + Arguments.of(100_000_000_000L), + Arguments.of(1_000_000_000_000L), + Arguments.of(10_000_000_000_000L), + Arguments.of(100_000_000_000_000L), + Arguments.of(1_000_000_000_000_000L), + Arguments.of(10_000_000_000_000_000L), + Arguments.of(100_000_000_000_000_000L), + Arguments.of(1_000_000_000_000_000_000L), + Arguments.of(Long.MAX_VALUE)); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReaderTest.java new file mode 100644 index 000000000..403a3f42b --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/NoOpBlockReaderTest.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.read; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.pbj.runtime.ParseException; +import java.io.IOException; +import java.util.Optional; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests for the {@link NoOpBlockReader} class. + */ +class NoOpBlockReaderTest { + private NoOpBlockReader toTest; + + @BeforeEach + void setUp() { + toTest = NoOpBlockReader.newInstance(); + } + + /** + * This test aims to verify that the + * {@link NoOpBlockReader#read(long)} does nothing and does not throw any + * exceptions. The no-op reader has no preconditions check as well. The + * method always returns an empty {@link Optional}. + * + * @param toRead parameterized, block number + */ + @ParameterizedTest + @MethodSource({"validBlockNumbers", "invalidBlockNumbers"}) + void testSuccessfulBlockReading(final long toRead) throws IOException, ParseException { + final Optional actual = toTest.read(toRead); + assertThat(actual).isNotNull().isEmpty(); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L), + Arguments.of(1L), + Arguments.of(2L), + Arguments.of(10L), + Arguments.of(100L), + Arguments.of(1_000L), + Arguments.of(10_000L), + Arguments.of(100_000L), + Arguments.of(1_000_000L), + Arguments.of(10_000_000L), + Arguments.of(100_000_000L), + Arguments.of(1_000_000_000L), + Arguments.of(10_000_000_000L), + Arguments.of(100_000_000_000L), + Arguments.of(1_000_000_000_000L), + Arguments.of(10_000_000_000_000L), + Arguments.of(100_000_000_000_000L), + Arguments.of(1_000_000_000_000_000L), + Arguments.of(10_000_000_000_000_000L), + Arguments.of(100_000_000_000_000_000L), + Arguments.of(1_000_000_000_000_000_000L), + Arguments.of(Long.MAX_VALUE)); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemoverTest.java similarity index 51% rename from server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java rename to server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemoverTest.java index fe7921a53..7db7eae22 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalDirRemoverTest.java @@ -16,15 +16,17 @@ package com.hedera.block.server.persistence.storage.remove; -import static java.lang.System.Logger; -import static java.lang.System.Logger.Level.INFO; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.from; +import static org.mockito.Mockito.mock; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReader; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; +import com.hedera.block.server.persistence.storage.write.BlockAsLocalDirWriter; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.util.PersistTestUtils; import com.hedera.block.server.util.TestConfigUtil; @@ -32,64 +34,64 @@ import com.hedera.hapi.block.BlockUnparsed; import com.hedera.pbj.runtime.ParseException; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; -public class BlockAsDirRemoverTest { - - private final Logger LOGGER = System.getLogger(getClass().getName()); - - private static final String TEMP_DIR = "block-node-unit-test-dir"; - - private Path testPath; +class BlockAsLocalDirRemoverTest { private BlockNodeContext blockNodeContext; private PersistenceStorageConfig testConfig; + private BlockAsLocalDirPathResolver pathResolverMock; + + @TempDir + private Path testLiveRootPath; @BeforeEach public void setUp() throws IOException { - testPath = Files.createTempDirectory(TEMP_DIR); - LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); - - blockNodeContext = - TestConfigUtil.getTestBlockNodeContext(Map.of("persistence.storage.rootPath", testPath.toString())); + blockNodeContext = TestConfigUtil.getTestBlockNodeContext( + Map.of(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString())); testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = PersistTestUtils.getTrainedBlockAsLocalDirPathResolver(Path.of(testConfigLiveRootPath)); } @Test public void testRemoveNonExistentBlock() throws IOException, ParseException { - // Write a block - final var blockItems = PersistTestUtils.generateBlockItemsUnparsed(1); + final List blockItems = PersistTestUtils.generateBlockItemsUnparsed(1); final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (BlockItemUnparsed blockItem : blockItems) { + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); + for (final BlockItemUnparsed blockItem : blockItems) { blockWriter.write(List.of(blockItem)); } // Remove a block that does not exist - final BlockRemover blockRemover = new BlockAsDirRemover(testPath); - blockRemover.remove(2); + final BlockRemover toTest = BlockAsLocalDirRemover.of(pathResolverMock); + toTest.remove(2); // Verify the block was not removed - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); - Optional blockOpt = blockReader.read(1); - assert (blockOpt.isPresent()); - assertEquals( - blockItems.getFirst().blockHeader(), - blockOpt.get().blockItems().getFirst().blockHeader()); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); + final Optional before = blockReader.read(1); + assertThat(before) + .isNotNull() + .isPresent() + .get() + .returns( + blockItems.getFirst().blockHeader(), + from(block -> block.blockItems().getFirst().blockHeader())); // Now remove the block - blockRemover.remove(1); + toTest.remove(1); // Verify the block is removed - blockOpt = blockReader.read(1); - assert (blockOpt.isEmpty()); + final Optional after = blockReader.read(1); + assertThat(after).isNotNull().isEmpty(); } } diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemoverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemoverTest.java new file mode 100644 index 000000000..4a03fa8e3 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsLocalFileRemoverTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; + +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Test class for {@link BlockAsLocalFileRemover}. + */ +class BlockAsLocalFileRemoverTest { + private BlockAsLocalFileRemover toTest; + + @BeforeEach + void setUp() { + toTest = BlockAsLocalFileRemover.newInstance(); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFileRemover#remove(long)} correctly deletes a block + * with the given block number. + * + * @param toRemove parameterized, block number + */ + @ParameterizedTest + @MethodSource("validBlockNumbers") + void testSuccessfulBlockDeletion(final long toRemove) { + // todo currently throws UnsupportedOperationException + assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> toTest.remove(toRemove)); + } + + /** + * This test aims to verify that the + * {@link BlockAsLocalFileRemover#remove(long)} correctly throws an + * {@link IllegalArgumentException} when an invalid block number is + * provided. A block number is invalid if it is a strictly negative number. + * + * @param toRemove parameterized, block number + */ + @ParameterizedTest + @MethodSource("invalidBlockNumbers") + void testInvalidBlockNumber(final long toRemove) { + assertThatIllegalArgumentException().isThrownBy(() -> toTest.remove(toRemove)); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L), + Arguments.of(1L), + Arguments.of(2L), + Arguments.of(10L), + Arguments.of(100L), + Arguments.of(1_000L), + Arguments.of(10_000L), + Arguments.of(100_000L), + Arguments.of(1_000_000L), + Arguments.of(10_000_000L), + Arguments.of(100_000_000L), + Arguments.of(1_000_000_000L), + Arguments.of(10_000_000_000L), + Arguments.of(100_000_000_000L), + Arguments.of(1_000_000_000_000L), + Arguments.of(10_000_000_000_000L), + Arguments.of(100_000_000_000_000L), + Arguments.of(1_000_000_000_000_000L), + Arguments.of(10_000_000_000_000_000L), + Arguments.of(100_000_000_000_000_000L), + Arguments.of(1_000_000_000_000_000_000L), + Arguments.of(Long.MAX_VALUE)); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemoverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemoverTest.java new file mode 100644 index 000000000..8e90fbacb --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/NoOpBlockRemoverTest.java @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence.storage.remove; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Test class for {@link NoOpBlockRemover}. + */ +class NoOpBlockRemoverTest { + private NoOpBlockRemover toTest; + + @BeforeEach + void setUp() { + toTest = NoOpBlockRemover.newInstance(); + } + + /** + * This test aims to verify that the + * {@link NoOpBlockRemover#remove(long)} does nothing and does not throw any + * exceptions. The no-op remover has no preconditions check as well. + * + * @param toRemove parameterized, block number + */ + @ParameterizedTest + @MethodSource({"validBlockNumbers", "invalidBlockNumbers"}) + void testSuccessfulBlockDeletion(final long toRemove) { + assertThatNoException().isThrownBy(() -> toTest.remove(toRemove)); + } + + /** + * Some valid block numbers. + * + * @return a stream of valid block numbers + */ + public static Stream validBlockNumbers() { + return Stream.of( + Arguments.of(0L), + Arguments.of(1L), + Arguments.of(2L), + Arguments.of(10L), + Arguments.of(100L), + Arguments.of(1_000L), + Arguments.of(10_000L), + Arguments.of(100_000L), + Arguments.of(1_000_000L), + Arguments.of(10_000_000L), + Arguments.of(100_000_000L), + Arguments.of(1_000_000_000L), + Arguments.of(10_000_000_000L), + Arguments.of(100_000_000_000L), + Arguments.of(1_000_000_000_000L), + Arguments.of(10_000_000_000_000L), + Arguments.of(100_000_000_000_000L), + Arguments.of(1_000_000_000_000_000L), + Arguments.of(10_000_000_000_000_000L), + Arguments.of(100_000_000_000_000_000L), + Arguments.of(1_000_000_000_000_000_000L), + Arguments.of(Long.MAX_VALUE)); + } + + /** + * Some invalid block numbers. + * + * @return a stream of invalid block numbers + */ + public static Stream invalidBlockNumbers() { + return Stream.of( + Arguments.of(-1L), + Arguments.of(-2L), + Arguments.of(-10L), + Arguments.of(-100L), + Arguments.of(-1_000L), + Arguments.of(-10_000L), + Arguments.of(-100_000L), + Arguments.of(-1_000_000L), + Arguments.of(-10_000_000L), + Arguments.of(-100_000_000L), + Arguments.of(-1_000_000_000L), + Arguments.of(-10_000_000_000L), + Arguments.of(-100_000_000_000L), + Arguments.of(-1_000_000_000_000L), + Arguments.of(-10_000_000_000_000L), + Arguments.of(-100_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000L), + Arguments.of(-10_000_000_000_000_000L), + Arguments.of(-100_000_000_000_000_000L), + Arguments.of(-1_000_000_000_000_000_000L), + Arguments.of(Long.MIN_VALUE)); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriterTest.java similarity index 75% rename from server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java rename to server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriterTest.java index e5583f9d4..cc45e84a9 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsLocalDirWriterTest.java @@ -16,12 +16,10 @@ package com.hedera.block.server.persistence.storage.write; -import static com.hedera.block.server.persistence.storage.read.BlockAsDirReaderTest.removeBlockReadPerms; +import static com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReaderTest.removeBlockReadPerms; +import static com.hedera.block.server.util.PersistTestUtils.PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; -import static com.hedera.block.server.util.TestConfigUtil.DEFAULT_TEST_FOLDER_PERMISSIONS; -import static java.lang.System.Logger; -import static java.lang.System.Logger.Level.ERROR; -import static java.lang.System.Logger.Level.INFO; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -31,14 +29,17 @@ import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; +import com.hedera.block.server.persistence.storage.read.BlockAsLocalDirReader; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.remove.BlockAsDirRemover; +import com.hedera.block.server.persistence.storage.remove.BlockAsLocalDirRemover; import com.hedera.block.server.persistence.storage.remove.BlockRemover; +import com.hedera.block.server.util.PersistTestUtils; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.BlockItemUnparsed; @@ -49,70 +50,54 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; -public class BlockAsDirWriterTest { - - private final Logger LOGGER = System.getLogger(getClass().getName()); - - private static final String TEMP_DIR = "block-node-unit-test-dir"; - private static final String PERSISTENCE_STORAGE_ROOT_PATH_KEY = "persistence.storage.rootPath"; - - private Path testPath; +public class BlockAsLocalDirWriterTest { private BlockNodeContext blockNodeContext; private PersistenceStorageConfig testConfig; - + private BlockAsLocalDirPathResolver pathResolverMock; private List blockItems; + @TempDir + private Path testLiveRootPath; + @BeforeEach public void setUp() throws IOException { - testPath = Files.createTempDirectory(TEMP_DIR); - LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); - - blockNodeContext = - TestConfigUtil.getTestBlockNodeContext(Map.of(PERSISTENCE_STORAGE_ROOT_PATH_KEY, testPath.toString())); + blockNodeContext = TestConfigUtil.getTestBlockNodeContext( + Map.of(PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY, testLiveRootPath.toString())); testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); blockItems = generateBlockItemsUnparsed(1); - } - @AfterEach - public void tearDown() { - if (!TestUtils.deleteDirectory(testPath.toFile())) { - LOGGER.log(ERROR, "Failed to delete temp directory: " + testPath.toString()); - } + final String testConfigLiveRootPath = testConfig.liveRootPath(); + assertThat(testConfigLiveRootPath).isEqualTo(testLiveRootPath.toString()); + pathResolverMock = PersistTestUtils.getTrainedBlockAsLocalDirPathResolver(Path.of(testConfigLiveRootPath)); } @Test public void testWriterAndReaderHappyPath() throws IOException, ParseException { - final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext) - .folderPermissions(DEFAULT_TEST_FOLDER_PERMISSIONS) - .build(); + final BlockWriter> blockWriter = + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); for (int i = 0; i < 10; i++) { + final Optional> result = blockWriter.write(List.of(blockItems.get(i))); if (i == 9) { - Optional> result = blockWriter.write(List.of(blockItems.get(i))); if (result.isPresent()) { - assertEquals(blockItems.get(i), result.get().get(0)); + assertEquals(blockItems.get(i), result.get().getFirst()); } else { fail("The optional should contain the last block proof block item"); } } else { - Optional> result = blockWriter.write(List.of(blockItems.get(i))); assertTrue(result.isEmpty()); } } // Confirm the block - BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); + BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); @@ -142,7 +127,7 @@ public void testWriterAndReaderHappyPath() throws IOException, ParseException { public void testRemoveBlockWritePerms() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); // Change the permissions on the block node root directory removeRootWritePerms(testConfig); @@ -153,8 +138,7 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { assertFalse(result.isPresent()); // Confirm we're able to read 1 block item - BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); + BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); assertEquals(1, blockOpt.get().blockItems().size()); @@ -186,14 +170,9 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { @Test public void testUnrecoverableIOExceptionOnWrite() throws IOException, ParseException { - - final BlockRemover blockRemover = new BlockAsDirRemover(Path.of(testConfig.rootPath())); - // Use a spy to simulate an IOException when the first block item is written final BlockWriter> blockWriter = - spy(BlockAsDirWriterBuilder.newBuilder(blockNodeContext) - .blockRemover(blockRemover) - .build()); + spy(BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock)); doThrow(IOException.class).when(blockWriter).write(blockItems); assertThrows(IOException.class, () -> blockWriter.write(blockItems)); } @@ -202,7 +181,7 @@ public void testUnrecoverableIOExceptionOnWrite() throws IOException, ParseExcep public void testRemoveRootDirReadPerm() throws IOException, ParseException { final BlockWriter> blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + BlockAsLocalDirWriter.of(blockNodeContext, mock(BlockRemover.class), pathResolverMock); // Write the first block item to create the block // directory @@ -220,7 +199,7 @@ public void testRemoveRootDirReadPerm() throws IOException, ParseException { if (i == 9) { result = blockWriter.write(List.of(blockItems.get(i))); if (result.isPresent()) { - assertEquals(blockItems.get(i), result.get().get(0)); + assertEquals(blockItems.get(i), result.get().getFirst()); } else { fail("The optional should contain the last block proof block item"); } @@ -230,8 +209,7 @@ public void testRemoveRootDirReadPerm() throws IOException, ParseException { } } - BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); + BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); assertEquals(10, blockOpt.get().blockItems().size()); @@ -240,23 +218,23 @@ public void testRemoveRootDirReadPerm() throws IOException, ParseException { @Test public void testPartialBlockRemoval() throws IOException, ParseException { final List blockItems = generateBlockItemsUnparsed(3); - final BlockRemover blockRemover = new BlockAsDirRemover(Path.of(testConfig.rootPath())); - + final BlockRemover blockRemover = BlockAsLocalDirRemover.of(pathResolverMock); // Use a spy of TestBlockAsDirWriter to proxy block items to the real writer - // for the first 22 block items. Then simulate an IOException on the 23rd block item + // for the first 22 block items. Then simulate an IOException on the 23rd block item // thrown from a protected write method in the real class. This should trigger the // blockRemover instance to remove the partially written block. - final TestBlockAsDirWriter blockWriter = spy(new TestBlockAsDirWriter(blockRemover, null, blockNodeContext)); + final TestBlockAsLocalDirWriter blockWriter = + spy(new TestBlockAsLocalDirWriter(blockRemover, blockNodeContext)); for (int i = 0; i < 23; i++) { // Prepare the block writer to call the actual write method // for 23 block items - doCallRealMethod().when(blockWriter).write(same(List.of(blockItems.get(i)))); + doCallRealMethod().when(blockWriter).write(any(Path.class), same(blockItems.get(i))); } // Simulate an IOException when writing the 24th block item // from an overridden write method in sub-class. - doThrow(IOException.class).when(blockWriter).write(any(), same(blockItems.get(23))); + doThrow(IOException.class).when(blockWriter).write(any(Path.class), same(blockItems.get(23))); // Now make the calls for (int i = 0; i < 23; i++) { @@ -265,19 +243,18 @@ public void testPartialBlockRemoval() throws IOException, ParseException { // The last block item in each block is the block proof // and should be returned by the write method assertTrue(result.isPresent()); - assertEquals(blockItems.get(i), result.get().get(0)); + assertEquals(blockItems.get(i), result.get().getFirst()); } else { // The write method should return an empty optional assertTrue(result.isEmpty()); } } - // Verify the IOException was thrown on the 23rd block item + // Verify the IOException was thrown on the 24th block item assertThrows(IOException.class, () -> blockWriter.write(List.of(blockItems.get(23)))); // Verify the partially written block was removed - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); + final BlockReader blockReader = BlockAsLocalDirReader.of(testConfig); Optional blockOpt = blockReader.read(3); assertTrue(blockOpt.isEmpty()); @@ -302,30 +279,27 @@ public void testPartialBlockRemoval() throws IOException, ParseException { } private void removeRootWritePerms(final PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(config.liveRootPath()); Files.setPosixFilePermissions(blockNodeRootPath, TestUtils.getNoWrite().value()); } private void removeRootReadPerms(final PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(config.liveRootPath()); Files.setPosixFilePermissions(blockNodeRootPath, TestUtils.getNoRead().value()); } private void removeBlockAllPerms(final int blockNumber, final PersistenceStorageConfig config) throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); + final Path blockNodeRootPath = Path.of(config.liveRootPath()); final Path blockPath = blockNodeRootPath.resolve(String.valueOf(blockNumber)); Files.setPosixFilePermissions(blockPath, TestUtils.getNoPerms().value()); } // TestBlockAsDirWriter overrides the write() method to allow a test spy to simulate an // IOException while allowing the real write() method to remain protected. - private static final class TestBlockAsDirWriter extends BlockAsDirWriter { - public TestBlockAsDirWriter( - final BlockRemover blockRemover, - final FileAttribute> filePerms, - final BlockNodeContext blockNodeContext) + private final class TestBlockAsLocalDirWriter extends BlockAsLocalDirWriter { + public TestBlockAsLocalDirWriter(final BlockRemover blockRemover, final BlockNodeContext blockNodeContext) throws IOException { - super(blockRemover, filePerms, blockNodeContext); + super(blockNodeContext, blockRemover, pathResolverMock); } @Override diff --git a/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java b/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java index eac41ae0f..195ad2edd 100644 --- a/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java +++ b/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java @@ -18,7 +18,11 @@ import static java.lang.System.Logger; import static java.lang.System.Logger.Level.INFO; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import com.hedera.block.server.persistence.storage.path.BlockAsLocalDirPathResolver; import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.stream.BlockProof; import com.hedera.hapi.block.stream.input.EventHeader; @@ -26,15 +30,17 @@ import com.hedera.hapi.node.base.SemanticVersion; import com.hedera.hapi.platform.event.EventCore; import com.hedera.pbj.runtime.io.buffer.Bytes; +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Objects; public final class PersistTestUtils { - private static final Logger LOGGER = System.getLogger(PersistTestUtils.class.getName()); + public static final String PERSISTENCE_STORAGE_LIVE_ROOT_PATH_KEY = "persistence.storage.liveRootPath"; private PersistTestUtils() {} @@ -105,4 +111,48 @@ public static byte[] reverseByteArray(byte[] input) { return reversed; } + + /** + * This method mocks and trains a {@link BlockAsLocalDirPathResolver}. It + * requires a path to the live root test directory to use as base (usually + * it would be a temp dir). The mock is trained to return the resolved path + * to a block by a given block number, using the live root test directory as + * a base. The mock captures anyLong input so it is dynamic in that sense. + * + * @param liveRootTestPath path to the live root test directory + * @return a trained mock that will return the resolved path to a block by a + * given block number + */ + public static BlockAsLocalDirPathResolver getTrainedBlockAsLocalDirPathResolver( + @NonNull final Path liveRootTestPath) { + return doTrainResolver(mock(BlockAsLocalDirPathResolver.class), liveRootTestPath); + } + + /** + * This method trains a {@link BlockAsLocalDirPathResolver} to return the + * resolved path to a block by a given block number, using the live root + * test directory as a base. The resolver is trained to return the resolved + * path to a block by a given block number, using the live root test + * directory as a base. The mock captures anyLong input so it is dynamic + * in that sense. + * + * @param resolverToTrain the resolver to train + * @param liveRootTestPath path to the live root test directory + * @return the trained resolver + */ + public static BlockAsLocalDirPathResolver trainAndReturnBlockAsLocalDirPathResolver( + @NonNull final BlockAsLocalDirPathResolver resolverToTrain, @NonNull final Path liveRootTestPath) { + return doTrainResolver(resolverToTrain, liveRootTestPath); + } + + private static BlockAsLocalDirPathResolver doTrainResolver( + @NonNull final BlockAsLocalDirPathResolver resolverToTrain, @NonNull final Path liveRootTestPath) { + Objects.requireNonNull(resolverToTrain); + Objects.requireNonNull(liveRootTestPath); + when(resolverToTrain.resolvePathToBlock(anyLong())).thenAnswer(invocation -> { + final long blockNumber = invocation.getArgument(0); + return Path.of(liveRootTestPath.resolve(Long.toString(blockNumber)).toString()); + }); + return resolverToTrain; + } } diff --git a/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java b/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java index 5253f9da2..b42e487d4 100644 --- a/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java +++ b/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java @@ -28,26 +28,12 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; import java.util.Collections; import java.util.Map; -import java.util.Set; public class TestConfigUtil { public static final String CONSUMER_TIMEOUT_THRESHOLD_KEY = "consumer.timeoutThresholdMillis"; public static final String MEDIATOR_RING_BUFFER_SIZE_KEY = "mediator.ringBufferSize"; - public static final FileAttribute> DEFAULT_TEST_FOLDER_PERMISSIONS = - PosixFilePermissions.asFileAttribute(Set.of( - PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, - PosixFilePermission.OWNER_EXECUTE, - PosixFilePermission.GROUP_READ, - PosixFilePermission.GROUP_EXECUTE, - PosixFilePermission.OTHERS_READ, - PosixFilePermission.OTHERS_EXECUTE)); - private static final String TEST_APP_PROPERTIES_FILE = "app.properties"; private TestConfigUtil() {} diff --git a/server/src/test/resources/app.properties b/server/src/test/resources/app.properties index f1ea37129..01e942648 100644 --- a/server/src/test/resources/app.properties +++ b/server/src/test/resources/app.properties @@ -2,3 +2,5 @@ prometheus.endpointEnabled=false mediator.ringBufferSize=1024 notifier.ringBufferSize=1024 +# @todo(#372) - default persistence type should be BLOCK_AS_LOCAL_FILE +persistence.storage.type=BLOCK_AS_LOCAL_DIRECTORY diff --git a/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java b/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java index 26bf66fe5..324916d20 100644 --- a/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java +++ b/suites/src/main/java/com/hedera/block/suites/persistence/positive/PositiveDataPersistenceTests.java @@ -36,9 +36,8 @@ */ @DisplayName("Positive Data Persistence Tests") public class PositiveDataPersistenceTests extends BaseSuite { - private final String[] GET_BLOCKS_COMMAND = new String[] {"ls", "data", "-1"}; - - private BlockStreamSimulatorApp blockStreamSimulatorApp; + // @todo(#371) - the default life/archive root path must be absolute starting from /opt + private final String[] GET_BLOCKS_COMMAND = new String[] {"ls", "hashgraph/blocknode/data/live", "-1"}; private Future simulatorThread; @@ -66,7 +65,7 @@ public void verifyBlockDataSavedInCorrectDirectory() throws InterruptedException String savedBlocksFolderBefore = getContainerCommandResult(GET_BLOCKS_COMMAND); int savedBlocksCountBefore = getSavedBlocksCount(savedBlocksFolderBefore); - blockStreamSimulatorApp = createBlockSimulator(); + final BlockStreamSimulatorApp blockStreamSimulatorApp = createBlockSimulator(); simulatorThread = startSimulatorInThread(blockStreamSimulatorApp); Thread.sleep(5000); blockStreamSimulatorApp.stop();