diff --git a/CHANGELOG.md b/CHANGELOG.md index 0085d5aae..cd3cacf17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - ![API] Add confidentiality property to WebSource [#518](https://github.com/nbbrd/sdmx-dl/issues/518) - ![API] Add missing properties methods in extensions points [#781](https://github.com/nbbrd/sdmx-dl/issues/781) +- ![CLI] Add check sources command [#515](https://github.com/nbbrd/sdmx-dl/issues/515) ### Changed diff --git a/sdmx-dl-api/src/main/java/sdmxdl/web/SdmxWebManager.java b/sdmx-dl-api/src/main/java/sdmxdl/web/SdmxWebManager.java index 627c73669..e06d3b607 100644 --- a/sdmx-dl-api/src/main/java/sdmxdl/web/SdmxWebManager.java +++ b/sdmx-dl-api/src/main/java/sdmxdl/web/SdmxWebManager.java @@ -70,45 +70,60 @@ public class SdmxWebManager extends SdmxManager { } @lombok.Singular - @NonNull List drivers; + @NonNull + List drivers; @lombok.Singular - @NonNull List monitors; + @NonNull + List monitors; @lombok.Builder.Default - @NonNull Networking networking = Networking.getDefault(); + @NonNull + Networking networking = Networking.getDefault(); @lombok.Builder.Default - @NonNull WebCaching caching = WebCaching.noOp(); + @NonNull + WebCaching caching = WebCaching.noOp(); - @Nullable EventListener onEvent; + @Nullable + EventListener onEvent; - @Nullable ErrorListener onError; + @Nullable + ErrorListener onError; @lombok.Singular - @NonNull List persistences; + @NonNull + List persistences; @lombok.Singular - @NonNull List authenticators; + @NonNull + List authenticators; @lombok.Builder.Default - @NonNull Registry registry = Registry.noOp(); + @NonNull + Registry registry = Registry.noOp(); - @Nullable Consumer onRegistryEvent; + @Nullable + Consumer onRegistryEvent; - @Nullable BiConsumer onRegistryError; + @Nullable + BiConsumer onRegistryError; @lombok.Getter(lazy = true) - @NonNull List customSources = initLazyCustomSources(getRegistry(), getPersistences(), getOnRegistryEvent(), getOnRegistryError()); + @NonNull + List customSources = initLazyCustomSources(getRegistry(), getPersistences(), getOnRegistryEvent(), getOnRegistryError()); @lombok.Getter(lazy = true) - @NonNull List defaultSources = initLazyDefaultSources(getDrivers()); + @NonNull + List defaultSources = initLazyDefaultSources(getDrivers()); @lombok.Getter(lazy = true) - @NonNull SortedMap sources = initLazySourceMap(getCustomSources(), getDefaultSources()); + @NonNull + SortedMap sources = initLazySourceMap(getCustomSources(), getDefaultSources()); @lombok.Getter(lazy = true, value = AccessLevel.PRIVATE) - @NonNull WebContext context = initLazyContext(); + @NonNull + WebContext context = initLazyContext(); public @NonNull Connection getConnection(@NonNull String name, @NonNull Languages languages) throws IOException { WebSource source = lookupSource(name) @@ -122,8 +137,6 @@ public class SdmxWebManager extends SdmxManager { Driver driver = lookupDriverById(source.getDriver()) .orElseThrow(() -> new IOException("Failed to find a suitable driver for '" + source + "'")); - checkSourceProperties(source, driver); - return driver.connect(source, languages, getContext()); } @@ -147,20 +160,6 @@ public class SdmxWebManager extends SdmxManager { return monitor.getReport(source, getContext()); } - private void checkSourceProperties(WebSource source, Driver driver) { - if (onEvent != null) { - Collection expected = new ArrayList<>(); - expected.addAll(driver.getDriverProperties()); - expected.addAll(networking.getNetworkingProperties()); - expected.addAll(caching.getWebCachingProperties()); - Collection found = source.getProperties().keySet(); - String diff = found.stream().filter(item -> !expected.contains(item)).sorted().collect(Collectors.joining(",")); - if (!diff.isEmpty()) { - onEvent.accept(source, "WEB_MANAGER", "Unexpected properties [" + diff + "]"); - } - } - } - private Optional lookupSource(String name) { return Optional.ofNullable(getSources().get(name)); } diff --git a/sdmx-dl-api/src/test/java/sdmxdl/web/SdmxWebManagerTest.java b/sdmx-dl-api/src/test/java/sdmxdl/web/SdmxWebManagerTest.java index 9ef4ea022..f51a79719 100644 --- a/sdmx-dl-api/src/test/java/sdmxdl/web/SdmxWebManagerTest.java +++ b/sdmx-dl-api/src/test/java/sdmxdl/web/SdmxWebManagerTest.java @@ -29,9 +29,7 @@ import java.io.IOException; import java.util.AbstractMap; -import java.util.ArrayList; import java.util.EnumSet; -import java.util.List; import static org.assertj.core.api.Assertions.*; import static sdmxdl.Languages.ANY; @@ -239,35 +237,6 @@ public void testGetConnectionOfSource() { assertThatCode(() -> manager.getConnection(sampleSource.toBuilder().id("other").build(), ANY).close()).doesNotThrowAnyException(); } - @SuppressWarnings("EmptyTryBlock") - @Test - public void testInvalidSourceProperties() throws IOException { - List events = new ArrayList<>(); - - SdmxWebManager manager = SdmxWebManager - .builder() - .driver(sampleDriver) - .onEvent((source, marker, event) -> events.add(source.getId() + ":" + event)) - .build(); - - WebSource noProp = sampleSource.toBuilder().id("noProp").clearProperties().build(); - try (Connection ignored = manager.getConnection(noProp, ANY)) { - } - assertThat(events).isEmpty(); - - WebSource validProp = sampleSource.toBuilder().id("validProp").build(); - try (Connection ignored = manager.getConnection(validProp, ANY)) { - } - assertThat(events).isEmpty(); - - WebSource invalidProp = sampleSource.toBuilder().id("invalidProp").property("boom", "123").build(); - try (Connection ignored = manager.getConnection(invalidProp, ANY)) { - } - assertThat(events).singleElement(as(STRING)) - .contains(invalidProp.getId()) - .contains("boom"); - } - private final DataRepository sample = DataRepository.builder().name("repo").build(); private final WebSource sampleSource = WebSource .builder() diff --git a/sdmx-dl-cli/src/main/java/internal/sdmxdl/cli/Plugin.java b/sdmx-dl-cli/src/main/java/internal/sdmxdl/cli/Plugin.java new file mode 100644 index 000000000..b0cf12e14 --- /dev/null +++ b/sdmx-dl-cli/src/main/java/internal/sdmxdl/cli/Plugin.java @@ -0,0 +1,79 @@ +package internal.sdmxdl.cli; + +import lombok.NonNull; +import sdmxdl.ext.Persistence; +import sdmxdl.file.spi.FileCaching; +import sdmxdl.web.SdmxWebManager; +import sdmxdl.web.spi.*; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@lombok.Value +public class Plugin { + + public enum Type { + DRIVER, + AUTHENTICATOR, + MONITOR, + PERSISTENCE, + REGISTRY, + WEB_CACHING, + FILE_CACHING, + NETWORKING + } + + public static List allOf(SdmxWebManager manager) { + List result = new ArrayList<>(); + manager.getDrivers().stream().map(Plugin::of).forEach(result::add); + manager.getAuthenticators().stream().map(Plugin::of).forEach(result::add); + manager.getMonitors().stream().map(Plugin::of).forEach(result::add); + manager.getPersistences().stream().map(Plugin::of).forEach(result::add); + result.add(Plugin.of(manager.getRegistry())); + result.add(Plugin.of(manager.getCaching())); + result.add(Plugin.of(manager.getNetworking())); + return result; + } + + static Plugin of(Driver o) { + return new Plugin(Type.DRIVER, o.getDriverId(), o.getDriverProperties()); + } + + static Plugin of(Authenticator o) { + return new Plugin(Type.AUTHENTICATOR, o.getAuthenticatorId(), o.getAuthenticatorProperties()); + } + + static Plugin of(Monitor o) { + return new Plugin(Type.MONITOR, o.getMonitorId(), o.getMonitorProperties()); + } + + static Plugin of(Persistence o) { + return new Plugin(Type.PERSISTENCE, o.getPersistenceId(), o.getPersistenceProperties()); + } + + static Plugin of(Registry o) { + return new Plugin(Type.REGISTRY, o.getRegistryId(), o.getRegistryProperties()); + } + + static Plugin of(WebCaching o) { + return new Plugin(Type.WEB_CACHING, o.getWebCachingId(), o.getWebCachingProperties()); + } + + static Plugin of(FileCaching o) { + return new Plugin(Type.FILE_CACHING, o.getFileCachingId(), o.getFileCachingProperties()); + } + + static Plugin of(Networking o) { + return new Plugin(Type.NETWORKING, o.getNetworkingId(), o.getNetworkingProperties()); + } + + @NonNull + Type type; + + @NonNull + String id; + + @NonNull + Collection Properties; +} diff --git a/sdmx-dl-cli/src/main/java/sdmxdl/cli/CheckCommand.java b/sdmx-dl-cli/src/main/java/sdmxdl/cli/CheckCommand.java index 98c1711cb..d6f1cfd65 100644 --- a/sdmx-dl-cli/src/main/java/sdmxdl/cli/CheckCommand.java +++ b/sdmx-dl-cli/src/main/java/sdmxdl/cli/CheckCommand.java @@ -30,6 +30,7 @@ CheckStatusCommand.class, CheckAccessCommand.class, CheckConfigCommand.class, + CheckSourcesCommand.class, CheckRulesCommand.class } ) diff --git a/sdmx-dl-cli/src/main/java/sdmxdl/cli/CheckSourcesCommand.java b/sdmx-dl-cli/src/main/java/sdmxdl/cli/CheckSourcesCommand.java new file mode 100644 index 000000000..b76340d2e --- /dev/null +++ b/sdmx-dl-cli/src/main/java/sdmxdl/cli/CheckSourcesCommand.java @@ -0,0 +1,96 @@ +/* + * Copyright 2018 National Bank of Belgium + * + * Licensed under the EUPL, Version 1.1 or - as soon they will be approved + * by the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. + * You may obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Licence for the specific language governing permissions and + * limitations under the Licence. + */ +package sdmxdl.cli; + +import internal.sdmxdl.cli.Plugin; +import internal.sdmxdl.cli.WebSourcesOptions; +import internal.sdmxdl.cli.ext.CsvTable; +import internal.sdmxdl.cli.ext.RFC4180OutputOptions; +import picocli.CommandLine; +import sdmxdl.web.SdmxWebManager; +import sdmxdl.web.WebSource; +import sdmxdl.web.spi.Driver; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; + +/** + * @author Philippe Charles + */ +@CommandLine.Command(name = "sources") +@SuppressWarnings("FieldMayBeFinal") +public final class CheckSourcesCommand implements Callable { + + @CommandLine.Mixin + private WebSourcesOptions web; + + @CommandLine.Mixin + private final RFC4180OutputOptions csv = new RFC4180OutputOptions(); + + @Override + public Void call() throws Exception { + getTable().write(csv, getRows()); + return null; + } + + private CsvTable getTable() { + return CsvTable + .builderOf(SourceIssue.class) + .columnOf("ID", SourceIssue::getId) + .columnOf("Issue", SourceIssue::getIssue) + .build(); + } + + private Stream getRows() throws IOException { + SdmxWebManager manager = web.loadManager(); + Stream sources = web.isAllSources() ? WebSourcesOptions.getAllSourceNames(manager) : web.getSources().stream(); + List plugins = Plugin.allOf(manager); + return web.applyParallel(sources).map(source -> check(source, manager, plugins)); + } + + private SourceIssue check(String sourceID, SdmxWebManager manager, List plugins) { + WebSource webSource = manager.getSources().get(sourceID); + if (webSource == null) { + return new SourceIssue(sourceID, "Source not found"); + } + Optional driver = manager.getDrivers().stream().filter(o -> o.getDriverId().equals(webSource.getDriver())).findFirst(); + if (!driver.isPresent()) { + return new SourceIssue(sourceID, "Driver not found"); + } + Collection expected = plugins.stream() + .filter(plugin -> !plugin.getType().equals(Plugin.Type.DRIVER) || plugin.getId().equals(driver.get().getDriverId())) + .map(Plugin::getProperties) + .flatMap(Collection::stream) + .collect(toList()); + Collection found = webSource.getProperties().keySet(); + String result = found.stream().filter(item -> !expected.contains(item)).sorted().collect(Collectors.joining(",")); + return new SourceIssue(sourceID, result.isEmpty() ? "No problem" : ("Unknown properties: " + result)); + } + + @lombok.Value + private static class SourceIssue { + String id; + String issue; + } +} diff --git a/sdmx-dl-cli/src/main/java/sdmxdl/cli/ListPluginsCommand.java b/sdmx-dl-cli/src/main/java/sdmxdl/cli/ListPluginsCommand.java index d6b480733..75dc85779 100644 --- a/sdmx-dl-cli/src/main/java/sdmxdl/cli/ListPluginsCommand.java +++ b/sdmx-dl-cli/src/main/java/sdmxdl/cli/ListPluginsCommand.java @@ -16,25 +16,18 @@ */ package sdmxdl.cli; +import internal.sdmxdl.cli.Plugin; import internal.sdmxdl.cli.WebOptions; import internal.sdmxdl.cli.ext.CsvTable; import internal.sdmxdl.cli.ext.CsvUtil; import internal.sdmxdl.cli.ext.RFC4180OutputOptions; -import lombok.NonNull; +import nbbrd.io.text.Formatter; import picocli.CommandLine; -import sdmxdl.ext.Persistence; -import sdmxdl.file.spi.FileCaching; -import sdmxdl.web.SdmxWebManager; -import sdmxdl.web.spi.*; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.concurrent.Callable; -import static java.util.Collections.emptyList; - /** * @author Philippe Charles */ @@ -56,67 +49,13 @@ public Void call() throws Exception { private CsvTable getTable() { return CsvTable .builderOf(Plugin.class) - .columnOf("Type", Plugin::getType) + .columnOf("Type", Plugin::getType, Formatter.onEnum()) .columnOf("Id", Plugin::getId) .columnOf("Properties", Plugin::getProperties, CsvUtil.DEFAULT_LIST_FORMATTER) .build(); } private List getRows() throws IOException { - SdmxWebManager manager = web.loadManager(); - List result = new ArrayList<>(); - manager.getDrivers().stream().map(Plugin::of).forEach(result::add); - manager.getAuthenticators().stream().map(Plugin::of).forEach(result::add); - manager.getMonitors().stream().map(Plugin::of).forEach(result::add); - manager.getPersistences().stream().map(Plugin::of).forEach(result::add); - result.add(Plugin.of(manager.getRegistry())); - result.add(Plugin.of(manager.getCaching())); - result.add(Plugin.of(manager.getNetworking())); - return result; - } - - @lombok.Value - private static class Plugin { - - static Plugin of(Driver o) { - return new Plugin("Driver", o.getDriverId(), o.getDriverProperties()); - } - - static Plugin of(Authenticator o) { - return new Plugin("Authenticator", o.getAuthenticatorId(), o.getAuthenticatorProperties()); - } - - static Plugin of(Monitor o) { - return new Plugin("Monitor", o.getMonitorId(), o.getMonitorProperties()); - } - - static Plugin of(Persistence o) { - return new Plugin("Persistence", o.getPersistenceId(), o.getPersistenceProperties()); - } - - static Plugin of(Registry o) { - return new Plugin("Registry", o.getRegistryId(), o.getRegistryProperties()); - } - - static Plugin of(WebCaching o) { - return new Plugin("WebCaching", o.getWebCachingId(), o.getWebCachingProperties()); - } - - static Plugin of(FileCaching o) { - return new Plugin("FileCaching", o.getFileCachingId(), o.getFileCachingProperties()); - } - - static Plugin of(Networking o) { - return new Plugin("Networking", o.getNetworkingId(), o.getNetworkingProperties()); - } - - @NonNull - String type; - - @NonNull - String id; - - @NonNull - Collection Properties; + return Plugin.allOf(web.loadManager()); } } diff --git a/sdmx-dl-cli/src/main/resources/sdmxdl/cli/Messages.properties b/sdmx-dl-cli/src/main/resources/sdmxdl/cli/Messages.properties index 3ed22e012..1bf614045 100644 --- a/sdmx-dl-cli/src/main/resources/sdmxdl/cli/Messages.properties +++ b/sdmx-dl-cli/src/main/resources/sdmxdl/cli/Messages.properties @@ -75,5 +75,6 @@ sdmx-dl.check.access.usage.description.1=Example: @|faint sdmx-dl check access E sdmx-dl.check.status.usage.description.0=Check service availability. sdmx-dl.check.status.usage.description.1=Example: @|faint sdmx-dl check status ECB|@ sdmx-dl.check.config.usage.description=Check sdmx-dl configuration. +sdmx-dl.check.sources.usage.description=Check sources configuration. # Setup sdmx-dl.setup.usage.description.0=Setup ${ROOT-COMMAND-NAME}. diff --git a/sdmx-dl-cli/src/test/java/sdmxdl/cli/CheckCommandTest.java b/sdmx-dl-cli/src/test/java/sdmxdl/cli/CheckCommandTest.java index 2d240d3ad..0e2c5e000 100644 --- a/sdmx-dl-cli/src/test/java/sdmxdl/cli/CheckCommandTest.java +++ b/sdmx-dl-cli/src/test/java/sdmxdl/cli/CheckCommandTest.java @@ -14,7 +14,7 @@ public void testHelp() { CommandWatcher watcher = CommandWatcher.on(cmd); assertThat(cmd.execute()).isEqualTo(CommandLine.ExitCode.OK); - assertThat(watcher.getOut()).isNotEmpty().contains("status"); + assertThat(watcher.getOut()).isNotEmpty().contains("status", "access", "config", "sources"); assertThat(watcher.getErr()).isEmpty(); } } diff --git a/sdmx-dl-cli/src/test/java/sdmxdl/cli/CheckSourcesCommandTest.java b/sdmx-dl-cli/src/test/java/sdmxdl/cli/CheckSourcesCommandTest.java new file mode 100644 index 000000000..daabcd656 --- /dev/null +++ b/sdmx-dl-cli/src/test/java/sdmxdl/cli/CheckSourcesCommandTest.java @@ -0,0 +1,48 @@ +package sdmxdl.cli; + +import _test.CommandWatcher; +import _test.FileSample; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import picocli.CommandLine; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.data.Index.atIndex; + +public class CheckSourcesCommandTest { + + @Test + public void testHelp() { + CommandLine cmd = new CommandLine(new CheckSourcesCommand()); + CommandWatcher watcher = CommandWatcher.on(cmd); + + assertThat(cmd.execute()).isEqualTo(CommandLine.ExitCode.USAGE); + assertThat(watcher.getOut()).isEmpty(); + assertThat(watcher.getErr()).isNotEmpty(); + } + + @Test + public void testContent(@TempDir Path temp) throws IOException { + CommandLine cmd = new CommandLine(new CheckSourcesCommand()); + CommandWatcher watcher = CommandWatcher.on(cmd); + + File src = FileSample.create(temp); + File out = temp.resolve("out.csv").toFile(); + + assertThat(cmd.execute("sample", "--no-log", "-s", src.getPath(), "-o", out.getPath())) + .isEqualTo(CommandLine.ExitCode.OK); + assertThat(watcher.getOut()) + .isEmpty(); + assertThat(watcher.getErr()) + .isEmpty(); + + assertThat(FileSample.readAll(out)) + .contains("ID,Issue", atIndex(0)) + .contains("sample,Driver not found", atIndex(1)) + .hasSize(2); + } +} diff --git a/sdmx-dl-cli/src/test/java/sdmxdl/cli/ListPluginsCommandTest.java b/sdmx-dl-cli/src/test/java/sdmxdl/cli/ListPluginsCommandTest.java index 76053c4ff..c59d061a6 100644 --- a/sdmx-dl-cli/src/test/java/sdmxdl/cli/ListPluginsCommandTest.java +++ b/sdmx-dl-cli/src/test/java/sdmxdl/cli/ListPluginsCommandTest.java @@ -44,7 +44,7 @@ public void testContent(@TempDir Path temp) throws IOException { assertThat(FileSample.readAll(out)) .contains("Type,Id,Properties", atIndex(0)) - .contains("Registry,RI_REGISTRY,sdmxdl.registry.sourcesFile") + .contains("REGISTRY,RI_REGISTRY,sdmxdl.registry.sourcesFile") .hasSizeGreaterThan(3); } }