From 17c96b9671d1379618d4b5b2286148baa3ac5d8b Mon Sep 17 00:00:00 2001 From: undx Date: Thu, 6 Jun 2024 17:06:40 +0200 Subject: [PATCH 01/15] feat(TCOMP-2475): default undefined version to 1 --- .../design/extension/repository/RepositoryModelBuilder.java | 2 +- .../runtime/manager/reflect/MigrationHandlerFactory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java b/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java index 34398cee2dc46..a117b25b90fba 100644 --- a/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java +++ b/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java @@ -141,7 +141,7 @@ private Config createConfig(final String plugin, final ComponentManager.AllServi .findMigrationHandler(() -> singletonList(c.getMeta()), clazz, services)); } } else { - c.setVersion(-1); + c.setVersion(1); } } if (c.getMigrationHandler() == null) { diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java index 6bab1a9a7007a..5cbea3981f237 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java @@ -74,7 +74,7 @@ private MigrationHandler createHandler(final List parameterMetas, return (Function, Map>) map -> buildMigrationFunction(p, handler, p.getPath(), map, - ofNullable(jType.getAnnotation(Version.class)).map(Version::value).orElse(-1)); + ofNullable(jType.getAnnotation(Version.class)).map(Version::value).orElse(1)); }) .filter(Objects::nonNull) .reduce(NO_MIGRATION, From bd915288f8ed8b271f90b6ff567670801ea7906d Mon Sep 17 00:00:00 2001 From: Axel Catoire Date: Tue, 11 Jun 2024 17:27:05 +0200 Subject: [PATCH 02/15] feat(TCOMP-2475): Improve API test https://jira.talendforge.org/browse/TCOMP-2475 --- .../tck_component_migrate_api_test.json | 150 +++++++++++++----- 1 file changed, 112 insertions(+), 38 deletions(-) diff --git a/talend-component-maven-plugin/src/it/web/test/tck-component-migrate-api-test/tck_component_migrate_api_test.json b/talend-component-maven-plugin/src/it/web/test/tck-component-migrate-api-test/tck_component_migrate_api_test.json index d8b90415946de..a8dff73ddd91d 100644 --- a/talend-component-maven-plugin/src/it/web/test/tck-component-migrate-api-test/tck_component_migrate_api_test.json +++ b/talend-component-maven-plugin/src/it/web/test/tck-component-migrate-api-test/tck_component_migrate_api_test.json @@ -5,7 +5,7 @@ "entity": { "type": "Project", "description": "To run the test you need to run a component server. \nTesting documentation can be found [here](https://github.com/Talend/component-runtime/tree/master/talend-component-maven-plugin/src/it/web) \nApi documentation is [here](https://talend.github.io/component-runtime/main/latest/rest-openapi.html)", - "id": "752c0a55-71fa-4767-af35-92e5e502d4b3", + "id": "e3f5f654-6a1c-41ee-8e4e-9d784f31e67b", "name": "tck-component-migrate-api-test" }, "children": [ @@ -39,7 +39,7 @@ "path": "/api/v1/component/migrate/${\"maper_id\"}/4" }, "description": "Expected: 400", - "id": "88069cff-1af4-4c23-9961-4d9df05b872f", + "id": "2b4f4d3a-047f-448f-8dd9-3760fb10a257", "name": "component/migrate/id/configurationVersion - empty body", "headers": [ { @@ -87,9 +87,83 @@ "host": "${\"server-ip\"}:${\"server-port\"}", "path": "/api/v1/component/migrate/${\"mapper_id\"}/10" }, - "description": "We have an executed migration from 10 to 3 \nTODO https://jira.talendforge.org/browse/TCOMP-2359 \nApi documentation is [here](https://talend.github.io/component-runtime/main/latest/rest-openapi.html)", - "id": "b5913ed5-35c9-4ff6-b19c-c3f36ce75ff0", - "name": "component/migrate/id/configurationVersion - invalid migration version", + "description": "Migration from 3 to 10 is not possible because 3 doesn't know 10. \nIt is normal to have a non migrated response. \nApi documentation is [here](https://talend.github.io/component-runtime/main/latest/rest-openapi.html)", + "id": "750b1859-e2e4-4619-bee6-0a11bc00eb9a", + "name": "component/migrate/id/configurationVersion - impossible migration (higher version)", + "headers": [ + { + "enabled": true, + "name": "Content-Type", + "value": "application/json" + } + ], + "assertions": [ + { + "comparison": "Equals", + "subject": "ResponseStatus", + "path": "code", + "value": "200" + }, + { + "comparison": "DoesNotExist", + "subject": "ResponseJsonBody", + "path": "$.level", + "value": "EXTRA" + }, + { + "comparison": "DoesNotExist", + "subject": "ResponseJsonBody", + "path": "$.incomingVersion", + "value": "\"1\"" + }, + { + "comparison": "DoesNotExist", + "subject": "ResponseJsonBody", + "path": "$.currentVersion", + "value": "\"3\"" + }, + { + "comparison": "Equals", + "subject": "ResponseJsonBody", + "path": "$.data", + "value": "input data" + } + ] + } + }, + { + "entity": { + "type": "Request", + "method": { + "requestBody": true, + "link": "http://tools.ietf.org/html/rfc7231#section-4.3.3", + "name": "POST" + }, + "body": { + "formBody": { + "overrideContentType": true, + "encoding": "application/x-www-form-urlencoded", + "items": [] + }, + "bodyType": "Text", + "textBodyEditorHeight": 150, + "textBody": "{\n \"data\": \"input data\"\n}\n" + }, + "uri": { + "query": { + "delimiter": "&", + "items": [] + }, + "scheme": { + "name": "http", + "version": "V11" + }, + "host": "${\"server-ip\"}:${\"server-port\"}", + "path": "/api/v1/component/migrate/${\"mapper_id\"}/3" + }, + "description": "Migration from 2 to 2 is not possible. \nIt is normal to have a non migrated response. \nApi documentation is [here](https://talend.github.io/component-runtime/main/latest/rest-openapi.html)", + "id": "cdaf9065-68b4-4bca-b95d-746584e1b934", + "name": "component/migrate/id/configurationVersion - impossible migration (same version)", "headers": [ { "enabled": true, @@ -162,7 +236,7 @@ "path": "/api/v1/component/migrate/${\"mapper_id\"}/2" }, "description": "\nApi documentation is [here](https://talend.github.io/component-runtime/main/latest/rest-openapi.html)", - "id": "cbc2af93-2693-4d18-bb53-a9086154d5b2", + "id": "b346f36c-dc84-4fb2-93ef-7040ee5ac54e", "name": "component/migrate/id/configurationVersion - with demo mapper", "headers": [ { @@ -236,7 +310,7 @@ "path": "/api/v1/component/migrate/${\"output_id\"}/1" }, "description": "\nApi documentation is [here](https://talend.github.io/component-runtime/main/latest/rest-openapi.html)", - "id": "0801c25e-bcf4-4f84-82c8-7e2f887474c8", + "id": "d14ec015-5fcb-4bc6-92b4-f9abdb543574", "name": "component/migrate/id/configurationVersion - with demo output", "headers": [ { @@ -309,7 +383,7 @@ "path": "/api/v1/component/migrate/wrong_id/4" }, "description": "Expected: 400", - "id": "027f22b3-77b6-4742-9017-8f2ad7375db8", + "id": "bb737277-2a82-4e5d-b926-5d855e5d0007", "name": "component/migrate/id/configurationVersion - wrong id", "headers": [ { @@ -336,60 +410,60 @@ "id": "7cc306c5-b3f9-4f07-8476-7fa2db669034", "name": "component_runtime_ci", "variables": { - "56f6cb1b-e758-42d2-bfd2-b8b5464654b5": { - "name": "output_id", - "value": "c2FtcGxlLWNvbm5lY3RvciN0aGVfZmFtaWx5I1RoZU91dHB1dDE", + "ed8bbd5f-ff31-40c3-963c-fc18dbb6f72a": { + "name": "server-ip", + "value": "localhost", "enabled": true, - "createdAt": "2023-05-15T09:45:49.917Z", + "createdAt": "2023-07-13T13:32:37.182Z", "private": false }, - "f7314dca-1c04-496c-ad19-44a4c8009360": { - "name": "mapper_id", - "value": "c2FtcGxlLWNvbm5lY3RvciN0aGVfZmFtaWx5I1RoZU1hcHBlcjE", + "28ea0f59-b89b-474a-9f77-6f3d73dd9046": { + "name": "datastore_id", + "value": "c2FtcGxlLWNvbm5lY3RvciN0aGVfZmFtaWx5I2RhdGFzdG9yZSNUaGVDb25uZWN0aW9u", "enabled": true, - "createdAt": "2023-05-15T09:45:49.917Z", + "createdAt": "2023-07-13T13:32:37.182Z", "private": false }, - "f0e9a692-6866-4c95-8b76-5f5d363e3ed1": { - "name": "dataset_id", - "value": "c2FtcGxlLWNvbm5lY3RvciN0aGVfZmFtaWx5I2RhdGFzZXQjVGhlRGF0YXNldA", + "d2298e59-d08d-48af-bb6e-c52e7810c209": { + "name": "server-port", + "value": "8081", "enabled": true, - "createdAt": "2023-05-15T09:45:49.917Z", + "createdAt": "2023-07-13T13:32:37.182Z", "private": false }, - "1266d066-8fd3-47f6-ad74-260e39b9d472": { - "name": "server-ip", - "value": "localhost", + "4fde58c1-0bef-461c-b576-a21289ccdbe1": { + "name": "mapper_id", + "value": "c2FtcGxlLWNvbm5lY3RvciN0aGVfZmFtaWx5I1RoZU1hcHBlcjE", "enabled": true, - "createdAt": "2023-05-15T09:45:49.917Z", + "createdAt": "2023-07-13T13:32:37.182Z", "private": false }, - "f05e4bab-fee2-4637-869f-bb228225d2ea": { - "name": "server-port", - "value": "8081", + "52dc8012-b6f2-41d3-8e87-549e41251792": { + "name": "httpbin_addr", + "value": "tal-rd22.talend.lan:8084", "enabled": true, - "createdAt": "2023-05-15T09:45:49.917Z", + "createdAt": "2023-07-13T13:32:37.182Z", "private": false }, - "ec6e694a-f4a6-4558-a788-eb3e36ee893e": { - "name": "datastore_id", - "value": "c2FtcGxlLWNvbm5lY3RvciN0aGVfZmFtaWx5I2RhdGFzdG9yZSNUaGVDb25uZWN0aW9u", + "6205aaff-1547-4dcc-bcb5-188fb9d26b51": { + "name": "dataset_id", + "value": "c2FtcGxlLWNvbm5lY3RvciN0aGVfZmFtaWx5I2RhdGFzZXQjVGhlRGF0YXNldA", "enabled": true, - "createdAt": "2023-05-15T09:45:49.917Z", + "createdAt": "2023-07-13T13:32:37.182Z", "private": false }, - "c803040f-9503-4201-a576-aee684b93510": { + "851c35ce-8117-4f11-adaa-9fe7823cb869": { "name": "family_id", "value": "c2FtcGxlLWNvbm5lY3RvciN0aGVfZmFtaWx5", "enabled": true, - "createdAt": "2023-05-15T09:45:49.917Z", + "createdAt": "2023-07-13T13:32:37.182Z", "private": false }, - "838469ba-346a-4f73-b769-031f529306f0": { - "name": "httpbin_addr", - "value": "tal-rd22.talend.lan:8084", + "442413a8-bd29-40db-8f74-5f2d6ebb6509": { + "name": "output_id", + "value": "c2FtcGxlLWNvbm5lY3RvciN0aGVfZmFtaWx5I1RoZU91dHB1dDE", "enabled": true, - "createdAt": "2023-05-24T07:29:46.274Z", + "createdAt": "2023-07-13T13:32:37.182Z", "private": false } } From 46c6681f07b86b080ef0add2482ee9e17a6a5265 Mon Sep 17 00:00:00 2001 From: undx Date: Fri, 7 Jun 2024 10:30:54 +0200 Subject: [PATCH 03/15] feat(TCOMP-2475): restore previous state (-1) for reproducing --- .../design/extension/repository/RepositoryModelBuilder.java | 2 +- .../runtime/manager/reflect/MigrationHandlerFactory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java b/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java index a117b25b90fba..34398cee2dc46 100644 --- a/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java +++ b/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java @@ -141,7 +141,7 @@ private Config createConfig(final String plugin, final ComponentManager.AllServi .findMigrationHandler(() -> singletonList(c.getMeta()), clazz, services)); } } else { - c.setVersion(1); + c.setVersion(-1); } } if (c.getMigrationHandler() == null) { diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java index 5cbea3981f237..6bab1a9a7007a 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java @@ -74,7 +74,7 @@ private MigrationHandler createHandler(final List parameterMetas, return (Function, Map>) map -> buildMigrationFunction(p, handler, p.getPath(), map, - ofNullable(jType.getAnnotation(Version.class)).map(Version::value).orElse(1)); + ofNullable(jType.getAnnotation(Version.class)).map(Version::value).orElse(-1)); }) .filter(Objects::nonNull) .reduce(NO_MIGRATION, From 7ec34031d5c20ba4994babdb052882077fe572fc Mon Sep 17 00:00:00 2001 From: undx Date: Fri, 7 Jun 2024 10:31:14 +0200 Subject: [PATCH 04/15] feat(TCOMP-2475): add logging --- .../sdk/component/server/front/ComponentResourceImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java index 2178845937269..573427667c7e4 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java @@ -566,9 +566,14 @@ public Map migrate(final String id, final int version, final Map .status(Status.NOT_FOUND) .entity(new ErrorPayload(COMPONENT_MISSING, "Didn't find component " + id)) .build())); - if (version > comp.getVersion() || version == comp.getVersion()) { + if (version != comp.getVersion()) { + log.info("Version mismatch - component: {} VS configuration: {}.", comp.getVersion(), version); + } + if (version >= comp.getVersion()) { + log.info("Not migrating component."); return config; } + log.info("Migrating component from {} to {}.", comp.getVersion(), version); return ofNullable(componentDao.findById(id)) .orElseThrow(() -> new WebApplicationException(Response .status(Response.Status.NOT_FOUND) From 7e28e0132699c8b523073d791c89a7bcb3f315c8 Mon Sep 17 00:00:00 2001 From: undx Date: Fri, 7 Jun 2024 10:33:13 +0200 Subject: [PATCH 05/15] feat(TCOMP-2475): add migration tests case environment (draft) --- .../front/ComponentResourceImplTest.java | 25 ++- .../ConfigurationTypeResourceImplTest.java | 2 +- .../front/EnvironmentResourceImplTest.java | 6 +- .../component/server/front/MigrationTest.java | 180 ++++++++++++++++++ .../service/ComponentManagerServiceTest.java | 13 +- .../server/test/ComponentClient.java | 12 ++ .../component/server/test/InitTestInfra.java | 30 +++ .../migration/AbstractMigrationHandler.java | 81 ++++++++ .../test/migration/MigrationDataSet.java | 38 ++++ .../test/migration/MigrationDataStore.java | 36 ++++ .../server/test/migration/MigrationInput.java | 42 ++++ .../server/test/migration/package-info.java | 18 ++ 12 files changed, 464 insertions(+), 19 deletions(-) create mode 100644 component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java create mode 100644 component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/AbstractMigrationHandler.java create mode 100644 component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationDataSet.java create mode 100644 component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationDataStore.java create mode 100644 component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationInput.java create mode 100644 component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/package-info.java diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java index 367466e900b53..a972bfc8bb1c8 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java @@ -328,7 +328,7 @@ void getIconIndex() { .accept(APPLICATION_SVG_XML_TYPE) .get(); assertNotNull(icons); - assertEquals(406, icons.getStatus()); + assertEquals(406, icons.getStatus()); // inexistant theme (no fallback) icons = base.path("component/icon/index") .queryParam("theme", "dak") @@ -357,7 +357,8 @@ void getIconIndex() { assertTrue(content.contains( "data-connector=\"standalone\" data-family=\"chain\" data-theme=\"light\" data-type=\"connector\" id=\"myicon-light\"")); assertTrue( - content.contains("data-family=\"file\" data-theme=\"light\" data-type=\"family\" id=\"file-family-light\"")); + content.contains( + "data-family=\"file\" data-theme=\"light\" data-type=\"family\" id=\"file-family-light\"")); // light theme content = base.path("component/icon/index") .queryParam("theme", "light") @@ -371,7 +372,8 @@ void getIconIndex() { assertTrue(content.contains( "data-connector=\"standalone\" data-family=\"chain\" data-theme=\"light\" data-type=\"connector\" id=\"myicon-light\"")); assertTrue( - content.contains("data-family=\"file\" data-theme=\"light\" data-type=\"family\" id=\"file-family-light\"")); + content.contains( + "data-family=\"file\" data-theme=\"light\" data-type=\"family\" id=\"file-family-light\"")); // dark theme content = base.path("component/icon/index") @@ -384,9 +386,11 @@ void getIconIndex() { content.startsWith( "")); assertTrue(content - .contains("data-connector=\"input\" data-family=\"jdbc\" data-theme=\"dark\" data-type=\"connector\" id=\"db-input-dark\"")); + .contains( + "data-connector=\"input\" data-family=\"jdbc\" data-theme=\"dark\" data-type=\"connector\" id=\"db-input-dark\"")); assertTrue(content - .contains("data-connector=\"output\" data-family=\"jdbc\" data-theme=\"dark\" data-type=\"connector\" id=\"db-input-dark\"")); + .contains( + "data-connector=\"output\" data-family=\"jdbc\" data-theme=\"dark\" data-type=\"connector\" id=\"db-input-dark\"")); // theme = all content = base.path("component/icon/index") .queryParam("theme", "all") @@ -399,13 +403,16 @@ void getIconIndex() { content.startsWith( "")); assertTrue(content - .contains("data-connector=\"input\" data-family=\"jdbc\" data-theme=\"dark\" data-type=\"connector\" id=\"db-input-dark\"")); + .contains( + "data-connector=\"input\" data-family=\"jdbc\" data-theme=\"dark\" data-type=\"connector\" id=\"db-input-dark\"")); assertTrue(content - .contains("data-connector=\"output\" data-family=\"jdbc\" data-theme=\"dark\" data-type=\"connector\" id=\"db-input-dark\"")); + .contains( + "data-connector=\"output\" data-family=\"jdbc\" data-theme=\"dark\" data-type=\"connector\" id=\"db-input-dark\"")); assertTrue(content.contains( "data-connector=\"standalone\" data-family=\"chain\" data-theme=\"light\" data-type=\"connector\" id=\"myicon-light\"")); assertTrue( - content.contains("data-family=\"file\" data-theme=\"dark\" data-type=\"family\" id=\"file-family-dark\"")); + content.contains( + "data-family=\"file\" data-theme=\"dark\" data-type=\"family\" id=\"file-family-dark\"")); } @Test @@ -709,7 +716,7 @@ private void assertComponent(final String plugin, final String family, final Str } private void assertIndex(final ComponentIndices index) { - assertEquals(12, index.getComponents().size()); + assertEquals(13, index.getComponents().size()); final List list = new ArrayList<>(index.getComponents()); list.sort(Comparator.comparing(o -> o.getId().getFamily() + "#" + o.getId().getName())); diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java index 003b5c3dbb3ce..32b362fbfaea7 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java @@ -184,7 +184,7 @@ void migrateOver8196DefaultByteBuffer() { } private void assertIndex(final ConfigTypeNodes index) { - assertEquals(5, index.getNodes().size()); + assertEquals(8, index.getNodes().size()); index.getNodes().keySet().forEach(Assertions::assertNotNull); // assert no null ids // assert there is at least one parent node assertTrue(index.getNodes().values().stream().anyMatch(n -> n.getParentId() == null)); diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/EnvironmentResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/EnvironmentResourceImplTest.java index 521284547d4d7..c3e87f8dcc047 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/EnvironmentResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/EnvironmentResourceImplTest.java @@ -24,7 +24,6 @@ import java.util.stream.Stream; import javax.inject.Inject; -import javax.json.JsonObject; import javax.ws.rs.client.WebTarget; import org.apache.meecrowave.junit5.MonoMeecrowaveConfig; @@ -49,8 +48,9 @@ void environment() { assertTrue(environment.getLastUpdated().compareTo(new Date(0)) > 0); final Connectors connectors = environment.getConnectors(); assertTrue(("1.2.3").equals(connectors.getVersion()) || ("1.26.0-SNAPSHOT").equals(connectors.getVersion())); - assertEquals("3a507eb7e52c9acd14c247d62bffecdee6493fc08f9cf69f65b941a64fcbf179", connectors.getPluginsHash()); + assertEquals("4960c7dbe95b9df086f06ee6057cc57dd3c3d152be25ee71d964db00e6adbd52", connectors.getPluginsHash()); assertEquals(Arrays.asList("another-test-component", "collection-of-object", "component-with-user-jars", - "file-component", "jdbc-component", "the-test-component"), connectors.getPluginsList()); + "file-component", "jdbc-component", "migration-component", "the-test-component"), + connectors.getPluginsList()); } } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java new file mode 100644 index 0000000000000..62d6d57f1c121 --- /dev/null +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java @@ -0,0 +1,180 @@ +/** + * Copyright (C) 2006-2024 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.server.front; + +import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.HashMap; +import java.util.Map; + +import javax.inject.Inject; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; + +import org.apache.meecrowave.junit5.MonoMeecrowaveConfig; +import org.junit.jupiter.api.Test; +import org.talend.sdk.component.server.front.model.ConfigTypeNodes; +import org.talend.sdk.component.server.test.ComponentClient; +import org.talend.sdk.component.server.test.migration.MigrationDataSet; +import org.talend.sdk.component.server.test.migration.MigrationDataStore; +import org.talend.sdk.component.server.test.migration.MigrationInput; + +@MonoMeecrowaveConfig +public class MigrationTest { + + @Inject + private ComponentClient client; + + @Inject + private WebTarget base; + + private final String dataStoreId = "bWlncmF0aW9uLWNvbXBvbmVudCNtaWdyYXRpb24jZGF0YXN0b3JlI2RhdGFzdG9yZQ"; + + private final String dataSetId = "bWlncmF0aW9uLWNvbXBvbmVudCNtaWdyYXRpb24jZGF0YXNldCNkYXRhc2V0"; + + /** + * component tests + * registry version: 5 + */ + @Test + void migrateComponentLower() { + final int incomingVersion = 3; + Map conf = new HashMap<>(); + conf.put("configuration.inputKey", "keylevel0"); + final Map migrated = migrateComponent(getInputComponentId(), incomingVersion, conf); + assertEquals("INPUT", migrated.get("level")); + assertEquals(String.valueOf(MigrationInput.Version), migrated.get("currentVersion")); + assertEquals(String.valueOf(incomingVersion), migrated.get("incomingVersion")); + } + + @Test + void migrateComponentEqual() { + Map conf = new HashMap<>(); + conf.put("configuration.inputKey", "keylevel0"); + final Map migrated = migrateComponent(getInputComponentId(), 5, conf); + assertEquals(conf, migrated); + } + + @Test + void migrateComponentGreater() { + Map conf = new HashMap<>(); + conf.put("configuration.inputKey", "keylevel0"); + final Map migrated = migrateComponent(getInputComponentId(), 6, conf); + assertEquals(conf, migrated); + } + + /** + * dataset tests + * registry version: undefined. + */ + @Test + void migrateDataSetLower() { + final int incomingVersion = -1; + Map conf = new HashMap<>(); + conf.put("configuration.dataSetKey", "keylevel0"); + final Map migrated = migrateConfigurationType(getDataSetID(), incomingVersion, conf); + assertEquals("DATASET", migrated.get("configuration.level")); + assertEquals(String.valueOf(MigrationDataSet.Version), migrated.get("configuration.currentVersion")); + assertEquals(String.valueOf(incomingVersion), migrated.get("configuration.incomingVersion")); + } + + @Test + void migrateDataSetEqual() { + Map conf = new HashMap<>(); + conf.put("configuration.dataSetKey", "keylevel0"); + // Version value not set defaults to 1 + final Map migrated = migrateConfigurationType(getDataSetID(), 1, conf); + assertEquals(conf, migrated); + } + + @Test + void migrateDataSetGreater() { + Map conf = new HashMap<>(); + conf.put("configuration.dataSetKey", "keylevel0"); + final Map migrated = migrateConfigurationType(getDataSetID(), 3, conf); + assertEquals(conf, migrated); + } + + /** + * datastore tests + * registry version: 2 + */ + @Test + void migrateDataStoreLower() { + final int incomingVersion = 1; + Map conf = new HashMap<>(); + conf.put("configuration.dataStoreKey", "keylevel0"); + final Map migrated = migrateConfigurationType(getDataStoreID(), incomingVersion, conf); + assertEquals("DATASTORE", migrated.get("configuration.level")); + assertEquals(String.valueOf(MigrationDataStore.Version), migrated.get("configuration.currentVersion")); + assertEquals(String.valueOf(incomingVersion), migrated.get("configuration.incomingVersion")); + } + + @Test + void migrateDataStoreEqual() { + Map conf = new HashMap<>(); + conf.put("configuration.dataStoreKey", "keylevel0"); + final Map migrated = migrateConfigurationType(getDataStoreID(), 2, conf); + assertEquals(conf, migrated); + } + + @Test + void migrateDataStoreGreater() { + Map conf = new HashMap<>(); + conf.put("configuration.dataStoreKey", "keylevel0"); + final Map migrated = migrateConfigurationType(getDataStoreID(), 3, conf); + assertEquals(conf, migrated); + } + + /** + * end tests + */ + + private Map migrateComponent(final String component, final int version, + final Map config) { + return callMigrate("component", component, version, config); + } + + private Map migrateConfigurationType(final String configuration, final int version, + final Map config) { + return callMigrate("configurationtype", configuration, version, config); + } + + private Map callMigrate(final String endpoint, final String id, final int version, + final Map config) { + return base + .path(String.format("/%s/migrate/%s/%d", endpoint, id, version)) + .request(APPLICATION_JSON_TYPE) + .post(Entity.entity(config, APPLICATION_JSON_TYPE)) + .readEntity(Map.class); + } + + private String getInputComponentId() { + return client.getComponentId("migration", "Input"); + } + + private String getDataSetID() { + ConfigTypeNodes index = client.fetchConfigTypeNodes(); + return index.getNodes().get(dataSetId).getId(); + } + + private String getDataStoreID() { + ConfigTypeNodes index = client.fetchConfigTypeNodes(); + return index.getNodes().get(dataStoreId).getId(); + } + +} diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ComponentManagerServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ComponentManagerServiceTest.java index f235ced5e282e..3914b3c2fa06f 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ComponentManagerServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ComponentManagerServiceTest.java @@ -63,10 +63,11 @@ class ComponentManagerServiceTest { @Inject private ComponentActionDao componentActionDao; - public static final String PLUGINS_HASH = "3a507eb7e52c9acd14c247d62bffecdee6493fc08f9cf69f65b941a64fcbf179"; + public static final String PLUGINS_HASH = "4960c7dbe95b9df086f06ee6057cc57dd3c3d152be25ee71d964db00e6adbd52"; public static final List PLUGINS_LIST = Arrays.asList("another-test-component", "collection-of-object", - "component-with-user-jars", "file-component", "jdbc-component", "the-test-component"); + "component-with-user-jars", "file-component", "jdbc-component", "migration-component", + "the-test-component"); @Test void deployExistingPlugin() { @@ -144,9 +145,9 @@ void undeployNonExistingPlugin() { @Order(1) void checkPluginsNotReloaded() throws Exception { assertEquals("1.2.3", componentManagerService.getConnectors().getVersion()); - assertEquals(6, componentManagerService.manager().getContainer().findAll().stream().count()); + assertEquals(7, componentManagerService.manager().getContainer().findAll().stream().count()); Thread.sleep(6000); - assertEquals(6, componentManagerService.manager().getContainer().findAll().stream().count()); + assertEquals(7, componentManagerService.manager().getContainer().findAll().stream().count()); assertEquals(PLUGINS_HASH, componentManagerService.getConnectors().getPluginsHash()); assertEquals(PLUGINS_LIST, componentManagerService.getConnectors().getPluginsList()); } @@ -155,12 +156,12 @@ void checkPluginsNotReloaded() throws Exception { @Order(10) void checkPluginsReloaded() throws Exception { assertEquals("1.2.3", componentManagerService.getConnectors().getVersion()); - assertEquals(6, componentManagerService.manager().getContainer().findAll().stream().count()); + assertEquals(7, componentManagerService.manager().getContainer().findAll().stream().count()); assertEquals(PLUGINS_HASH, componentManagerService.getConnectors().getPluginsHash()); assertEquals(PLUGINS_LIST, componentManagerService.getConnectors().getPluginsList()); writeVersion("1.26.0-SNAPSHOT"); Thread.sleep(6000); - assertEquals(6, componentManagerService.manager().getContainer().findAll().stream().count()); + assertEquals(7, componentManagerService.manager().getContainer().findAll().stream().count()); final String gav = "org.talend.test1:the-test-component:jar:1.2.6:compile"; String pluginID = getPluginId(gav); assertNotNull(pluginID); diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/ComponentClient.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/ComponentClient.java index 70f6f85302eda..3dfdc7c6ca3ce 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/ComponentClient.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/ComponentClient.java @@ -22,6 +22,7 @@ import javax.ws.rs.client.WebTarget; import org.talend.sdk.component.server.front.model.ComponentIndices; +import org.talend.sdk.component.server.front.model.ConfigTypeNodes; @ApplicationScoped public class ComponentClient { @@ -38,6 +39,17 @@ public ComponentIndices fetchIndex() { .get(ComponentIndices.class); } + public ConfigTypeNodes fetchConfigTypeNodes() { + return base + .path("configurationtype/index") + .queryParam("lightPayload", false) + .queryParam("query", "") + .queryParam("lang", "en") + .request(APPLICATION_JSON_TYPE) + .header("Accept-Encoding", "gzip") + .get(ConfigTypeNodes.class); + } + public String getComponentId(final String family, final String component) { return fetchIndex() .getComponents() diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/InitTestInfra.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/InitTestInfra.java index 8178fbc16c5d7..33a941f87c8f1 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/InitTestInfra.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/InitTestInfra.java @@ -199,6 +199,12 @@ private String createM2(final String tempDir) { final String version = "0.0.1"; createComponent(m2, groupId, artifactId, version, generator::createCustomPlugin); } + { + final String groupId = "org.talend.test"; + final String artifactId = "migration-component"; + final String version = "0.0.1"; + createComponent(m2, groupId, artifactId, version, generator::createMigrationPlugin); + } if (Boolean.getBoolean("components.server.beam.active")) { final String groupId = System.getProperty("components.sample.beam.groupId"); final String artifactId = System.getProperty("components.sample.beam.artifactId"); @@ -386,6 +392,30 @@ private File createCustomPlugin(final File target) { }); } + public File createMigrationPlugin(final File target) { + return createRepackaging(target, "org/talend/sdk/component/server/test/migration", out -> { + try { + out.putNextEntry(new JarEntry("TALEND-INF/dependencies.txt")); + out.closeEntry(); + out.putNextEntry(new JarEntry("org/talend/test/generated/migration_component/Messages.properties")); + new Properties() { + + { + put("migration.datastore.migration._displayName", "JDBC DataStore"); + put("migration.dataset.migration._displayName", "JDBC DataSet"); + put("migration.Database/migration/Standard._category", "DB/Std/Yes"); + put("migration.parameters.user.custom._displayName", "My Custom Action"); + put("org.talend.test.generated.jdbc_component.JdbcService$I18n.read", "God save the queen"); + } + }.store(out, "i18n for the config types"); + + } catch (final IOException e) { + fail(e.getMessage()); + } + }); + } + + private File createRepackaging(final File target, final String sourcePackage, final Consumer custom) { try (final JarOutputStream outputStream = new JarOutputStream(new FileOutputStream(target))) { diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/AbstractMigrationHandler.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/AbstractMigrationHandler.java new file mode 100644 index 0000000000000..2e82becb39d19 --- /dev/null +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/AbstractMigrationHandler.java @@ -0,0 +1,81 @@ +/** + * Copyright (C) 2006-2024 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.server.test.migration; + +import java.util.HashMap; +import java.util.Map; + +import org.talend.sdk.component.api.component.MigrationHandler; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractMigrationHandler implements MigrationHandler { + + protected abstract String getLevel(); + + protected abstract int getCurrentVersion(); + + @Override + public Map migrate(int incomingVersion, Map incomingData) { + Map migrated = new HashMap<>(incomingData); + migrated.put("level", getLevel()); + migrated.put("incomingVersion", String.valueOf(incomingVersion)); + migrated.put("currentVersion", String.valueOf(getCurrentVersion())); + return migrated; + } + + public static class InputHandler extends AbstractMigrationHandler { + + @Override + protected String getLevel() { + return "INPUT"; + } + + @Override + protected int getCurrentVersion() { + return MigrationInput.Version; + } + + } + + public static class DataSetHandler extends AbstractMigrationHandler { + + @Override + protected String getLevel() { + return "DATASET"; + } + + @Override + protected int getCurrentVersion() { + return MigrationDataSet.Version; + } + + } + + public static class DataStoreHandler extends AbstractMigrationHandler { + + @Override + protected String getLevel() { + return "DATASTORE"; + } + + @Override + protected int getCurrentVersion() { + return MigrationDataStore.Version; + } + } +} diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationDataSet.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationDataSet.java new file mode 100644 index 0000000000000..7b5734972dbd1 --- /dev/null +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationDataSet.java @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2006-2024 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.server.test.migration; + +import java.io.Serializable; + +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.configuration.type.DataSet; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +@DataSet("dataset") +@Version(migrationHandler = AbstractMigrationHandler.DataSetHandler.class) +public class MigrationDataSet implements Serializable { + + public final static int Version = -1; + + @Option + private final MigrationDataStore dataStore; + + @Option + private final String dataSetKey; +} diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationDataStore.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationDataStore.java new file mode 100644 index 0000000000000..ea2367910e32e --- /dev/null +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationDataStore.java @@ -0,0 +1,36 @@ +/** + * Copyright (C) 2006-2024 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.server.test.migration; + +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.configuration.type.DataStore; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +@DataStore("datastore") +@Version(value = MigrationDataStore.Version, migrationHandler = AbstractMigrationHandler.DataStoreHandler.class) +public class MigrationDataStore { + + public static final int Version = 2; + + @Option + private final String dataStoreKey; + + @Option + private final int revision; +} diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationInput.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationInput.java new file mode 100644 index 0000000000000..65261a96441af --- /dev/null +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/MigrationInput.java @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2006-2024 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.talend.sdk.component.server.test.migration; + +import java.io.Serializable; + +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.input.Emitter; +import org.talend.sdk.component.api.input.Producer; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +@Emitter(name = "Input", family = "migration") +@Version(value = MigrationInput.Version, migrationHandler = AbstractMigrationHandler.InputHandler.class) +public class MigrationInput implements Serializable { + + public static final int Version = 5; + + @Option + private final MigrationDataSet dataSet; + + @Producer + public Object next() { + return null; + } + +} diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/package-info.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/package-info.java new file mode 100644 index 0000000000000..fd6286c9a5fea --- /dev/null +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/migration/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2006-2024 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + */ + +@Components(family = "migration", categories = "Testing/${family}") +package org.talend.sdk.component.server.test.migration; + +import org.talend.sdk.component.api.component.Components; From 1f6ce3888e2534a045704420bbb6be26c073bca1 Mon Sep 17 00:00:00 2001 From: undx Date: Wed, 19 Jun 2024 15:47:02 +0200 Subject: [PATCH 06/15] feat(TCOMP-2475): add logging informations --- .../sdk/component/server/front/ComponentResourceImpl.java | 8 +++----- .../server/front/ConfigurationTypeResourceImpl.java | 5 +++++ .../talend/sdk/component/server/test/InitTestInfra.java | 1 - 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java index 573427667c7e4..56751f92f2979 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java @@ -566,14 +566,12 @@ public Map migrate(final String id, final int version, final Map .status(Status.NOT_FOUND) .entity(new ErrorPayload(COMPONENT_MISSING, "Didn't find component " + id)) .build())); - if (version != comp.getVersion()) { - log.info("Version mismatch - component: {} VS configuration: {}.", comp.getVersion(), version); - } if (version >= comp.getVersion()) { - log.info("Not migrating component."); + log.debug("[Component#migrate] Skipping {}#{}.", comp.getParent().getName(), comp.getName()); return config; } - log.info("Migrating component from {} to {}.", comp.getVersion(), version); + log.info("[Component#migrate] {}#{} registry version {} - incoming: {}.", comp.getParent().getName(), + comp.getName(), comp.getVersion(), version); return ofNullable(componentDao.findById(id)) .orElseThrow(() -> new WebApplicationException(Response .status(Response.Status.NOT_FOUND) diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImpl.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImpl.java index 4b1da3a9a5fb3..1de854b9fe30d 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImpl.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImpl.java @@ -178,6 +178,11 @@ public Map migrate(final String id, final int version, final Map final String versionKey = configuration.getMeta().getPath() + ".__version"; final boolean addedVersion = configToMigrate.putIfAbsent(versionKey, Integer.toString(version)) == null; try { + if (configuration.getVersion() != version) { + log.info("[ConfigurationType#migrate] {}#{} registry: {} - incoming: {}.", + configuration.getKey().getFamily(), configuration.getKey().getConfigName(), + configuration.getVersion(), version); + } final Map migrated = configuration.getMigrationHandler().migrate(version, configToMigrate); if (addedVersion) { migrated.remove(versionKey); diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/InitTestInfra.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/InitTestInfra.java index 33a941f87c8f1..35c81fcccf620 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/InitTestInfra.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/InitTestInfra.java @@ -415,7 +415,6 @@ public File createMigrationPlugin(final File target) { }); } - private File createRepackaging(final File target, final String sourcePackage, final Consumer custom) { try (final JarOutputStream outputStream = new JarOutputStream(new FileOutputStream(target))) { From 77ccbfdfdc2c07f3fe1c2aaa906b13b264925f4d Mon Sep 17 00:00:00 2001 From: undx Date: Wed, 19 Jun 2024 16:37:51 +0200 Subject: [PATCH 07/15] feat(TCOMP-2475): default undefined version to 1 --- .../design/extension/repository/RepositoryModelBuilder.java | 2 +- .../runtime/manager/reflect/MigrationHandlerFactory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java b/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java index 34398cee2dc46..a117b25b90fba 100644 --- a/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java +++ b/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java @@ -141,7 +141,7 @@ private Config createConfig(final String plugin, final ComponentManager.AllServi .findMigrationHandler(() -> singletonList(c.getMeta()), clazz, services)); } } else { - c.setVersion(-1); + c.setVersion(1); } } if (c.getMigrationHandler() == null) { diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java index 6bab1a9a7007a..5cbea3981f237 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java @@ -74,7 +74,7 @@ private MigrationHandler createHandler(final List parameterMetas, return (Function, Map>) map -> buildMigrationFunction(p, handler, p.getPath(), map, - ofNullable(jType.getAnnotation(Version.class)).map(Version::value).orElse(-1)); + ofNullable(jType.getAnnotation(Version.class)).map(Version::value).orElse(1)); }) .filter(Objects::nonNull) .reduce(NO_MIGRATION, From 119fe7b3b2fed883394298cac949ee9e18e6b8c8 Mon Sep 17 00:00:00 2001 From: undx Date: Wed, 19 Jun 2024 17:00:21 +0200 Subject: [PATCH 08/15] feat(TCOMP-2475): sonar comment --- .../org/talend/sdk/component/server/front/MigrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java index 62d6d57f1c121..e791c6ff4f401 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java @@ -34,7 +34,7 @@ import org.talend.sdk.component.server.test.migration.MigrationInput; @MonoMeecrowaveConfig -public class MigrationTest { +class MigrationTest { @Inject private ComponentClient client; From 320d449b70374a3a2ca2156dcd7749d66ca6dd6c Mon Sep 17 00:00:00 2001 From: undx Date: Wed, 19 Jun 2024 18:31:55 +0200 Subject: [PATCH 09/15] feat(TCOMP-2475): skip migration from upper version --- .../component/server/front/ComponentResourceImpl.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java index 56751f92f2979..bb41b3102e406 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/front/ComponentResourceImpl.java @@ -566,12 +566,15 @@ public Map migrate(final String id, final int version, final Map .status(Status.NOT_FOUND) .entity(new ErrorPayload(COMPONENT_MISSING, "Didn't find component " + id)) .build())); - if (version >= comp.getVersion()) { - log.debug("[Component#migrate] Skipping {}#{}.", comp.getParent().getName(), comp.getName()); + if (version > comp.getVersion()) { + log.warn("[Component#migrate] Skipping {}#{} configuration migration due to incoming {} > registry {}.", + comp.getParent().getName(), comp.getName(), version, comp.getVersion()); return config; } - log.info("[Component#migrate] {}#{} registry version {} - incoming: {}.", comp.getParent().getName(), - comp.getName(), comp.getVersion(), version); + if (comp.getVersion() != version) { + log.info("[Component#migrate] {}#{} registry version {} - incoming: {}.", comp.getParent().getName(), + comp.getName(), comp.getVersion(), version); + } return ofNullable(componentDao.findById(id)) .orElseThrow(() -> new WebApplicationException(Response .status(Response.Status.NOT_FOUND) From cc61774ffc6bf1d22e162c6d5a9116d73a90e476 Mon Sep 17 00:00:00 2001 From: undx Date: Wed, 19 Jun 2024 18:46:36 +0200 Subject: [PATCH 10/15] feat(TCOMP-2475): adapt component migration test --- .../talend/sdk/component/server/front/MigrationTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java index e791c6ff4f401..8f08a47bc1755 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java @@ -65,8 +65,10 @@ void migrateComponentLower() { void migrateComponentEqual() { Map conf = new HashMap<>(); conf.put("configuration.inputKey", "keylevel0"); - final Map migrated = migrateComponent(getInputComponentId(), 5, conf); - assertEquals(conf, migrated); + final Map migrated = migrateComponent(getInputComponentId(), MigrationInput.Version, conf); + assertEquals("INPUT", migrated.get("level")); + assertEquals(String.valueOf(MigrationInput.Version), migrated.get("currentVersion")); + assertEquals(String.valueOf(MigrationInput.Version), migrated.get("incomingVersion")); } @Test @@ -128,7 +130,7 @@ void migrateDataStoreLower() { void migrateDataStoreEqual() { Map conf = new HashMap<>(); conf.put("configuration.dataStoreKey", "keylevel0"); - final Map migrated = migrateConfigurationType(getDataStoreID(), 2, conf); + final Map migrated = migrateConfigurationType(getDataStoreID(), MigrationDataStore.Version, conf); assertEquals(conf, migrated); } From 987393aed269ab5649eb7fd8ff0ff6eef72c4311 Mon Sep 17 00:00:00 2001 From: undx Date: Wed, 19 Jun 2024 18:50:47 +0200 Subject: [PATCH 11/15] feat(TCOMP-2475): adapt old component migration test --- .../component/server/front/ComponentResourceImplTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java index a972bfc8bb1c8..76293e72c455a 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java @@ -202,7 +202,7 @@ void noMigrateWithUpperVersion() { } @Test - void noMigrateWithEqualVersion() { + void migrateWithEqualVersion() { final Map migrated = base .path("component/migrate/{id}/{version}") .resolveTemplate("id", client.getJdbcId()) @@ -215,8 +215,8 @@ void noMigrateWithEqualVersion() { } }, APPLICATION_JSON_TYPE), new GenericType>() { }); - assertEquals(1, migrated.size()); - assertEquals(null, migrated.get("migrated")); + assertEquals(2, migrated.size()); + assertEquals("true", migrated.get("migrated")); } @Test From 1d7aed5d326d6c3afe2e7ddb8fe954f7c2beffa2 Mon Sep 17 00:00:00 2001 From: undx Date: Wed, 19 Jun 2024 18:52:42 +0200 Subject: [PATCH 12/15] feat(TCOMP-2475): chore format --- .../org/talend/sdk/component/server/front/MigrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java index 8f08a47bc1755..566d8b7975342 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/MigrationTest.java @@ -130,7 +130,8 @@ void migrateDataStoreLower() { void migrateDataStoreEqual() { Map conf = new HashMap<>(); conf.put("configuration.dataStoreKey", "keylevel0"); - final Map migrated = migrateConfigurationType(getDataStoreID(), MigrationDataStore.Version, conf); + final Map migrated = + migrateConfigurationType(getDataStoreID(), MigrationDataStore.Version, conf); assertEquals(conf, migrated); } From 08d60dee68356e3a59a0973bdd7ee942443e467d Mon Sep 17 00:00:00 2001 From: undx Date: Thu, 20 Jun 2024 15:58:17 +0200 Subject: [PATCH 13/15] feat(TCOMP-2475): restore initial defaults --- .../design/extension/repository/RepositoryModelBuilder.java | 2 +- .../runtime/manager/reflect/MigrationHandlerFactory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java b/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java index a117b25b90fba..34398cee2dc46 100644 --- a/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java +++ b/component-runtime-design-extension/src/main/java/org/talend/sdk/component/design/extension/repository/RepositoryModelBuilder.java @@ -141,7 +141,7 @@ private Config createConfig(final String plugin, final ComponentManager.AllServi .findMigrationHandler(() -> singletonList(c.getMeta()), clazz, services)); } } else { - c.setVersion(1); + c.setVersion(-1); } } if (c.getMigrationHandler() == null) { diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java index 5cbea3981f237..6bab1a9a7007a 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/MigrationHandlerFactory.java @@ -74,7 +74,7 @@ private MigrationHandler createHandler(final List parameterMetas, return (Function, Map>) map -> buildMigrationFunction(p, handler, p.getPath(), map, - ofNullable(jType.getAnnotation(Version.class)).map(Version::value).orElse(1)); + ofNullable(jType.getAnnotation(Version.class)).map(Version::value).orElse(-1)); }) .filter(Objects::nonNull) .reduce(NO_MIGRATION, From da0f96abbe1ad58750805b0b8dce93fbf1d35b41 Mon Sep 17 00:00:00 2001 From: undx Date: Thu, 20 Jun 2024 16:08:23 +0200 Subject: [PATCH 14/15] feat(TCOMP-2475): remove reverted pr test --- .../front/ComponentResourceImplTest.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java index 76293e72c455a..36c7e6ac0a22e 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ComponentResourceImplTest.java @@ -201,24 +201,6 @@ void noMigrateWithUpperVersion() { assertEquals(null, migrated.get("migrated")); } - @Test - void migrateWithEqualVersion() { - final Map migrated = base - .path("component/migrate/{id}/{version}") - .resolveTemplate("id", client.getJdbcId()) - .resolveTemplate("version", 2) - .request(APPLICATION_JSON_TYPE) - .post(entity(new HashMap() { - - { - put("going", "nowhere"); - } - }, APPLICATION_JSON_TYPE), new GenericType>() { - }); - assertEquals(2, migrated.size()); - assertEquals("true", migrated.get("migrated")); - } - @Test void searchIcon() { assertNotNull(base.path("component/icon/custom/{familyId}/{iconKey}") From 316b0395685c4453f4622a7bc319e8e09272a96a Mon Sep 17 00:00:00 2001 From: Axel Catoire Date: Wed, 26 Jun 2024 16:38:21 +0200 Subject: [PATCH 15/15] feat(TCOMP-2475): Deactivate equal API test https://jira.talendforge.org/browse/TCOMP-2475 --- .../tck_component_migrate_api_test.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/talend-component-maven-plugin/src/it/web/test/tck-component-migrate-api-test/tck_component_migrate_api_test.json b/talend-component-maven-plugin/src/it/web/test/tck-component-migrate-api-test/tck_component_migrate_api_test.json index a8dff73ddd91d..96449bfc679d1 100644 --- a/talend-component-maven-plugin/src/it/web/test/tck-component-migrate-api-test/tck_component_migrate_api_test.json +++ b/talend-component-maven-plugin/src/it/web/test/tck-component-migrate-api-test/tck_component_migrate_api_test.json @@ -161,9 +161,9 @@ "host": "${\"server-ip\"}:${\"server-port\"}", "path": "/api/v1/component/migrate/${\"mapper_id\"}/3" }, - "description": "Migration from 2 to 2 is not possible. \nIt is normal to have a non migrated response. \nApi documentation is [here](https://talend.github.io/component-runtime/main/latest/rest-openapi.html)", + "description": "Migration from 2 to 2 is not possible. \nIt is normal to have a non migrated response. \n\nFIXME This test is disabled waiting for https://jira.talendforge.org/browse/TCOMP-2754 \n\nApi documentation is [here](https://talend.github.io/component-runtime/main/latest/rest-openapi.html)", "id": "cdaf9065-68b4-4bca-b95d-746584e1b934", - "name": "component/migrate/id/configurationVersion - impossible migration (same version)", + "name": "component/migrate/id/configurationVersion - impossible migration (same version) FIXME", "headers": [ { "enabled": true, @@ -181,24 +181,28 @@ { "comparison": "DoesNotExist", "subject": "ResponseJsonBody", + "enabled": false, "path": "$.level", "value": "EXTRA" }, { "comparison": "DoesNotExist", "subject": "ResponseJsonBody", + "enabled": false, "path": "$.incomingVersion", "value": "\"1\"" }, { "comparison": "DoesNotExist", "subject": "ResponseJsonBody", + "enabled": false, "path": "$.currentVersion", "value": "\"3\"" }, { "comparison": "Equals", "subject": "ResponseJsonBody", + "enabled": false, "path": "$.data", "value": "input data" }