Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

yyin/tcomp 2260 combine validation #718

Open
wants to merge 20 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,7 @@ documentation/rebuildpdf.bat

# release process
.build/
.build_source/
.build_source/
# direnv
.envrc
.env
34 changes: 23 additions & 11 deletions .jenkins/scripts/docker_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,32 @@ set -xe
# Parameters:
# $1: docker tag version
# $2: should tag as latest
# $3: requested image, if not given, all will be pushed

tag="${1?Missing tag}"
latest="${2:-false}"
_TAG="${1?Missing tag}"
_IS_LATEST="${2-'false'}"
_ONLY_ONE_IMAGE="${3-''}"

dockerBuild() {
echo ">> Building and pushing $1:${tag}"
cd "images/${1}-image"
mvn package jib:build@build -Ddocker.talend.image.tag=${tag}
if [[ ${latest} == 'true' ]]; then
mvn package jib:build@build -Ddocker.talend.image.tag=latest
_IMAGE="${1}"
echo ">> Building and push $_IMAGE:${_TAG}"

mvn package jib:build@build \
--file "images/${_IMAGE}-image/pom.xml" \
--define docker.talend.image.tag="${_TAG}"

if [[ ${_IS_LATEST} == 'true' ]]; then
mvn package jib:build@build \
--file "images/${_IMAGE}-image/pom.xml" \
--define docker.talend.image.tag=latest
fi
cd ../..
}

dockerBuild "component-server"
dockerBuild "component-starter-server"
dockerBuild "remote-engine-customizer"
if [[ -n "${_ONLY_ONE_IMAGE}" ]]; then
dockerBuild "${_ONLY_ONE_IMAGE}"
else
dockerBuild "component-server"
dockerBuild "component-starter-server"
dockerBuild "remote-engine-customizer"
fi

584 changes: 477 additions & 107 deletions Jenkinsfile

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion ci/Jenkinsfile-scan
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ final def veracodeCredentials = usernamePassword(
final def nexusCredentials = usernamePassword(
credentialsId: 'nexus-artifact-zl-credentials',
usernameVariable: 'NEXUS_USER',
passwordVariable: 'NEXUS_PASSWORD')
passwordVariable: 'NEXUS_PASS')

String git_branch_name = ""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.lang.reflect.Type;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
Expand Down Expand Up @@ -208,19 +209,29 @@ public Function<Map<String, String>, Object[]> parameterFactory(final Executable

return config -> {
final Map<String, String> notNullConfig = ofNullable(config).orElseGet(Collections::emptyMap);
final PayloadValidator visitor = new PayloadValidator();
if (!visitor.skip) {
visitor.globalPayload = new PayloadMapper((a, b) -> {
}).visitAndMap(metas, notNullConfig);
final PayloadMapper payloadMapper = new PayloadMapper(visitor);
payloadMapper.setGlobalPayload(visitor.globalPayload);
payloadMapper.visitAndMap(metas, notNullConfig);
visitor.throwIfFailed();
}
checkPayload(metas, notNullConfig);
return factories.stream().map(f -> f.apply(notNullConfig)).toArray(Object[]::new);
};
}

public static void checkPayload(final List<ParameterMeta> metas, final Map<String, String> notNullConfig) {
JsonObject globalPayload = new PayloadMapper((a, b) -> {
}).visitAndMap(metas, notNullConfig);
checkWithPayload(metas, notNullConfig, globalPayload);
}

public static void checkWithPayload(final List<ParameterMeta> metas, final Map<String, String> notNullConfig,
final JsonObject payload) {
final PayloadValidator visitor = new PayloadValidator();
if (!visitor.skip) {
visitor.globalPayload = payload;
final PayloadMapper payloadMapper = new PayloadMapper(visitor);
payloadMapper.setGlobalPayload(payload);
payloadMapper.visitAndMap(metas, notNullConfig);
visitor.throwIfFailed();
}
}

public Function<Supplier<Object>, Object> createContextualSupplier(final ClassLoader loader) {
return supplier -> {
final Thread thread = Thread.currentThread();
Expand Down Expand Up @@ -319,7 +330,7 @@ private Object createList(final ClassLoader loader, final Function<Supplier<Obje
final String configName = String.format("%s[%d]", name, paramIdx);
if (!config.containsKey(configName)) {
if (config.keySet().stream().anyMatch(k -> k.startsWith(configName + "."))) { // object
// mapping
// mapping
if (paramIdx == 0) {
args = findArgsName(itemClass);
}
Expand Down Expand Up @@ -887,6 +898,8 @@ public interface Messages {
String uniqueItems(String property);

String pattern(String property, String pattern);

String enumValues(String property, String enums, String val);
}

@RequiredArgsConstructor
Expand All @@ -908,7 +921,6 @@ public void onParameter(final ParameterMeta meta, final JsonValue value) {
if (!VISIBILITY_SERVICE.build(meta).isVisible(globalPayload)) {
return;
}

if (Boolean.parseBoolean(meta.getMetadata().get("tcomp::validation::required"))
&& value == JsonValue.NULL) {
errors.add(MESSAGES.required(meta.getPath()));
Expand Down Expand Up @@ -996,6 +1008,20 @@ public void onParameter(final ParameterMeta meta, final JsonValue value) {
}
}
}
{
final String enumValues = metadata.get("tcomp::validation::enumValues");
if (enumValues != null) {// value = null or not in enumValues: add error
if (value == null) {
errors.add(MESSAGES.enumValues(meta.getPath(), enumValues, null));
} else {
final String val = JsonValue.class.cast(value).toString().replace("\"", "");
String[] enums = enumValues.substring(1, enumValues.length() - 1).split(",");
if (Arrays.stream(enums).noneMatch(s -> s.trim().equalsIgnoreCase(val))) {
errors.add(MESSAGES.enumValues(meta.getPath(), enumValues, val));
}
}
}
}
}

private void throwIfFailed() {
Expand All @@ -1007,7 +1033,7 @@ private void throwIfFailed() {

/**
* Helper function for creating an instance from a configuration map.
*
*
* @param clazz Class of the wanted instance.
* @param <T> Type managed
* @return function that generate the wanted instance when calling
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ private void onProperty(final String contextualPrefix, final Collection<Paramete
break;
}
case BOOLEAN:
final String boolValue = config.get(newPath);
final String boolValue = getValue(config, newPath, definition);
if (boolValue == null || boolValue.isEmpty()) {
parameterVisitor.onParameter(definition, JsonValue.NULL);
} else {
Expand All @@ -98,7 +98,7 @@ private void onProperty(final String contextualPrefix, final Collection<Paramete
.ifPresent(v -> json.add(name, Boolean.parseBoolean(v)));
break;
case NUMBER:
final String numberValue = config.get(newPath);
final String numberValue = getValue(config, newPath, definition);
if (numberValue == null || numberValue.isEmpty()) {
parameterVisitor.onParameter(definition, JsonValue.NULL);
} else {
Expand All @@ -114,7 +114,7 @@ private void onProperty(final String contextualPrefix, final Collection<Paramete
break;
case ENUM:
case STRING: {
final String value = config.get(newPath);
final String value = getValue(config, newPath, definition);
parameterVisitor.onParameter(definition, value == null ? JsonValue.NULL : jsonp.createValue(value));
ofNullable(value).ifPresent(v -> json.add(name, v));
break;
Expand All @@ -123,6 +123,11 @@ private void onProperty(final String contextualPrefix, final Collection<Paramete
}
}

private static String getValue(final Map<String, String> config, final String newPath,
final ParameterMeta definition) {
return config.get(newPath) == null ? config.get(definition.getPath()) : config.get(newPath);
}

private void onObject(final Collection<ParameterMeta> definitions, final ParameterMeta meta,
final Map<String, String> config, final JsonObjectBuilder json, final String name,
final String currentPath) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@ org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.minI
org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.maxItems = Length of property ''{0}'' should be < {1}, got {2}.
org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.uniqueItems = ''{0}'' has duplicated items.
org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.pattern = ''{0}'' does not match ''{1}''.
org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.enumValues = Invalid value for Property ''{0}'' expected: ''{1}'', got {2}.
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.M2_REPOSITORY;
import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.STUDIO_MVN_REPOSITORY;
import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.TALEND_COMPONENT_MANAGER_M2_REPOSITORY;
import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.TALEND_COMPONENT_MANAGER_M2_SETTINGS;
Expand All @@ -39,6 +38,8 @@

class MavenRepositoryResolverTest {

static String M2_REPOSITORY = ".m2" + File.separator + "repository";

private final MavenRepositoryResolver resolver = new MavenRepositoryDefaultResolver();

final MavenRepositoryDefaultResolver mavenSettingsOnlyResolver = new MavenRepositoryDefaultResolver() {
Expand All @@ -63,7 +64,7 @@ public Path get(final String path) {
}
};

private final String fallback = System.getProperty("user.home") + "/" + M2_REPOSITORY;
private final String fallback = System.getProperty("user.home") + File.separator + M2_REPOSITORY;

private final Path repository = Paths.get(new File("target/test-classes").getAbsolutePath());

Expand Down Expand Up @@ -164,7 +165,8 @@ void discoverFromSettingsWindowsPath() {
mavenSettingsOnlyResolver.setHandler(handlerNoExistCheck);
final Path m2 = mavenSettingsOnlyResolver.discover();
assertNotNull(m2);
assertEquals("C:/Users/maven/repository", m2.toString());
assertEquals("C:" + File.separator + "Users" + File.separator + "maven" + File.separator + "repository",
m2.toString());
System.clearProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS);
}

Expand All @@ -174,7 +176,8 @@ void discoverFromSettingsTildePath() {
mavenSettingsOnlyResolver.setHandler(handlerNoExistCheck);
final Path m2 = mavenSettingsOnlyResolver.discover();
assertNotNull(m2);
assertEquals(System.getProperty("user.home") + "/mvn_home_dev/repository", m2.toString());
assertEquals(System.getProperty("user.home") + File.separator + "mvn_home_dev" + File.separator + "repository",
m2.toString());
System.clearProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS);
}

Expand All @@ -184,7 +187,8 @@ void discoverFromSettingsUserHomeProperty() {
mavenSettingsOnlyResolver.setHandler(handlerNoExistCheck);
final Path m2 = mavenSettingsOnlyResolver.discover();
assertNotNull(m2);
assertEquals(System.getProperty("user.home") + "/mvn_home_dev/repository", m2.toString());
assertEquals(System.getProperty("user.home") + File.separator + "mvn_home_dev" + File.separator + "repository",
m2.toString());
System.clearProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,7 @@ public class ConfigTypeNode {

private Collection<ActionReference> actions;

public void visibility(final SimplePropertyDefinition spd) {
// no-op
}
}
Loading