Complete review of unit & integration tests

vtl-api/src/main/java/it/bancaditalia/oss/vtl/config/VTLProperty.java

@@ -86,6 +86,16 @@ public default List<String> getValues()
 		return getValue() == null ? null : isMultiple() ? Arrays.asList(getValue().split(",")) : singletonList(getValue());
 	}
 
+	/**
+	 * Change the value for this property
+	 * 
+	 * @param newValue The new value for this property
+	 */
+	public default void setValue(Class<?> newValue)
+	{
+		setValue(newValue.getName());
+	}
+
 	/**
 	 * Change the values for this property. If the property is not multiple, behaviour is undefined
 	 * 

vtl-api/src/main/java/it/bancaditalia/oss/vtl/exceptions/VTLMissingComponentsException.java

@@ -54,7 +54,7 @@ public VTLMissingComponentsException(Set<? extends DataStructureComponent<?, ?,
 
 	public VTLMissingComponentsException(DataStructureComponent<?, ?, ?> missing, Collection<? extends DataStructureComponent<?, ?, ?>> operand)
 	{
-		super("Component " + missing.getVariable().getName() + " not found in " + operand);
+		super("Component " + missing + " not found in " + operand);
 	}
 
 	public VTLMissingComponentsException(Set<? extends DataStructureComponent<?, ?, ?>> missing, Map<? extends DataStructureComponent<?, ?, ?>, ? extends ScalarValue<?, ?, ?, ?>> datapoint)

vtl-api/src/main/java/it/bancaditalia/oss/vtl/exceptions/VTLUnboundAliasException.java

@@ -29,17 +29,17 @@ public class VTLUnboundAliasException extends VTLException {
 
 	private static final long serialVersionUID = 1L;
 
-	private final String name;
+	private final String alias;
 
 	public VTLUnboundAliasException(String alias) 
 	{
 		super("Alias '" + alias + "' is not bound to this transformation scheme.");
 
-		this.name = alias;
+		this.alias = alias;
 	}
 
-	public String getName()
+	public String getAlias()
 	{
-		return name;
+		return alias;
 	}
 }

vtl-api/src/main/java/it/bancaditalia/oss/vtl/model/data/DataPoint.java

@@ -199,12 +199,12 @@ public default boolean matches(Map<? extends DataStructureComponent<? extends Id
 	 * @param names The names of the component
 	 * @return map with the values for the chosen components
 	 */
-	public default Map<String, ScalarValue<?,?,?,?>> getValuesByNames(Collection<String> names)
+	public default Map<DataStructureComponent<?, ?, ?>, ScalarValue<?,?,?,?>> getValuesByNames(Collection<String> names)
 	{
 		return Utils.getStream(keySet())
 				.map(c -> new SimpleEntry<>(c.getVariable().getName(), c))
 				.filter(entryByKey(names::contains))
-				.collect(Collectors.toMap(Entry::getKey, e -> get(e.getValue())));
+				.collect(Collectors.toMap(Entry::getValue, e -> get(e.getValue())));
 
 	}
 	/**

vtl-api/src/main/java/it/bancaditalia/oss/vtl/model/data/DataSet.java

@@ -384,14 +384,6 @@ public default DataSet union(SerFunction<DataPoint, Lineage> lineageOp, List<Dat
 	 */
 	public DataSet union(SerFunction<DataPoint, Lineage> lineageOp, List<DataSet> others, boolean check);
 
-	/**
-	 * Creates a new dataset as containing all the datapoints of this dataset that don't have the same identifiers as the ones in the other dataset.
-	 * 
-	 * @param other the other dataset
-	 * @return the set difference between the two datasets.
-	 */
-	public DataSet setDiff(DataSet other);
-
 	/**
 	 * <b>NOTE</b>: The default implementation traverses this DataSet entirely.
 	 * 

vtl-api/src/main/java/it/bancaditalia/oss/vtl/model/data/DataStructureComponent.java

@@ -108,4 +108,12 @@ public default DataStructureComponent<R, S, D> getRenamed(MetadataRepository rep
 	{
 		return ((Variable<S, D>) repo.createTempVariable(newName, getVariable().getDomain())).as(getRole());
 	}
+
+	public default int defaultHashCode()
+	{
+		int prime = 31;
+		int result = prime + getVariable().hashCode();
+		result = prime * result + getRole().hashCode();
+		return result;
+	}
 }

vtl-api/src/main/java/it/bancaditalia/oss/vtl/model/data/Variable.java

@@ -99,4 +99,17 @@ public static String normalizeAlias(String alias)
 	{
 		return getDomain().cast(value);
 	}
+
+	/**
+	 * A default implementation to initialize the hashCode
+	 * 
+	 * @return A default variable hashCode.
+	 */
+	public default int defaultHashCode()
+	{
+		int prime = 31;
+		int result = prime + getDomain().hashCode();
+		result = prime * result + getName().hashCode();
+		return result;
+	}
 }

vtl-api/src/main/java/it/bancaditalia/oss/vtl/model/transform/analytic/LimitCriterion.java

@@ -40,5 +40,5 @@ public static enum LimitDirection
 	/**
 	 * @return The distance of the bound from current data point
 	 */
-	public long getCount();
+	public int getCount();
 }

vtl-api/src/main/java/it/bancaditalia/oss/vtl/session/MetadataRepository.java

@@ -50,6 +50,14 @@ public interface MetadataRepository
 	 */
 	public ValueDomainSubset<?, ?> getDomain(String name);
 
+	/**
+	 * Returns the source definition for the dataset with the specified name if it's defined.
+	 * 
+	 * @param name the name of the dataset
+	 * @return the source definition or null if it's not defined.
+	 */
+	public String getDatasetSource(String name);
+
 	/**
 	 * Returns a dataset structure with the specified name if it exists.
 	 * 

vtl-api/src/main/java/it/bancaditalia/oss/vtl/util/SerCollectors.java

@@ -40,6 +40,7 @@
 import java.util.Objects;
 import java.util.Optional;
 import java.util.OptionalDouble;
+import java.util.OptionalLong;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -124,21 +125,21 @@ public static <T, U> SerCollector<T, U[], U> reducing(U identity, SerFunction<?
     	return collectingAndThen(summarizingDouble(mapper), SerDoubleSumAvgCount::getSum);
     }
 
-    public static <T> SerCollector<T, ?, Long> summingLong(SerToLongFunction<? super T> mapper)
+    public static <T> SerCollector<T, ?, OptionalLong> summingLong(SerToLongFunction<? super T> mapper)
     {
         return new SerCollector<>(
-                () -> new long[1],
-                (a, t) -> { a[0] += mapper.applyAsLong(t); },
-                (a, b) -> { a[0] += b[0]; return a; },
-                a -> a[0], EnumSet.of(CONCURRENT, UNORDERED));
+                () -> new long[2],
+                (a, t) -> { a[0] += mapper.applyAsLong(t); a[1]++; },
+                (a, b) -> { a[0] += b[0]; a[1] += b[1]; return a; },
+                a -> a[1] <= 0 ? OptionalLong.empty() : OptionalLong.of(a[0]), EnumSet.of(CONCURRENT, UNORDERED));
     }
 
     public static <T> SerCollector<T, ?, OptionalDouble> averagingDouble(SerToDoubleFunction<? super T> mapper) 
     {
     	return collectingAndThen(summarizingDouble(mapper), SerDoubleSumAvgCount::getAverage);
     }
 
-    public static <T> SerCollector<T, SerDoubleSumAvgCount, SerDoubleSumAvgCount> summarizingDouble(SerToDoubleFunction<? super T> mapper)
+    public static <T> SerCollector<T, ?, SerDoubleSumAvgCount> summarizingDouble(SerToDoubleFunction<? super T> mapper)
     {
         return SerCollector.of(
                 () -> new SerDoubleSumAvgCount(),

vtl-api/src/main/java/it/bancaditalia/oss/vtl/util/SerDoubleSumAvgCount.java

@@ -91,7 +91,7 @@ public final long getCount()
 
     public final OptionalDouble getSum()
     {
-    	if (getCount() < 0)
+    	if (count <= 0)
     		return OptionalDouble.empty();
 
         return OptionalDouble.of(internalSum());

vtl-bundles/vtl-coverage/pom.xml

@@ -136,6 +136,12 @@
 			<artifactId>junit-jupiter-params</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.hamcrest</groupId>
+			<artifactId>hamcrest</artifactId>
+			<version>2.2</version>
+			<scope>test</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.apache.logging.log4j</groupId>
 			<artifactId>log4j-slf4j2-impl</artifactId>

vtl-bundles/vtl-coverage/src/test/java/it/bancaditalia/oss/vtl/coverage/tests/IntegrationTestSuite.java

@@ -19,110 +19,125 @@
  */
 package it.bancaditalia.oss.vtl.coverage.tests;
 
+import static it.bancaditalia.oss.vtl.config.VTLGeneralProperties.ENVIRONMENT_IMPLEMENTATION;
+import static it.bancaditalia.oss.vtl.config.VTLGeneralProperties.METADATA_REPOSITORY;
+import static it.bancaditalia.oss.vtl.impl.environment.CSVPathEnvironment.VTL_CSV_ENVIRONMENT_SEARCH_PATH;
+import static it.bancaditalia.oss.vtl.impl.meta.json.JsonMetadataRepository.JSON_METADATA_URL;
+import static it.bancaditalia.oss.vtl.util.SerCollectors.toList;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.anyOf;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.URISyntaxException;
-import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
+import org.hamcrest.Matchers;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
 import it.bancaditalia.oss.vtl.config.ConfigurationManagerFactory;
-import it.bancaditalia.oss.vtl.impl.meta.InMemoryMetadataRepository;
-import it.bancaditalia.oss.vtl.impl.session.VTLSessionImpl;
+import it.bancaditalia.oss.vtl.impl.environment.CSVPathEnvironment;
+import it.bancaditalia.oss.vtl.impl.environment.WorkspaceImpl;
+import it.bancaditalia.oss.vtl.impl.meta.json.JsonMetadataRepository;
 import it.bancaditalia.oss.vtl.model.data.DataPoint;
 import it.bancaditalia.oss.vtl.model.data.DataSet;
-import it.bancaditalia.oss.vtl.model.data.VTLValue;
+import it.bancaditalia.oss.vtl.model.data.DataStructureComponent;
+import it.bancaditalia.oss.vtl.session.VTLSession;
 
 public class IntegrationTestSuite
 {
 	static {
 		System.setProperty("vtl.sequential", "true");
 	}
+
+	private static Path root;
 
 	public static Stream<Arguments> test() throws IOException, URISyntaxException
 	{
-		URL root = IntegrationTestSuite.class.getResource("vtl");
-		Pattern pattern = Pattern.compile("'csv:([^']+)'");
+		root = Paths.get(IntegrationTestSuite.class.getResource("../tests").toURI());
+
+		METADATA_REPOSITORY.setValue(JsonMetadataRepository.class);
+		ENVIRONMENT_IMPLEMENTATION.setValues(CSVPathEnvironment.class, WorkspaceImpl.class);
+
 		List<Arguments> tests = new ArrayList<>();
-		try (BufferedReader dirReader = new BufferedReader(new InputStreamReader(IntegrationTestSuite.class.getResourceAsStream("vtl"), UTF_8)))
+		String testName;
+		StringBuilder testCode = new StringBuilder();
+
+		try (BufferedReader dirReader = new BufferedReader(new InputStreamReader(IntegrationTestSuite.class.getResourceAsStream("../tests"), UTF_8)))
 		{
-			String testName;
-			StringBuilder testCode = new StringBuilder();
-			StringBuilder parsedLine = new StringBuilder();
 			while ((testName = dirReader.readLine()) != null)
-			{
-				try (BufferedReader testReader = new BufferedReader(new InputStreamReader(new URL(root, "vtl/" + testName).openStream(), UTF_8)))
+				if (!testName.endsWith(".class"))
 				{
-					String testLine;
-					int headerLines = 20;
-					while ((testLine = testReader.readLine()) != null)
+					try (BufferedReader testReader = Files.newBufferedReader(root.resolve(testName).resolve(testName + ".vtl")))
 					{
-						if (--headerLines > 0)
-							continue;
-						Matcher matcher = pattern.matcher(testLine);
-						int start = 0;
-						while (matcher.find())
+						String testLine;
+						int headerLines = 20;
+						while ((testLine = testReader.readLine()) != null)
 						{
-							parsedLine.append(testLine.substring(start, matcher.start(1)))
-								.append(new URL(root, "data/" + matcher.group(1)).toString());
-							start = matcher.end(1);
+							if (--headerLines > 0)
+								continue;
+							testCode.append(testLine).append(System.lineSeparator());
 						}
-						testCode.append(parsedLine.append(testLine.substring(start)).toString())
-								.append(System.lineSeparator());
-						parsedLine.setLength(0);
 					}
+					tests.add(Arguments.of(testName, testCode.toString()));
+					testCode.setLength(0);
 				}
-				tests.add(Arguments.of(testName, testCode.toString()));
-				testCode.setLength(0);
-			}
 		}
 
 		return tests.stream();
 	}
 
 	@ParameterizedTest(name = "{0}")
 	@MethodSource
-	public void test(String testName, String testCode)
+	public synchronized void test(final String testName, final String testCode) throws IOException, URISyntaxException
 	{
-		try
-		{
-			((InMemoryMetadataRepository) ConfigurationManagerFactory.getInstance().getMetadataRepository()).clearVariables();
-
-			System.out.println("------------------------------------------------------------------------------------------------");
-			System.out.println("                                        " + testName);
-			System.out.println("------------------------------------------------------------------------------------------------");
-			System.out.println();
-			System.out.println(testCode);
-			System.out.println("------------------------------------------------------------------------------------------------");
-			VTLSessionImpl session = new VTLSessionImpl(testCode);
-			session.compile();
-			VTLValue result = session.resolve("test_result");
-			if (result instanceof DataSet)
-				try (Stream<DataPoint> stream = ((DataSet) result).stream())
-				{
-					stream.forEach(dp -> System.out.print("."));
-					System.out.println();
-				}
-		}
-		catch (RuntimeException e)
+		System.out.println("------------------------------------------------------------------------------------------------");
+		System.out.println("                                        " + testName);
+		System.out.println("------------------------------------------------------------------------------------------------");
+		System.out.println();
+		System.out.println(testCode);
+		System.out.println("------------------------------------------------------------------------------------------------");
+
+		VTL_CSV_ENVIRONMENT_SEARCH_PATH.setValues(root.resolve(testName).toString());
+		JSON_METADATA_URL.setValue(IntegrationTestSuite.class.getResource(testName + "/" + testName + ".json").toString());
+		VTLSession session = ConfigurationManagerFactory.getInstance().createSession(testCode);
+
+		session.compile();
+
+		DataSet expected = session.resolve("expected", DataSet.class);
+		DataSet result = session.resolve("test_result", DataSet.class);
+
+		for (DataStructureComponent<?, ?, ?> comp: expected.getMetadata())
+			assertTrue(result.getMetadata().contains(comp), "Expected component " + comp + " is missing in " + result.getMetadata());
+		for (DataStructureComponent<?, ?, ?> comp: result.getMetadata())
+			assertTrue(expected.getMetadata().contains(comp), "Unexpected component " + comp + " in result.");
+
+		List<DataPoint> resDPs;
+		List<DataPoint> expectedDPs;
+		try (Stream<DataPoint> resStream = result.stream(); Stream<DataPoint> expStream = expected.stream())
 		{
-			throw new RuntimeException("Error in testName\n" + testCode + "\n\n" + e.getClass().getName() + ": " + e.getMessage() + "\n", e) {
-				private static final long serialVersionUID = 1L;
-
-				{
-					setStackTrace(e.getStackTrace());
-				}
-			};
+			resDPs = resStream.collect(toList());
+			expectedDPs = expStream.collect(toList());
 		}
+
+		System.out.println("Expected:");
+		expectedDPs.forEach(System.out::println);
+		System.out.println("Actual:");
+		resDPs.forEach(System.out::println);
+
+		for (DataPoint dp: resDPs)
+			assertThat(dp, anyOf(expectedDPs.stream().map(Matchers::equalTo).collect(toList())));
+		for (DataPoint dp: expectedDPs)
+			assertThat(dp, anyOf(resDPs.stream().map(Matchers::equalTo).collect(toList())));
 	}
 }

vtl-bundles/vtl-coverage/src/test/resources/it/bancaditalia/oss/vtl/coverage/tests/abs/abs.csv

@@ -0,0 +1,5 @@
+id1,m1
+a,3.346
+b,3.671
+c,6.1893434
+d,