Faster access to the list of dataflows

BUILD

@@ -1 +1 @@
-20230210-1402
+20230707-0756

JAVA/configuration.properties

@@ -53,6 +53,8 @@ java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
 # providers.PROVIDER_ID.read.timeout = 10000
 # providers.PROVIDER_ID.connect.timeout = 10000
 # providers.PROVIDER_ID.description = YOUR_DESCRIPTION
+# sdmxversion must be SDMX_V2 or SDMX_V3
+# providers.PROVIDER_ID.sdmxversion = SDMX_V2
 
 # If you want to add a new provider, you can do it using the external.providers key.
 #external.providers = TEST1,TEST2

JAVA/src/main/java/it/bancaditalia/oss/sdmx/api/PortableTimeSeries.java

@@ -20,7 +20,7 @@
 */
 package it.bancaditalia.oss.sdmx.api;
 
-import static java.util.stream.Collectors.joining;
+
 
 import java.io.Serializable;
 import java.util.AbstractList;

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/Provider.java

@@ -52,7 +52,6 @@ public class Provider {
 	private boolean needsCredentials;
 	private boolean needsURLEncoding;
 	private boolean supportsCompression;
-	private boolean full = false;
 	private boolean isCustom = false;
 
 	// key: flow id (full) --> flow
@@ -174,13 +173,6 @@ public void setNeedsCredentials(boolean needsCredentials) {
 		this.needsCredentials = needsCredentials;
 	}
 
-	public void setFull(boolean full) {
-		this.full = full;
-	}
-
-	public boolean isFull() {
-		return full;
-	}
 
 	public boolean isNeedsURLEncoding() {
 		return needsURLEncoding;

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/RestSdmx30Client.java

@@ -160,9 +160,9 @@ protected URL buildDSDQuery(String dsd, String agency, String version, boolean f
 			throw new RuntimeException("Invalid query parameters: agency=" + agency + " dsd=" + dsd + " endpoint=" + endpoint);
 	}
 
-	protected URL buildFlowQuery(String dataflow, String agency, String version) throws SdmxException
+	protected URL buildFlowQuery(String dataflow, String agency, String version, String detail) throws SdmxException
 	{
-		return Sdmx30Queries.createDataflowQuery(endpoint, dataflow, agency, version).buildQuery();
+		return Sdmx30Queries.createDataflowQuery(endpoint, dataflow, agency, version, detail).buildQuery();
 	}
 
 	protected URL buildCodelistQuery(String codeList, String agency, String version) throws SdmxException

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/RestSdmxClient.java

@@ -49,13 +49,10 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.zip.GZIPInputStream;
-import java.util.zip.InflaterInputStream;
-import java.util.zip.ZipInputStream;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.HttpsURLConnection;
 import javax.net.ssl.SSLSocketFactory;
-import javax.swing.LayoutFocusTraversalPolicy;
 import javax.xml.stream.XMLEventReader;
 import javax.xml.stream.XMLInputFactory;
 import javax.xml.stream.XMLStreamException;
@@ -107,6 +104,7 @@ public class RestSdmxClient implements GenericSDMXClient
 	protected /* final */ URI		endpoint;
 	protected boolean				needsCredentials				= false;
 	protected boolean				containsCredentials				= false;
+	protected boolean				RaisedWarning				    = false;
 	protected String				user							= null;
 	protected String				pw								= null;
 	protected int					readTimeout;
@@ -190,11 +188,17 @@ public void setMaxRedirects(int maxRedirects)
 		this.maxRedirects = maxRedirects;
 	}
 
+	public void setRaisedWarning(boolean RaisedWarning)
+	{
+		this.RaisedWarning = RaisedWarning;
+	}
+
+
 	@Override
 	public Map<String, Dataflow> getDataflows() throws SdmxException
 	{
 		Map<String, Dataflow> result = null;
-		URL query = buildFlowQuery(ALL_AGENCIES, "all", latestKeyword);
+		URL query = buildFlowQuery(ALL_AGENCIES, "all", latestKeyword,"allstubs");
 		List<Dataflow> flows = runQuery(new DataflowParser(), query, null, "dataflow_all");
 		if (flows.size() > 0)
 		{
@@ -215,7 +219,7 @@ public Dataflow getDataflow(String dataflow, String agency, String version) thro
 		Dataflow result = null;
 		if(agency == null) agency = ALL_AGENCIES;
 		if(version == null) version = this.latestKeyword;
-		URL query = buildFlowQuery(dataflow, agency, version);
+		URL query = buildFlowQuery(dataflow, agency, version, null);
 		List<Dataflow> flows = runQuery(new DataflowParser(), query, null, "dataflow_" + dataflow);
 		if (flows.size() >= 1)
 			result = flows.get(0);
@@ -360,7 +364,7 @@ protected final <T> T runQuery(Parser<T> parser, URL query, String acceptHeader,
 		URLConnection conn = null;
 		URL url = null;
 		LOGGER.log(Level.INFO, "Contacting web service with query: {0}", query);
-
+		LOGGER.log(Level.FINE, "Supports compression: {0}", this.supportsCompression);
 		try
 		{
 			int code;
@@ -410,9 +414,20 @@ protected final <T> T runQuery(Parser<T> parser, URL query, String acceptHeader,
 					code = conn instanceof HttpURLConnection ? ((HttpURLConnection) conn).getResponseCode() : HttpURLConnection.HTTP_OK;
 					conn = url.openConnection(proxy);
 					((HttpURLConnection) conn).setRequestMethod("GET");
+					((HttpURLConnection) conn).setInstanceFollowRedirects(false);
+					handleHttpHeaders((HttpURLConnection) conn, acceptHeader);
+					code = conn instanceof HttpURLConnection ? ((HttpURLConnection) conn).getResponseCode() : HttpURLConnection.HTTP_OK;
+
+				}
+				if (code == HttpURLConnection.HTTP_INTERNAL_ERROR)
+				{
+					LOGGER.log(Level.SEVERE, "Error on the provider side. Second attempt...");
+					conn = url.openConnection(proxy);
+					((HttpURLConnection) conn).setRequestMethod("GET");
 					code = conn instanceof HttpURLConnection ? ((HttpURLConnection) conn).getResponseCode() : HttpURLConnection.HTTP_OK;
 
 				}
+
 				if (isRedirection(code))
 				{
 					URL redirection = getRedirectionURL(conn, code);
@@ -440,13 +455,27 @@ protected final <T> T runQuery(Parser<T> parser, URL query, String acceptHeader,
 				String encoding = conn.getContentEncoding() == null ? "" : conn.getContentEncoding();
 				if (encoding.equalsIgnoreCase("gzip"))
 					stream = new GZIPInputStream(stream);
-				else if (encoding.equalsIgnoreCase("deflate"))
+				else if(this.supportsCompression)
+				{
+					String disposition = conn.getHeaderField("Content-Disposition") == null ? "" : conn.getHeaderField("Content-Disposition");
+					LOGGER.fine("Content-Disposition: " + disposition );
+					if(disposition.contains(".gz"))
+					{
+						if(this.RaisedWarning == false)
+							{
+							LOGGER.warning("Content-Encoding header is missing in the response");
+							setRaisedWarning(true);
+							}
+						stream = new GZIPInputStream(stream);
+					}
+				}
+					/*else if (encoding.equalsIgnoreCase("deflate"))
 					stream = new InflaterInputStream(stream);
 				else if (conn.getContentType() != null && conn.getContentType().contains("application/octet-stream"))
 				{
 					stream = new ZipInputStream(stream);
 					((ZipInputStream) stream).getNextEntry();
-				}
+				}*/
 
 				if (Configuration.isDumpXml() && dumpName != null) // skip providers < sdmx v2.1
 				{
@@ -471,6 +500,16 @@ else if (conn.getContentType() != null && conn.getContentType().contains("applic
 						stream = new ByteArrayInputStream(baos.toByteArray());
 					}
 				}
+				else if (this.supportsCompression)
+				{
+					ByteArrayOutputStream baos = new ByteArrayOutputStream();
+					byte[] buf = new byte[4096];
+					int i;
+					while ((i = stream.read(buf, 0, 4096)) > 0)
+						baos.write(buf, 0, i);
+					baos.close();
+					stream = new ByteArrayInputStream(baos.toByteArray());
+				}
 
 				try (Reader reader = new InputStreamReader(stream, StandardCharsets.UTF_8))
 				{
@@ -537,7 +576,10 @@ protected void handleHttpHeaders(HttpURLConnection conn, String acceptHeader)
 		}
 		if (supportsCompression)
 		{
-			conn.addRequestProperty("Accept-Encoding", "gzip,deflate");
+			LOGGER.fine("--> Ask for compression");
+			// with deflate, response is not always compressed...
+			//conn.addRequestProperty("Accept-Encoding", "gzip,deflate");
+			conn.addRequestProperty("Accept-Encoding", "gzip");
 		}
 		if (acceptHeader != null && !"".equals(acceptHeader))
 			conn.setRequestProperty("Accept", acceptHeader);
@@ -563,9 +605,9 @@ protected URL buildDSDQuery(String dsd, String agency, String version, boolean f
 			throw new RuntimeException("Invalid query parameters: agency=" + agency + " dsd=" + dsd + " endpoint=" + endpoint);
 	}
 
-	protected URL buildFlowQuery(String dataflow, String agency, String version) throws SdmxException
+	protected URL buildFlowQuery(String dataflow, String agency, String version, String detail) throws SdmxException
 	{
-		return Sdmx21Queries.createDataflowQuery(endpoint, dataflow, agency, version).buildSdmx21Query();
+		return Sdmx21Queries.createDataflowQuery(endpoint, dataflow, agency, version, detail).buildSdmx21Query();
 	}
 
 	protected URL buildCodelistQuery(String codeList, String agency, String version) throws SdmxException

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/SdmxClientHandler.java

@@ -294,7 +294,8 @@ public static SDMXReference getDSDIdentifier(String providerName, String dataflo
 		if (result == null)
 		{
 			LOGGER.finer("DSD identifier for dataflow " + dataflow + " not cached. Calling Provider.");
-			Dataflow df = getClient(providerName).getDataflow(dataflow, null, null);
+			result = provider.getFlow(dataflow);
+			Dataflow df = getClient(providerName).getDataflow(result.getId(), result.getAgency(), result.getVersion());
 			if (df != null)
 			{
 				provider.setFlow(df);
@@ -468,14 +469,14 @@ public static Map<String, Dataflow> getFlowObjects(String provider, String patte
 		Map<String, Dataflow> flows = null;
 		Provider p = getProvider(provider);
 		flows = p.getFlows();
-		if (flows == null || flows.size() == 0 || !p.isFull())
+		if (flows == null || flows.size() == 0)
 		{
 			LOGGER.fine("Flows for " + provider + " not cached. Calling Provider.");
 			flows = getClient(provider).getDataflows();
 			if (flows != null && flows.size() != 0)
 			{
 				p.setFlows(flows);
-				p.setFull(true);
+
 			}
 			else
 				throw new SdmxXmlContentException("Could not get dataflows from provider: '" + provider + "'");
@@ -744,7 +745,6 @@ private static Map<String, Dataflow> filterFlows(Map<String, Dataflow> flows, St
 		{
 			if (pattern != null && !pattern.trim().isEmpty())
 				pattern = pattern.replaceAll("\\*", ".*").replaceAll("\\?", ".");
-
 			for (Entry<String, Dataflow> entry : flows.entrySet())
 			{
 				String trueName;
@@ -762,7 +762,8 @@ private static Map<String, Dataflow> filterFlows(Map<String, Dataflow> flows, St
 					result.put(entry.getKey(), entry.getValue());
 			}
 		}
-
+		else
+			LOGGER.fine("No flows to filter");
 		LOGGER.exiting(sourceClass, sourceMethod);
 		return result;
 	}

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/custom/BBK.java

@@ -50,8 +50,9 @@ public BBK() throws URISyntaxException {
 	}
 
 	@Override
-	protected URL buildFlowQuery(String dataflow, String agency, String version) throws SdmxException{
+	protected URL buildFlowQuery(String dataflow, String agency, String version, String detail) throws SdmxException{
 		try {
+
 			return new URL(endpoint + "/metadata/dataflow/BBK" + (dataflow.equalsIgnoreCase("all") ? "" : ("/" + dataflow)));
 		} catch (MalformedURLException e) {
 			throw new SdmxInvalidParameterException("Invalid query parameters: dataflow: " + dataflow + ", endpoint=" + endpoint);

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/custom/DotStat.java

@@ -58,7 +58,7 @@ public DotStat(String name, URI endpoint, boolean needsCredentials) {
 	@Override
 	public Dataflow getDataflow(String dataflow, String agency, String version) throws SdmxException {
 		// OECD (and .Stat infrastructure) does not handle flows. We simulate it
-		URL query = buildFlowQuery(dataflow, ALL_AGENCIES, LATEST_VERSION );
+		URL query = buildFlowQuery(dataflow, ALL_AGENCIES, LATEST_VERSION, null );
 		List<DataFlowStructure> dsds = runQuery(new DataStructureParser(), query, null, null);
 		if(dsds.size() > 0)
 		{
@@ -74,7 +74,7 @@ public Dataflow getDataflow(String dataflow, String agency, String version) thro
 	@Override
 	public Map<String, Dataflow> getDataflows() throws SdmxException {
 		// OECD (and .Stat infrastructure) does not handle flows. We simulate it
-		URL query = buildFlowQuery("ALL", ALL_AGENCIES, LATEST_VERSION );
+		URL query = buildFlowQuery("ALL", ALL_AGENCIES, LATEST_VERSION, "allstubs" );
 		List<DataFlowStructure> dsds = runQuery(new DataStructureParser(), query, null, null);
 		if(dsds.size() > 0)
 		{
@@ -94,7 +94,7 @@ public Map<String, Dataflow> getDataflows() throws SdmxException {
 	}
 
 	@Override
-	protected URL buildFlowQuery(String flow, String agency, String version)  throws SdmxException{
+	protected URL buildFlowQuery(String flow, String agency, String version, String detail)  throws SdmxException{
 		return(buildDSDQuery(flow, agency, version, false));
 	}
 

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/custom/EUROSTAT.java

@@ -56,7 +56,7 @@ public class EUROSTAT extends RestSdmxClient
 
 	public EUROSTAT() throws URISyntaxException
 	{
-		super("Eurostat", new URI(EUROSTAT_PROVIDER), false, false, false);
+		super("Eurostat", new URI(EUROSTAT_PROVIDER), false, false, true);
 	}
 
 	@Override

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/custom/EUROSTAT_COMEXT.java

@@ -33,6 +33,6 @@ public class EUROSTAT_COMEXT extends RestSdmxClient{
 	private static final String EUROSTAT_COMEXT_PROVIDER = "https://ec.europa.eu/eurostat/api/comext/dissemination/sdmx/2.1";
 
 	public EUROSTAT_COMEXT() throws URISyntaxException {
-		super("Eurostat", new URI(EUROSTAT_COMEXT_PROVIDER), false, false, false);
+		super("Eurostat", new URI(EUROSTAT_COMEXT_PROVIDER), false, false, true);
 	}
 }

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/custom/EUROSTAT_COMP.java

@@ -33,6 +33,6 @@ public class EUROSTAT_COMP extends RestSdmxClient{
 	private static final String EUROSTAT_COMP_PROVIDER = "https://webgate.ec.europa.eu/comp/redisstat/api/dissemination/sdmx/2.1";
 
 	public EUROSTAT_COMP() throws URISyntaxException {
-		super("Eurostat", new URI(EUROSTAT_COMP_PROVIDER), false, false, false);
+		super("Eurostat", new URI(EUROSTAT_COMP_PROVIDER), false, false, true);
 	}
 }

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/custom/EUROSTAT_EMPL.java

@@ -33,6 +33,6 @@ public class EUROSTAT_EMPL extends RestSdmxClient{
 	private static final String EUROSTAT_EMPL_PROVIDER = "https://webgate.ec.europa.eu/empl/redisstat/api/dissemination/sdmx/2.1";
 
 	public EUROSTAT_EMPL() throws URISyntaxException {
-		super("Eurostat", new URI(EUROSTAT_EMPL_PROVIDER), false, false, false);
+		super("Eurostat", new URI(EUROSTAT_EMPL_PROVIDER), false, false, true);
 	}
 }

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/custom/EUROSTAT_GROW.java

@@ -34,6 +34,6 @@ public class EUROSTAT_GROW extends RestSdmxClient{
 
 
 	public EUROSTAT_GROW() throws URISyntaxException {
-		super("Eurostat", new URI(EUROSTAT_GROW_PROVIDER), false, false, false);
+		super("Eurostat", new URI(EUROSTAT_GROW_PROVIDER), false, false, true);
 	}
 }

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/custom/IMF2.java

@@ -46,7 +46,7 @@ public IMF2() throws URISyntaxException {
 	}
 
 	@Override
-	protected URL buildFlowQuery(String flow, String agency, String version) throws SdmxException{
+	protected URL buildFlowQuery(String flow, String agency, String version, String detail) throws SdmxException{
 		if( endpoint!=null){
 			try {
 				return new RestQueryBuilder(endpoint).addPath("Dataflow").build();

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/custom/RestSdmx20Client.java

@@ -59,7 +59,7 @@ public RestSdmx20Client(String name, URI endpoint, boolean needsCredentials, Str
 	public Map<String, Dataflow> getDataflows() throws SdmxException
 	{
 
-		URL query = buildFlowQuery("ALL", null, null);
+		URL query = buildFlowQuery("ALL", null, null, null);
 		List<Dataflow> dfs = runQuery(new DataflowParser(), query, null, null);
 		if (dfs.size() > 0)
 		{
@@ -78,7 +78,7 @@ public Map<String, Dataflow> getDataflows() throws SdmxException
 	@Override
 	public Dataflow getDataflow(String dataflow, String agency, String version) throws SdmxException
 	{
-		URL query = buildFlowQuery(dataflow, agency, version);
+		URL query = buildFlowQuery(dataflow, agency, version, null);
 		List<Dataflow> flows = runQuery(new DataflowParser(), query, null, null);
 		if (flows.size() >= 1)
 			for (Dataflow item : flows)

JAVA/src/main/java/it/bancaditalia/oss/sdmx/client/custom/WB.java

@@ -48,9 +48,9 @@ protected URL buildDataQuery(Dataflow dataflow, String resource, String startTim
 	}
 
 	@Override
-	protected URL buildFlowQuery(String flow, String agency, String version) throws SdmxException{
+	protected URL buildFlowQuery(String flow, String agency, String version, String detail) throws SdmxException{
 		try {
-			return Sdmx21Queries.createDataflowQuery(endpoint, flow, agency, version+"/").build();
+			return Sdmx21Queries.createDataflowQuery(endpoint, flow, agency, version+"/", null).build();
 		} catch (MalformedURLException e) {
 			throw SdmxExceptionFactory.wrap(e);
 		}