Merge remote-tracking branch 'origin/v1.X'

Changelog.rst

@@ -2,6 +2,19 @@
 Changelog
 #########
 
+2.0 (2023-03-03)
+----------------
+
+**Added**
+ - Added Webservices to search for datasets and dataflows in ECB, EUROSTAT,
+   BIS and ILO using a REST API.
+
+**Changes**
+ - Fixed read_xml to allow for more flexibility on structural validation and better error management.
+
+**Bugfixes**
+ - Fixed member reading on CubeRegion.
+
 1.3 (2022-31-05)
 ----------------
 **Added**

Pipfile

@@ -20,6 +20,8 @@ numpy = "==1.19.3"
 validators = "==0.19.0"
 requests = "==2.27.1"
 xmltodict = "==0.13.0"
+flask = "==2.2.3"
+flask-cors = "==3.0.10"
 
 [requires]
 python_version = "*"

sdmxthon/model/definitions.py

@@ -6,17 +6,13 @@
 from datetime import datetime
 from typing import List
 
-from sdmxthon.model.base import MaintainableArtefact, \
-    InternationalString
+from sdmxthon.model.base import InternationalString, MaintainableArtefact
 from sdmxthon.model.component import Component
-from sdmxthon.model.descriptors import ComponentList, DimensionDescriptor, \
-    AttributeDescriptor, MeasureDescriptor, GroupDimensionDescriptor
+from sdmxthon.model.descriptors import AttributeDescriptor, ComponentList, DimensionDescriptor, GroupDimensionDescriptor, MeasureDescriptor
 from sdmxthon.model.extras import ReferencePeriod, ReleaseCalendar
-from sdmxthon.model.utils import generic_setter, ConstraintRoleType, \
-    bool_setter
-from sdmxthon.utils.handlers import export_intern_data, add_indent, \
-    split_unique_id
-from sdmxthon.utils.mappings import structureAbbr, Data_Types_VTL, commonAbbr
+from sdmxthon.model.utils import ConstraintRoleType, bool_setter, generic_setter
+from sdmxthon.utils.handlers import add_indent, export_intern_data, split_unique_id
+from sdmxthon.utils.mappings import Data_Types_VTL, commonAbbr, structureAbbr
 
 
 class MemberSelection(object):
@@ -40,6 +36,8 @@ def sel_value(self) -> list:
 
     @sel_value.setter
     def sel_value(self, value):
+        if not isinstance(value, list):
+            value = [value]
         self._selValue = generic_setter(value, list)
 
     @property

sdmxthon/utils/xml_base.py

@@ -75,7 +75,7 @@ def validate_doc(infile):
         parser = etree.ETCompatXMLParser()
     except AttributeError:
         # fallback to xml.etree
-        parser = etree.XMLParser()
+        parser = etree.XMLParser(remove_blank_text=True)
 
     base_path = os.path.dirname(os.path.dirname(__file__))
     schema = os.path.join(base_path, pathToSchema)
@@ -86,17 +86,14 @@ def validate_doc(infile):
         infile = BytesIO(bytes(infile, "UTF_8"))
 
     doc = etree.parse(infile, parser=parser)
-
     if not xmlschema.validate(doc):
-        try:
-            xmlschema.assertValid(doc)
-        except DocumentInvalid as e:
-            if len(e.args) == 1 and \
-                    'xsi:type' in e.args[0] or \
-                    'abstract' in e.args[0]:
-                pass
-            else:
-                raise e
+        log_errors = list(xmlschema.error_log)
+        unhandled_errors = []
+        for e in log_errors:
+            if 'content type is empty' not in e.message:
+                unhandled_errors.append(e.message)
+        if len(unhandled_errors)>0:
+            raise Exception(';\n'.join(unhandled_errors))
 
 
 def cast(typ, value):

setup.py

@@ -11,7 +11,7 @@
     long_description_content_type='text/x-rst',
     packages=setuptools.find_packages(exclude=["testSuite"]),
     include_package_data=True,
-    version='1.4',
+    version='2.0',
     license='Apache 2.0',
     license_files='license.txt',
     author='MeaningfulData',

testApi.py

@@ -1,20 +1,15 @@
 # flake8: noqa
 
-import pandas as pd
-
-from sdmxthon.api.api import get_pandas_df, read_sdmx
-from sdmxthon.model.dataset import Dataset
-from sdmxthon.model.definitions import DataStructureDefinition
-from sdmxthon.utils.enums import MessageTypeEnum
-from sdmxthon.utils.handlers import first_element_dict
+from sdmxthon.api.api import read_sdmx
 
 data_file1 = "development_files/data_file1.xml"
 url_eu = "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/" \
          "EI_BSCO_M$DEFAULTVIEW/?format=sdmx_2.1_structured"
+url_ecb = "https://sdw-wsrest.ecb.europa.eu/service/dataflow/ECB/IVF/1.0?references=all&detail=full"
 
 
 def main():
-    message = read_sdmx(data_file1, validate=False)
+    message = read_sdmx(url_ecb, validate=False)
 
     # print(message.payload['ESTAT:EI_BSCO_M$DEFAULTVIEW(1.0)']
     # .data.to_csv('test.csv', index=False))

web_services/app.py

@@ -0,0 +1,95 @@
+import json
+
+import validators
+from flask import Flask, Response, request
+from flask_cors import CORS
+
+from web_services import BISRequest, BaseRequest, ECBRequest, EUROSTATRequest, ILORequest
+
+app = Flask(__name__)
+CORS(app)
+
+agencies = {'BIS': BISRequest, 'ECB': ECBRequest,
+            'ESTAT': EUROSTATRequest, 'ILO': ILORequest}
+
+
+# url to provide agencies information (name, code and base url)
+@app.route('/agencies', methods=['GET'])
+def get_agencies_info():
+    agencies_info = []
+    for i in agencies.values():
+        info = {'name': i.name,
+                'code': i.code,
+                'api_base_url': i.base_url}
+        agencies_info.append(info)
+    try:
+        return Response(json.dumps(agencies_info, indent=2), status=200)
+    except Exception as e:
+        return Response(str(e), status=500)
+
+
+# url to provide available dataflows for every agency
+@app.route('/dataflows/<agency_code>', methods=['GET'])
+def get_dataflows(agency_code):
+    if agency_code not in agencies.keys():
+        return Response('Agency name not allowed', status=400)
+    try:
+        x = agencies[agency_code]
+        dataflows = x.get_dataflows(params={'code': agency_code})
+    except Exception as e:
+        return Response(str(e), status=500)
+    return Response(json.dumps(dataflows, indent=2), status=200)
+
+
+# url to redirect to specific dataflow data
+@app.route('/dataflows/data/url/<agency_code>/<unique_id>', methods=['GET'])
+def get_data_url(agency_code, unique_id):
+    params = request.args.to_dict()
+    if agency_code not in agencies.keys():
+        return Response('Agency name not allowed', status=400)
+    try:
+        x = agencies[agency_code]
+        url_str = x.get_data_url(unique_id=unique_id,
+                                 params=params)
+        return url_str, 200
+    except Exception as e:
+        return Response(str(e), status=500)
+
+
+@app.route('/dataflows/url/<agency_code>/<unique_id>', methods=['GET'])
+def get_dataflow_metadata_url(agency_code, unique_id):
+    params = request.args.to_dict()
+    if agency_code not in agencies.keys():
+        return Response('Agency name not allowed', status=400)
+    try:
+        x = agencies[agency_code]
+        metadata_url_str = x.get_metadata_url(unique_id=unique_id,
+                                              params=params)
+        return metadata_url_str, 200
+    except Exception as e:
+        return Response(str(e), status=500)
+
+
+@app.route('/dataflows/code', methods=['GET'])
+def get_code_url():
+    params = request.args.to_dict()
+    print(params.keys())
+    if len(params.keys()) > 1:
+        return Response("Too much parameters, insert only url parameter", status=400)
+    if 'url' not in params.keys():
+        return Response("Invalid parameters, insert url parameter", status=400)
+    url = params['url']
+    if f"{url}" == '':
+        return Response('Empty url is not allowed', status=400)
+    if not validators.url(f"{url}"):
+        return Response(f"{url} is not a valid url", status=400)
+    try:
+        x = BaseRequest()
+        code_str = x.get_sdmxthon_code(url=url)
+        return code_str
+    except Exception as e:
+        return Response(str(e), status=500)
+
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0')

web_services/test.py

@@ -0,0 +1,171 @@
+import requests
+import random
+
+data = {'email': '[email protected]', 'pass': '1234'}
+get_agencies = requests.get('http://127.0.0.1:5000/agencies')
+
+
+def describe_GET_agencies_info():
+    def get():
+        response = requests.get('http://127.0.0.1:5000/agencies')
+        return response
+
+    def it_return_agencies_info():
+        response = get()
+        assert response.status_code == 200
+        assert response.json() == [
+            {"name": "Bank for International Settlements", "code": "BIS",
+             "api_base_url": "https://stats.bis.org/api/v1"},
+            {"name": "European Central Bank", "code": "ECB",
+             "api_base_url": "https://sdw-wsrest.ecb.europa.eu"},
+            {"name": "Eurostat", "code": "ESTAT",
+             "api_base_url": "https://ec.europa.eu/eurostat/api/dissemination"},
+            {"name": "International Labour Organization", "code": "ILO",
+             "api_base_url": "https://www.ilo.org/sdmx/rest"}]
+
+
+AGENCIES_CODES = ["BIS", "ESTAT", "ECB", "ILO"]
+
+
+def describe_GET_dataflows():
+    def get(agency_code):
+        url = 'http://127.0.0.1:5000/dataflows/{agency_code}'.format(
+            agency_code)
+        response = requests.get(url)
+        return response
+
+    def context_available_agency():
+        def it_return_agencies_info():
+            agency_code = random.choice(AGENCIES_CODES)
+            response = get(agency_code)
+            assert response.status_code == 200
+            assert response.json()[0].keys() == ["id", "unique_id",
+                                                 "name", "description",
+                                                 "version"]
+
+    def context_not_available_agency():
+        def it_respond_with_error_message():
+            response = get("other_agency")
+            assert response.status_code == 400
+            assert response.json() == "Agency name not allowed"
+
+
+DATA_PARAMS = {"BIS": {"key": "all", "detail": "full"},
+               "ECB": {"key": "all", "detail": "full",
+                       "provider_ref": "all"},
+               "ESTAT": "",
+               "ILO": {"key": "all", "detail": "full",
+                       "include_history": "false"}}
+UNIQUE_ID = {"BIS": "BIS:WS_CBPOL_D(1.0)",
+             "ECB": "ECB:AME(1.0)",
+             "ESTAT": "ESTAT:MED_MA6(1.0)",
+             "ILO": "ILO:DF_CLD_TPOP_SEX_AGE_GEO_NB(1.0)"}
+
+
+def describe_GET_dataflows_data():
+    def get(agency_code, unique_id, params):
+        url = 'http://127.0.0.1:5000/dataflows/data/url/{agency_code}/{unique_id}'.format(
+            agency_code=agency_code, unique_id=unique_id)
+        if params != "":
+            response = requests.get(url, params=params)
+        else:
+            response = requests.get(url)
+        return response
+
+    def context_available_params():
+        def it_return_dataflow_data_url():
+            agency_code = random.choice(AGENCIES_CODES)
+            response = get(agency_code=agency_code,
+                           unique_id=UNIQUE_ID[agency_code],
+                           params=DATA_PARAMS[agency_code])
+            assert response.status_code == 200
+            assert len(response.json()) >= 1
+
+    def context_not_available_agency():
+        def it_respond_with_error_message():
+            response = get(agency_code="other_agency", unique_id="unique_id",
+                           params="")
+            assert response.status_code == 400
+            assert response.json() == "Agency name not allowed"
+
+    def context_not_available_params():
+        def it_respond_with_error_message():
+            agency_code = random.choice(AGENCIES_CODES)
+            response = get(agency_code=agency_code,
+                           unique_id=UNIQUE_ID[agency_code],
+                           params={"other_param": "other_param"})
+            assert response.status_code == 500
+
+
+def describe_GET_dataflows_metadata():
+    def get(agency_code, unique_id):
+        url = 'http://127.0.0.1:5000/dataflows/url/{agency_code}/{unique_id}'.format(
+            agency_code=agency_code, unique_id=unique_id)
+        response = requests.get(url)
+        return response
+
+    def context_available_params():
+        def it_return_dataflow_metadata_url():
+            agency_code = random.choice(AGENCIES_CODES)
+            response = get(agency_code=agency_code,
+                           unique_id=UNIQUE_ID[agency_code])
+            assert response.status_code == 200
+            assert len(response.json()) >= 1
+
+    def context_not_available_agency():
+        def it_respond_with_error_message():
+            response = get(agency_code="other_agency", unique_id="unique_id")
+            assert response.status_code == 400
+            assert response.json() == "Agency name not allowed"
+
+
+def describe_GET_dataflows_code():
+    def get(params):
+        url = 'http://127.0.0.1:5000/dataflows/code?'
+        for key in params:
+            url = f"{url}{key}={params[key]}&"
+        url = url[:-1]
+        response = requests.get(url)
+        return response
+
+    def context_too_much_parameters():
+        def it_respond_with_error_message():
+            url = "https://stats.bis.org/api/v1/data/BIS,WS_CBPOL_D,1.0/all/all?detail=full"
+            params = {'url': url, 'other_parameter': 'other_parameter'}
+            response = get(params)
+            assert response.status_code == 400
+            assert response.json() == "Too much parameters, insert only url parameter"
+
+    def context_missing_url_parameter():
+        def it_respond_with_error_message():
+            params = {'other_parameter': 'other_parameter'}
+            response = get(params)
+            assert response.status_code == 400
+            assert response.json() == "Invalid parameters, insert url parameter"
+
+    def context_empty_url():
+        def it_respond_with_error_message():
+            params = {'url': ''}
+            response = get(params)
+            assert response.status_code == 400
+            assert response.json() == 'Empty url is not allowed'
+
+    def context_invalid_url():
+        def it_return_code():
+            url = "invalid_url"
+            params = {'url': url}
+            response = get(params)
+            assert response.status_code == 400
+            assert response.json() == f"{url} is not a valid url"
+
+    def context_valid_url():
+        def it_return_code():
+            url = "https://stats.bis.org/api/v1/data/BIS,WS_CBPOL_D,1.0/all/all?detail=full"
+            params = {'url': url}
+            response = get(params)
+            assert response.status_code == 200
+            assert response.json() == "from sdmxthon import read_sdmx<br/>" \
+                                      "if __name__ == 'main':<br/>" \
+                                      "&emsp;&emsp;message = read_sdmx('{url}', validate=True)<br/>" \
+                                      "&emsp;&emsp;print(message.content)".format(
+                url=url)