Version 2.3. Changelog and setup.py. Cleaned up code and flake8 forma…

…tting.

.gitignore

@@ -13,3 +13,4 @@ Pipfile.lock
 /testApi.py
 /xml_parsers_test.py
 testApi.py
+pypi.pypirc

Changelog.rst

@@ -2,6 +2,26 @@
 Changelog
 #########
 
+2.3 (2023-09-13)
+----------------
+**Added**
+ - Webservices and data discovery on BIS, ECB, ESTAT, ILO
+
+**Changes**
+
+**Bugfixes**
+ - Fixed duplication detection on ItemScheme. It is based now on ID instead of URN.
+
+2.2 (2023-07-04)
+----------------
+**Added**
+
+**Changes**
+ - International String is now based on str instead of object. NameableArtefacts can use str on __init__ method.
+ - Changed semantic validation to structure validation. Improved error messages and logic.
+
+**Bugfixes**
+
 2.1 (2023-03-14)
 ----------------
 **Added**

pypi.pypirc

@@ -0,0 +1,8 @@
+[distutils]
+  index-servers =
+    pypi
+
+[pypi]
+  repository = https://upload.pypi.org/legacy/
+  username = __token__
+  password = pypi-AgEIcHlwaS5vcmcCJGVlOTVhODk2LTA1NzMtNDI5Ny1hMDRiLTYzMTU1OTE1ZThmNQACEFsxLFsic2RteHRob24iXV0AAixbMixbIjY3ODI5MDExLTJlOTQtNGY5Mi1iOTAwLTE2ZmY2NmI3MWY2OCJdXQAABiDFuRniaohKJzjspEs95LvPbE51YpMaRddbB15miaXwVwpypi-AgEIcHlwaS5vcmcCJGVlOTVhODk2LTA1NzMtNDI5Ny1hMDRiLTYzMTU1OTE1ZThmNQACEFsxLFsic2RteHRob24iXV0AAixbMixbIjY3ODI5MDExLTJlOTQtNGY5Mi1iOTAwLTE2ZmY2NmI3MWY2OCJdXQAABiDFuRniaohKJzjspEs95LvPbE51YpMaRddbB15miaXwVw

sdmxthon/api/api.py

@@ -129,6 +129,7 @@ def xml_to_csv(data, output_path=None, validate=True,
 
         raise ValueError('No Datasets were parsed')
 
+
 def get_supported_agencies():
     "Returns the agencies supported by the API"
     from sdmxthon.webservices import webservices

sdmxthon/model/dataset.py

@@ -315,17 +315,18 @@ def structural_validation(self):
 
         """
         if self.data is None:
-            raise ValueError('The dataset should contain data to perform a structural validation')
+            raise ValueError('The dataset should contain data to perform '
+                             'a structural validation')
         elif self.structure is None:
-            raise ValueError('The dataset should contain a structure to perform a structural validation')
-
+            raise ValueError('The dataset should contain a structure to '
+                             'perform a structural validation')
+
         if not isinstance(self.data, DataFrame):
             raise ValueError(f'Data for dataset {self.structure.id} '
-                         f'is not well formed')
+                             f'is not well formed')
         elif not isinstance(self.structure, DataStructureDefinition):
             raise TypeError('structure must be a DataStructureDefinition')
-
-
+
         return validate_data(self.data, self.structure)
 
     def set_dimension_at_observation(self, dimAtObs):

sdmxthon/webservices/query_builder.py

@@ -22,21 +22,22 @@ def get_data_flows(self, agency_id, resources,
 
     @abstractmethod
     def get_data(self, flow, key=None, provider=None, start_period=None,
-                 end_period=None, updated_after=None, first_n_observations=None,
-                 last_n_observations=None, dimension_at_observation=None,
+                 end_period=None, updated_after=None,
+                 first_n_observations=None, last_n_observations=None,
+                 dimension_at_observation=None,
                  detail=None, include_history=None):
         """
         Returns query to retrieve data
         """
 
     @abstractmethod
     def get_dsds(self, resources, agency_id, version,
-                references=None, detail=None):
+                 references=None, detail=None):
         """
         Returns query to get dsd
         """
 
-    def validate_references(self, reference:str):
+    def validate_references(self, reference: str):
         """
         Validates that the refernce is one of the allowed values
         """
@@ -45,15 +46,15 @@ def validate_references(self, reference:str):
             raise ValueError(f"refernce must be one of the following values: "
                              f"{self.REFERENCES_OPTIONS}")
 
-    def validate_structural_detail(self, detail:str):
+    def validate_structural_detail(self, detail: str):
         """
         Validates that the detail is one of the allowed values
         """
         if detail not in self.STRUCTURE_DETAIL_OPTIONS:
             raise ValueError(f"detail must be one of the following values: "
                              f"{self.STRUCTURE_DETAIL_OPTIONS}")
 
-    def validate_data_detail(self, detail:str):
+    def validate_data_detail(self, detail: str):
         """
         Validates that the detail is one of the allwoed values
         """
@@ -70,98 +71,102 @@ class SdmxWs2p0(SdmxWebservice):
     REFERENCES_OPTIONS = ['none', 'parents', 'parentsandsiblings',
                           'children', 'descendants', 'all']
 
-    STRUCTURE_DETAIL_OPTIONS = ['allstubs', 'referencestubs', 'referencepartial',
-                      'allcompletestubs', 'referencecompletestubs',
-                      'full']
+    STRUCTURE_DETAIL_OPTIONS = ['allstubs', 'referencestubs',
+                                'referencepartial',
+                                'allcompletestubs', 'referencecompletestubs',
+                                'full']
 
     def get_data_flows(self, agency_id, resources=None,
                        version=None, references=None, detail=None) -> str:
-
         resources = resources if resources else "all"
         version = version if version else "latest"
 
         base_query = f"dataflow/{agency_id}/{resources}/{version}"
         references_query = f"?references={references}" if references else ""
         detail_query = f"?detail={detail}" if detail else ""
 
-        return base_query + references_query +detail_query
-
+        return base_query + references_query + detail_query
 
     def get_data(self, flow, key=None, provider=None, start_period=None,
-                 end_period=None, updated_after=None, first_n_observations=None,
-                 last_n_observations=None, dimension_at_observation=None,
+                 end_period=None, updated_after=None,
+                 first_n_observations=None, last_n_observations=None,
+                 dimension_at_observation=None,
                  detail=None, include_history=None):
-
         raise NotImplementedError("get_data not implemented for SDMX 2.0")
 
 
 class SdmxWs1(SdmxWebservice):
     """
-    Generic Sdmx Ws 1 implementation for 
-    queries that do not change
+    Generic Sdmx Ws 1 implementation for queries that do not change
     """
 
     REFERENCES_OPTIONS = ['none', 'parents', 'parentsandsiblings',
                           'children', 'descendants', 'all']
 
-    STRUCTURE_DETAIL_OPTIONS = ['allstubs', 'referencestubs', 'referencepartial',
-                      'allcompletestubs', 'referencecompletestubs',
-                      'full']
+    STRUCTURE_DETAIL_OPTIONS = ['allstubs', 'referencestubs',
+                                'referencepartial',
+                                'allcompletestubs', 'referencecompletestubs',
+                                'full']
 
     def get_data_flows(self, agency_id=None, resources=None,
                        version=None, references=None, detail=None) -> str:
-
         resources = resources if resources else "all"
         version = version if version else "latest"
         agency_id = agency_id if agency_id else "all"
 
         base_query = f"/dataflow/{agency_id}/{resources}/{version}"
-        references_query = f"?references={references}" if references else ""
-        detail_query = f"?detail={detail}" if detail else ""
-
-        return base_query + references_query +detail_query
+        params = ""
+        if references:
+            params += f"?references={references}"
+        if detail:
+            params += f"?detail={detail}"
 
+        return base_query + params
 
     def get_data(self, flow, key=None, provider=None, start_period=None,
-                 end_period=None, updated_after=None, first_n_observations=None,
-                 last_n_observations=None, dimension_at_observation=None,
+                 end_period=None, updated_after=None,
+                 first_n_observations=None, last_n_observations=None,
+                 dimension_at_observation=None,
                  detail=None, include_history=None) -> str:
-
         key = key if key else 'all'
         provider = provider if provider else 'all'
 
         base_query = f"/data/{flow}/{key}/{provider}"
-        start_period_query = f"?startPeriod={start_period}" if start_period else ""
-        end_period_query = f"?endPeriod={end_period}" if end_period else ""
-        updated_after_query = f"?updatedAfter={updated_after}" if updated_after else ""
-        first_n_observations_query = f"?firstNObservations={first_n_observations}" \
-            if first_n_observations else ""
-        last_n_observations_query = f"?lastNObservations={last_n_observations}" \
-            if last_n_observations else ""
-        dimension_at_observation_query = f"?dimensionAtObservation={dimension_at_observation}" \
-            if dimension_at_observation else ""
-        detail_query = f"?detail={detail}" if detail else ""
-        include_history_query = f"?includeHistory={include_history}" if include_history else ""
 
-        return base_query + start_period_query + end_period_query + \
-            updated_after_query + first_n_observations_query + \
-            last_n_observations_query + dimension_at_observation_query +\
-            detail_query + include_history_query
+        params = ""
+        if start_period:
+            params += f"?startPeriod={start_period}"
+        if end_period:
+            params += f"?endPeriod={end_period}"
+        if updated_after:
+            params += f"?updatedAfter={updated_after}"
+        if first_n_observations:
+            params += f"?firstNObservations={first_n_observations}"
+        if last_n_observations:
+            params += f"?lastNObservations={last_n_observations}"
+        if dimension_at_observation:
+            params += f"?dimensionAtObservation={dimension_at_observation}"
+        if detail:
+            params += f"?detail={detail}"
+        if include_history:
+            params += f"?includeHistory={include_history}"
 
+        return base_query + params
 
     def get_dsds(self, resources=None, agency_id=None, version=None,
-                references=None, detail=None):
-
+                 references=None, detail=None):
         resources = resources if resources else "all"
         version = version if version else "latest"
         agency_id = agency_id if agency_id else "all"
 
-
         base_query = f"/datastructure/{agency_id}/{resources}/{version}"
-        references_query = f"?references={references}" if references else ""
-        detail_query = f"?detail={detail}" if detail else ""
+        params = ""
+        if references:
+            params += f"?references={references}"
+        if detail:
+            params += f"?detail={detail}"
 
-        return base_query + references_query + detail_query
+        return base_query + params
 
 
 class SdmxWs1p5(SdmxWs1):
@@ -186,7 +191,6 @@ def __init__(self, ws_implementation):
             raise TypeError("QueryBuilder requies an SdmxWebservice object")
         self._ws_implementation = ws_implementation
 
-
     def id_builder(self, ids=None) -> str:
         "returns the string for the list of resources ids"
         if isinstance(ids, str):
@@ -197,8 +201,7 @@ def id_builder(self, ids=None) -> str:
             return "all"
         raise TypeError("Ids has to be string, list or None")
 
-
-    def get_data_flows(self, agency_id=None, resources=None, 
+    def get_data_flows(self, agency_id=None, resources=None,
                        version=None, references=None, detail=None) -> str:
         "Returns the get data flows query for the WS Implementation"
         resources = self.id_builder(resources)
@@ -213,9 +216,8 @@ def get_data_flows(self, agency_id=None, resources=None,
             agency_id, resources, version,
             references, detail)
 
-
-    def get_dsds(self, agency_id=None, resources=None, 
-                       version=None, references=None, detail=None) -> str:
+    def get_dsds(self, agency_id=None, resources=None,
+                 version=None, references=None, detail=None) -> str:
         "Returns the get data structures query for the WS Implementation"
         resources = self.id_builder(resources)
         agency_id = agency_id if agency_id else "all"
@@ -229,17 +231,21 @@ def get_dsds(self, agency_id=None, resources=None,
             agency_id, resources, version,
             references, detail)
 
-
     def get_data(self, flow, key=None, provider=None, start_period=None,
-                 end_period=None, updated_after=None, first_n_observations=None,
-                 last_n_observations=None, dimension_at_observation=None,
+                 end_period=None, updated_after=None,
+                 first_n_observations=None, last_n_observations=None,
+                 dimension_at_observation=None,
                  detail=None, include_history=None):
         "Returns the data query for the WS Implementation"
 
         provider = self.id_builder(provider)
         if detail:
             self._ws_implementation.validate_data_detail(detail)
 
-        return self._ws_implementation.get_data(flow, key,provider,
-            start_period,end_period, updated_after, first_n_observations, 
-            last_n_observations, dimension_at_observation, detail, include_history)
+        return self._ws_implementation.get_data(flow, key, provider,
+                                                start_period, end_period,
+                                                updated_after,
+                                                first_n_observations,
+                                                last_n_observations,
+                                                dimension_at_observation,
+                                                detail, include_history)