From 285bcceb9e91bfbbb52f51717c2ffd4869c75e35 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Sun, 9 Jul 2017 13:59:55 -0600 Subject: [PATCH 1/7] ENH: Add ability to access datasets by index (Fixes #134) When getting an item by string, work as before. For all else, treat as an index into values. Works for catalog refs as well. --- siphon/catalog.py | 16 ++++++++++++++-- siphon/tests/test_catalog.py | 11 +++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/siphon/catalog.py b/siphon/catalog.py index 9e4d8d2e4..0c2742674 100644 --- a/siphon/catalog.py +++ b/siphon/catalog.py @@ -24,6 +24,18 @@ log.setLevel(logging.ERROR) +class IndexableMapping(OrderedDict): + """Extend ``OrderedDict`` to allow index-based access to values.""" + + def __getitem__(self, item): + """Return an item either by index or name.""" + try: + item + '' # Raises if item not a string + return super(IndexableMapping, self).__getitem__(item) + except TypeError: + return list(self.values())[item] + + class TDSCatalog(object): """ Parse information from a THREDDS Client Catalog. @@ -79,9 +91,9 @@ def __init__(self, catalog_url): root = ET.fromstring(resp.text) self.catalog_name = root.attrib.get('name', 'No name found') - self.datasets = OrderedDict() + self.datasets = IndexableMapping() self.services = [] - self.catalog_refs = OrderedDict() + self.catalog_refs = IndexableMapping() self.metadata = {} self.ds_with_access_elements_to_process = [] service_skip_count = 0 diff --git a/siphon/tests/test_catalog.py b/siphon/tests/test_catalog.py index 305a9a534..82c0befce 100644 --- a/siphon/tests/test_catalog.py +++ b/siphon/tests/test_catalog.py @@ -116,6 +116,17 @@ def test_datasets_order(): 'Latest Collection for NAM CONUS 20km'] +@recorder.use_cassette('top_level_20km_rap_catalog') +def test_datasets_get_by_index(): + """Test that datasets can be accessed by index.""" + url = ('http://thredds.ucar.edu/thredds/catalog/grib/NCEP/NAM/' + 'CONUS_20km/noaaport/catalog.xml') + cat = TDSCatalog(url) + assert cat.datasets[0].name == 'Full Collection (Reference / Forecast Time) Dataset' + assert cat.datasets[1].name == 'Best NAM CONUS 20km Time Series' + assert cat.datasets[2].name == 'Latest Collection for NAM CONUS 20km' + + @recorder.use_cassette('top_level_cat') def test_catalog_ref_order(): """Test that catalog references are properly ordered.""" From ee47ff8323b6f102f544b7da3dc874707ecc1839 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Tue, 18 Jul 2017 18:56:35 -0600 Subject: [PATCH 2/7] ENH: Enable extracting datasets using datetimes (Fixes #135) Parses the keys for datasets and catalog refs and extracts a datetime using a regex. Then filter using nearest time or a time range. --- siphon/catalog.py | 104 +++++++++++++++++++++++++++++++++-- siphon/tests/test_catalog.py | 55 +++++++++++++++++- 2 files changed, 153 insertions(+), 6 deletions(-) diff --git a/siphon/catalog.py b/siphon/catalog.py index 0c2742674..34207f98f 100644 --- a/siphon/catalog.py +++ b/siphon/catalog.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2015 University Corporation for Atmospheric Research/Unidata. +# Copyright (c) 2013-2017 University Corporation for Atmospheric Research/Unidata. # Distributed under the terms of the MIT License. # SPDX-License-Identifier: MIT """ @@ -8,7 +8,9 @@ """ from collections import OrderedDict +from datetime import datetime import logging +import re import xml.etree.ElementTree as ET try: from urlparse import urljoin, urlparse @@ -36,6 +38,98 @@ def __getitem__(self, item): return list(self.values())[item] +class DatasetCollection(IndexableMapping): + """Extend ``IndexableMapping`` to allow datetime-based filter queries.""" + + default_regex = re.compile(r'(?P\d{4})(?P[01]\d)(?P[012]\d)_' + r'(?P[012]\d)(?P[0-5]\d)') + + def _get_datasets_with_times(self, regex): + # Set the default regex if we don't have one + if regex is None: + regex = self.default_regex + else: + regex = re.compile(regex) + + # Loop over the collection looking for keys that match our regex + found_date = False + for ds in self: + match = regex.search(ds) + + # If we find one, make a datetime and yield it along with the value + if match: + found_date = True + date_parts = match.groupdict() + dt = datetime(int(date_parts.get('year', 0)), int(date_parts.get('month', 0)), + int(date_parts.get('day', 0)), int(date_parts.get('hour', 0)), + int(date_parts.get('minute', 0)), + int(date_parts.get('second', 0)), + int(date_parts.get('microsecond', 0))) + yield dt, self[ds] + + # If we never found any keys that match, we should let the user know that rather + # than have it be the same as if nothing matched filters + if not found_date: + raise ValueError('No datasets with times found.') + + def filter_time_nearest(self, time, regex=None): + """Filter keys for an item closest to the desired time. + + Loops over all keys in the collection and uses `regex` to extract and build + `datetime`s. The collection of `datetime`s is compared to `start` and the value that + has a `datetime` closest to that requested is returned.If none of the keys in the + collection match the regex, indicating that the keys are not date/time-based, + a ``ValueError`` is raised. + + Parameters + ---------- + time : ``datetime.datetime`` + The desired time + regex : str, optional + The regular expression to use to extract date/time information from the key. If + given, this should contain named groups: 'year', 'month', 'day', 'hour', 'minute', + 'second', and 'microsecond', as appropriate. When a match is found, any of those + groups missing from the pattern will be assigned a value of 0. The default pattern + looks for patterns like: 20171118_2356. + + Returns + ------- + The value with a time closest to that desired + + """ + return min(self._get_datasets_with_times(regex), + key=lambda i: abs((i[0] - time).total_seconds()))[-1] + + def filter_time_range(self, start, end, regex=None): + """Filter keys for all items within the desired time range. + + Loops over all keys in the collection and uses `regex` to extract and build + `datetime`s. From the collection of `datetime`s, all values within `start` and `end` + (inclusive) are returned. If none of the keys in the collection match the regex, + indicating that the keys are not date/time-based, a ``ValueError`` is raised. + + Parameters + ---------- + start : ``datetime.datetime`` + The start of the desired time range, inclusive + end : ``datetime.datetime`` + The end of the desired time range, inclusive + regex : str, optional + The regular expression to use to extract date/time information from the key. If + given, this should contain named groups: 'year', 'month', 'day', 'hour', 'minute', + 'second', and 'microsecond', as appropriate. When a match is found, any of those + groups missing from the pattern will be assigned a value of 0. The default pattern + looks for patterns like: 20171118_2356. + + Returns + ------- + All values corresponding to times within the specified range + + """ + return [item[-1] for item in self._get_datasets_with_times(regex) + if start <= item[0] <= end] + + class TDSCatalog(object): """ Parse information from a THREDDS Client Catalog. @@ -46,12 +140,12 @@ class TDSCatalog(object): The url path of the catalog to parse. base_tds_url : str The top level server address - datasets : dict[str, Dataset] + datasets : DatasetCollection[str, Dataset] A dictionary of :class:`Dataset` objects, whose keys are the name of the dataset's name services : List A list of :class:`SimpleService` listed in the catalog - catalog_refs : dict[str, CatalogRef] + catalog_refs : DatasetCollection[str, CatalogRef] A dictionary of :class:`CatalogRef` objects whose keys are the name of the catalog ref title. @@ -91,9 +185,9 @@ def __init__(self, catalog_url): root = ET.fromstring(resp.text) self.catalog_name = root.attrib.get('name', 'No name found') - self.datasets = IndexableMapping() + self.datasets = DatasetCollection() self.services = [] - self.catalog_refs = IndexableMapping() + self.catalog_refs = DatasetCollection() self.metadata = {} self.ds_with_access_elements_to_process = [] service_skip_count = 0 diff --git a/siphon/tests/test_catalog.py b/siphon/tests/test_catalog.py index 82c0befce..b9194ed0a 100644 --- a/siphon/tests/test_catalog.py +++ b/siphon/tests/test_catalog.py @@ -1,11 +1,14 @@ -# Copyright (c) 2013-2015 University Corporation for Atmospheric Research/Unidata. +# Copyright (c) 2013-2017 University Corporation for Atmospheric Research/Unidata. # Distributed under the terms of the MIT License. # SPDX-License-Identifier: MIT """Test the catalog access API.""" +from datetime import datetime import logging import warnings +import pytest + from siphon.catalog import get_latest_access_url, TDSCatalog from siphon.testing import get_recorder @@ -127,6 +130,56 @@ def test_datasets_get_by_index(): assert cat.datasets[2].name == 'Latest Collection for NAM CONUS 20km' +@recorder.use_cassette('top_level_20km_rap_catalog') +def test_datasets_nearest_time(): + """Test getting dataset by time using filenames.""" + url = ('http://thredds.ucar.edu/thredds/catalog/grib/NCEP/NAM/' + 'CONUS_20km/noaaport/catalog.xml') + cat = TDSCatalog(url) + nearest = cat.catalog_refs.filter_time_nearest(datetime(2015, 5, 28, 17)) + assert nearest.title == 'NAM_CONUS_20km_noaaport_20150528_1800.grib1' + + +@recorder.use_cassette('top_level_20km_rap_catalog') +def test_datasets_nearest_time_raises(): + """Test getting dataset by time using filenames.""" + url = ('http://thredds.ucar.edu/thredds/catalog/grib/NCEP/NAM/' + 'CONUS_20km/noaaport/catalog.xml') + cat = TDSCatalog(url) + + # Datasets doesn't have any timed datasets + with pytest.raises(ValueError): + cat.datasets.filter_time_nearest(datetime(2015, 5, 28, 17)) + + +@recorder.use_cassette('top_level_20km_rap_catalog') +def test_datasets_time_range(): + """Test getting datasets by time range using filenames.""" + url = ('http://thredds.ucar.edu/thredds/catalog/grib/NCEP/NAM/' + 'CONUS_20km/noaaport/catalog.xml') + cat = TDSCatalog(url) + in_range = cat.catalog_refs.filter_time_range(datetime(2015, 5, 28, 0), + datetime(2015, 5, 29, 0)) + titles = [item.title for item in in_range] + assert titles == ['NAM_CONUS_20km_noaaport_20150528_0000.grib1', + 'NAM_CONUS_20km_noaaport_20150528_0600.grib1', + 'NAM_CONUS_20km_noaaport_20150528_1200.grib1', + 'NAM_CONUS_20km_noaaport_20150528_1800.grib1', + 'NAM_CONUS_20km_noaaport_20150529_0000.grib1'] + + +@recorder.use_cassette('top_level_20km_rap_catalog') +def test_datasets_time_range_raises(): + """Test getting datasets by time range using filenames.""" + url = ('http://thredds.ucar.edu/thredds/catalog/grib/NCEP/NAM/' + 'CONUS_20km/noaaport/catalog.xml') + cat = TDSCatalog(url) + + # No time-based dataset names + with pytest.raises(ValueError): + cat.datasets.filter_time_range(datetime(2015, 5, 28, 0), datetime(2015, 5, 29, 0)) + + @recorder.use_cassette('top_level_cat') def test_catalog_ref_order(): """Test that catalog references are properly ordered.""" From 01fedf9039c0493298153dd6720180fc78bc7141 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Wed, 19 Jul 2017 18:07:17 -0600 Subject: [PATCH 3/7] ENH: Adding simpler access methods (Fixes #136) --- siphon/catalog.py | 114 +++++++++++++++++++++++++ siphon/tests/fixtures/cat_to_cdmr | 77 +++++++++++++++++ siphon/tests/fixtures/cat_to_open | 67 +++++++++++++++ siphon/tests/fixtures/cat_to_opendap | 39 +++++++++ siphon/tests/fixtures/cat_to_subset | 123 +++++++++++++++++++++++++++ siphon/tests/test_catalog_access.py | 117 +++++++++++++++++++++++++ 6 files changed, 537 insertions(+) create mode 100644 siphon/tests/fixtures/cat_to_cdmr create mode 100644 siphon/tests/fixtures/cat_to_open create mode 100644 siphon/tests/fixtures/cat_to_opendap create mode 100644 siphon/tests/fixtures/cat_to_subset create mode 100644 siphon/tests/test_catalog_access.py diff --git a/siphon/catalog.py b/siphon/catalog.py index 34207f98f..068c53e02 100644 --- a/siphon/catalog.py +++ b/siphon/catalog.py @@ -448,6 +448,120 @@ def add_access_element_info(self, access_element): url_path = access_element.attrib['urlPath'] self.access_element_info[service_name] = url_path + def download(self, filename): + """Download the dataset to a local file. + + Parameters + ---------- + filename : str + The full path to which the dataset will be saved + + """ + with self.remote_open() as infile: + with open(filename, 'wb') as outfile: + outfile.write(infile.read()) + + def remote_open(self): + """Open the remote dataset for random access. + + Get a file-like object for reading from the remote dataset, providing random access, + similar to a local file. + + Returns + ------- + A random access, file-like object + + """ + return self.access_with_service('HTTPServer') + + def remote_access(self, service=None): + """Access the remote dataset. + + Open the remote dataset and get a netCDF4-compatible `Dataset` object providing + index-based subsetting capabilities. + + Parameters + ---------- + service : str, optional + The name of the service to use for access to the dataset, either + 'CdmRemote' or 'OPENDAP'. Defaults to 'CdmRemote'. + + Returns + ------- + Dataset + Object for netCDF4-like access to the dataset + + """ + if service is None: + service = 'CdmRemote' if 'CdmRemote' in self.access_urls else 'OPENDAP' + + if service not in ('CdmRemote', 'OPENDAP'): + raise ValueError(service + ' is not a valid service for remote_access') + + return self.access_with_service(service) + + def subset(self, service=None): + """Subset the dataset. + + Open the remote dataset and get a client for talking to ``service``. + + Parameters + ---------- + service : str, optional + The name of the service for subsetting the dataset. Defaults to 'NetcdfSubset'. + + Returns + ------- + a client for communicating using ``service`` + + """ + if service is None: + service = 'NetcdfSubset' + + if service not in ('NetcdfSubset',): + raise ValueError(service + ' is not a valid service for subset') + + return self.access_with_service(service) + + def access_with_service(self, service): + """Access the dataset using a particular service. + + Return an Python object capable of communicating with the server using the particular + service. For instance, for 'HTTPServer' this is a file-like object capable of + HTTP communication; for OPENDAP this is a netCDF4 dataset. + + Parameters + ---------- + service : str + The name of the service for accessing the dataset + + Returns + ------- + An instance appropriate for communicating using ``service``. + + """ + if service == 'CdmRemote': + from .cdmr import Dataset as CDMRDataset + provider = CDMRDataset + elif service == 'OPENDAP': + try: + from netCDF4 import Dataset as NC4Dataset + provider = NC4Dataset + except ImportError: + raise ImportError('OPENDAP access requires netCDF4-python to be installed.') + elif service == 'NetcdfSubset': + from .ncss import NCSS + provider = NCSS + elif service == 'HTTPServer': + provider = urlopen + else: + raise ValueError(service + ' is not an access method supported by Siphon') + + try: + return provider(self.access_urls[service]) + except KeyError: + raise ValueError(service + ' is not available for this dataset') + class SimpleService(object): """Hold information about an access service enabled on a dataset. diff --git a/siphon/tests/fixtures/cat_to_cdmr b/siphon/tests/fixtures/cat_to_cdmr new file mode 100644 index 000000000..75f307679 --- /dev/null +++ b/siphon/tests/fixtures/cat_to_cdmr @@ -0,0 +1,77 @@ +interactions: +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + User-Agent: [Siphon (0.4.1+25.g2ba7c2e.dirty)] + method: GET + uri: http://thredds.ucar.edu/thredds/catalog/nexrad/level3/NMD/FTG/20170719/catalog.xml?dataset=NWS/NEXRAD3/NMD/FTG/20170719/Level3_FTG_NMD_20170719_2337.nids + response: + body: + string: !!binary | + H4sIAAAAAAAAA5xVW2/aMBR+n7T/YHnPxKRZx0AhHSJlQ2opKqBNe0EmdsFaEkeOw2W/fse5AaFU + 1d6Sc77znfuxe7ePQrTlKhUy7mPbamPE40AyEa/7eDEftb7iO+/jBzegmoZyjQAep3280TrpEbLb + 7awsFgyUVhZQZXGWkZhGPE1owFOiN4ozlpJxvB0WBGSb+8hpevtQxH/OyHaOJdWa2N1ul+RajAxd + Hz/wLQ+d5Wj+fTl59Jc3bbvT7tjd5Y3jdCyIIMVnWVg2hqgRclOutiLgJYuiTNAQo1I6PyQgHMoo + kVnMMFrRFP4LywvbOGDSuDmzfZreT/zBtDKtEzbQIcGIvM71Yz6fzkDCVYPvVNGgfBEhL1TXeQMW + KR5JzZspsui5lDdYa4vrpHEQNUs2GT4+XFAZ3HWWjLGgwbLw/eEFi8FdZxGpbJCMZ08XHICqKVxS + wos/M6op1++fqrEP6f6ckcn9r+eB7xDAEcCTCkvewZGpcEr1BkrJ9zCCJMxN/oeqqopJYyb+cgTb + p2EfVwfNQWt/ubXaLqm0J2iOdF6uCFb7RXCGPcPeandadnd+4/Q+Oz3H/p2b1mZaRHwoYeDoupKZ + hmiqdMPa6fTa4LdQ1UiWKaphI71bFIk4gwiBv5KVPsgrTtyIa2pyQCLecCU0Z32sVcbxSRRFUyfQ + Rq/Y6rrRuawG0kxvJHAcPLhNxZEqL5ZLjrpjzKAwg+VBs8eDh6KWueAMMpIqotorhqLlFLBSeATK + IIt4rPOEUX7OejChL2+fT/NHPkFOFG5AYaSFDs2+FFCU61D+mUDVjsty4bPouRLrjYbpGCnOwwOi + WypCugo5RH2KPpIk2SoUKZS+FoHQbAzayoCuspCqQx/74xH2FsPBM1lMxv5gPnDz039qE0jgDrRZ + gLeThlcngqj6OM2SRCr9rYk4S5I0A3RJNTLlzpdrbl4uUj5d8P0PAAD//wMA2aScMu4GAAA= + headers: + Access-Control-Allow-Origin: ['*'] + Connection: [Keep-Alive] + Content-Encoding: [gzip] + Content-Language: [en] + Content-Type: [application/xml;charset=UTF-8] + Date: ['Fri, 21 Jul 2017 19:33:40 GMT'] + Keep-Alive: ['timeout=5, max=100'] + Server: [Apache] + Vary: [Accept-Encoding] + X-Frame-Options: [SAMEORIGIN] + status: {code: 200, message: '200'} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + User-Agent: [Siphon (0.4.1+25.g2ba7c2e.dirty)] + method: GET + uri: http://thredds.ucar.edu/thredds/cdmremote/nexrad/level3/NMD/FTG/20170719/Level3_FTG_NMD_20170719_2337.nids?req=header + response: + body: + string: !!binary | + H4sIAAAAAAAAA3RTTU8TURRtQaB9FCwfQTCKIy4wL3SmZTCFuhE6SEygJQxKwmbymHm0ozPz6ps3 + LfwG/4G6ceHChRsXuDJRlpq4Uhcu1cSFbtyZaIKv81EKqV00zT33nnvuO6fPf77//DcJ7lYZqxUk + iVUpNgxX9HRERWx4UUHSDZtimzAsOXiPIkOycB1bslRaU6SbmyvSbDaXz+ZzC9KqX9d4TeOYFtW1 + WVnOi45puFMPkiAGBdDDTGbhdGw8DkdKPqfgzwqyoCCG4EWQuIf3G4Qart80sKVuZObnletC6Zai + wgsgpVOMGKGag+yAJ1EqLy5KpS0VroD+CPWo5YNz4YmNRkN0dEMXHYKQWCF1iSCJb0c0+K5RYng6 + c8Uqsy14GaQ5velUNOSxKqEm2/fZksGqolKEQ6BPMeumgWm6azw+1XV0xLUPbDS5VhE/0jNwupcD + iRvFb4+eflV/w0kwGMDEqbThr7enfj37891ujS9aJ8YfvjxYfvL2DZwAg+uBSJXxBcTxFXXzJ4ci + GD4JlaK3GVWwU8dUWiKexaXOFMszt1VOdbZcw9RvRdYa4bv8G2Jd8AqYuEMsz8ZFwudQBa8jxjAN + GIOmD3AcDKr4vocdHZc8eyd6goEkPA/SwbiqI6cdi6XgNOjnFuNi0yFsBPKaQclk85ncwuasXJiT + C9ncdlPeMg9acEk7RwyOgVRQKe+uoZob1OMxuAT6XM+2EQ18ksNoWWG0DB4tAVEshGkSfM+FlumO + Z1k8nCNR9LQ60dGOZ0V8PZhSQuElni/C39NpKgvymdKKhE+YDr/J5cp7dwm1EfOx/tXw39JM7iQY + rmDi1vhVyNIsxDSer3QP139maezLZCcc7YX4dOXxaZw4x/OH7w6ynfBo/vDT3g9u7Lk2nHvLTJ3/ + aJEoH1+9+G9TxOQ3CWDUz+lpj7qbHTH+gRIYYaaNNT3MkOYyRFknx+V8IXttG2bA0MkB7HQMSNg+ + ChJNR4/F+1tb1UhtVDVdrpcfdKzwavwfAAAA//8DAPnaGqMCBQAA + headers: + Access-Control-Allow-Origin: ['*'] + Connection: [Keep-Alive] + Content-Description: [ncstream] + Content-Encoding: [gzip] + Content-Type: [application/octet-stream] + Date: ['Fri, 21 Jul 2017 19:33:40 GMT'] + Keep-Alive: ['timeout=5, max=100'] + Server: [Apache] + Vary: [Accept-Encoding] + X-Frame-Options: [SAMEORIGIN] + status: {code: 200, message: '200'} +version: 1 diff --git a/siphon/tests/fixtures/cat_to_open b/siphon/tests/fixtures/cat_to_open new file mode 100644 index 000000000..638b221cc --- /dev/null +++ b/siphon/tests/fixtures/cat_to_open @@ -0,0 +1,67 @@ +interactions: +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + User-Agent: [Siphon (0.4.1+19.g2bea7f3.dirty)] + method: GET + uri: http://thredds.ucar.edu/thredds/catalog/nexrad/level3/NMD/FTG/20170719/catalog.xml?dataset=NWS/NEXRAD3/NMD/FTG/20170719/Level3_FTG_NMD_20170719_2337.nids + response: + body: + string: !!binary | + H4sIAAAAAAAAA5xVW2/aMBR+n7T/YHnPxKRZx0AhHSJlQ2opKqBNe0EmdsFaEkeOw2W/fse5AaFU + 1d6Sc77znfuxe7ePQrTlKhUy7mPbamPE40AyEa/7eDEftb7iO+/jBzegmoZyjQAep3280TrpEbLb + 7awsFgyUVhZQZXGWkZhGPE1owFOiN4ozlpJxvB0WBGSb+8hpevtQxH/OyHaOJdWa2N1ul+RajAxd + Hz/wLQ+d5Wj+fTl59Jc3bbvT7tjd5Y3jdCyIIMVnWVg2hqgRclOutiLgJYuiTNAQo1I6PyQgHMoo + kVnMMFrRFP4LywvbOGDSuDmzfZreT/zBtDKtEzbQIcGIvM71Yz6fzkDCVYPvVNGgfBEhL1TXeQMW + KR5JzZspsui5lDdYa4vrpHEQNUs2GT4+XFAZ3HWWjLGgwbLw/eEFi8FdZxGpbJCMZ08XHICqKVxS + wos/M6op1++fqrEP6f6ckcn9r+eB7xDAEcCTCkvewZGpcEr1BkrJ9zCCJMxN/oeqqopJYyb+cgTb + p2EfVwfNQWt/ubXaLqm0J2iOdF6uCFb7RXCGPcPeandadnd+4/Q+Oz3H/p2b1mZaRHwoYeDoupKZ + hmiqdMPa6fTa4LdQ1UiWKaphI71bFIk4gwiBv5KVPsgrTtyIa2pyQCLecCU0Z32sVcbxSRRFUyfQ + Rq/Y6rrRuawG0kxvJHAcPLhNxZEqL5ZLjrpjzKAwg+VBs8eDh6KWueAMMpIqotorhqLlFLBSeATK + IIt4rPOEUX7OejChL2+fT/NHPkFOFG5AYaSFDs2+FFCU61D+mUDVjsty4bPouRLrjYbpGCnOwwOi + WypCugo5RH2KPpIk2SoUKZS+FoHQbAzayoCuspCqQx/74xH2FsPBM1lMxv5gPnDz039qE0jgDrRZ + gLeThlcngqj6OM2SRCr9rYk4S5I0A3RJNTLlzpdrbl4uUj5d8P0PAAD//wMA2aScMu4GAAA= + headers: + Access-Control-Allow-Origin: ['*'] + Connection: [Keep-Alive] + Content-Encoding: [gzip] + Content-Language: [en] + Content-Type: [application/xml;charset=UTF-8] + Date: ['Wed, 19 Jul 2017 23:50:19 GMT'] + Keep-Alive: ['timeout=5, max=100'] + Server: [Apache] + Vary: [Accept-Encoding] + X-Frame-Options: [SAMEORIGIN] + status: {code: 200, message: '200'} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + User-Agent: [Siphon (0.4.1+19.g2bea7f3.dirty)] + method: GET + uri: http://thredds.ucar.edu/thredds/fileServer/nexrad/level3/NMD/FTG/20170719/Level3_FTG_NMD_20170719_2337.nids + response: + body: + string: !!binary | + H4sIAAAAAAAAA2Lk5eUyNTNSAFLBLqHBxqYK3k7+oQqGlkbGxuZAQT9fF7cQdyCDodf5OgOjbxsD + A0MFYzSQZPr/n4Fhdtb/f+nzRF8x9DIwMVzh5WTgAanyUQSrbQWqYgRiVgaSACMKD2g1MwAAAP// + AwCw7dZ/pQAAAA== + headers: + Accept-Ranges: [bytes] + Access-Control-Allow-Origin: ['*'] + Connection: [Keep-Alive] + Content-Disposition: [attachment; filename="Level3_FTG_NMD_20170719_2337.nids"] + Content-Encoding: [gzip] + Content-Type: [application/octet-stream] + Date: ['Wed, 19 Jul 2017 23:50:19 GMT'] + Keep-Alive: ['timeout=5, max=100'] + Server: [Apache] + Vary: [Accept-Encoding] + X-Frame-Options: [SAMEORIGIN] + status: {code: 200, message: '200'} +version: 1 diff --git a/siphon/tests/fixtures/cat_to_opendap b/siphon/tests/fixtures/cat_to_opendap new file mode 100644 index 000000000..5bc23dea7 --- /dev/null +++ b/siphon/tests/fixtures/cat_to_opendap @@ -0,0 +1,39 @@ +interactions: +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + User-Agent: [Siphon (0.4.1+19.g2bea7f3.dirty)] + method: GET + uri: http://thredds.ucar.edu/thredds/catalog/nexrad/level3/NMD/FTG/20170719/catalog.xml?dataset=NWS/NEXRAD3/NMD/FTG/20170719/Level3_FTG_NMD_20170719_2337.nids + response: + body: + string: !!binary | + H4sIAAAAAAAAA5xVW2/aMBR+n7T/YHnPxKRZx0AhHSJlQ2opKqBNe0EmdsFaEkeOw2W/fse5AaFU + 1d6Sc77znfuxe7ePQrTlKhUy7mPbamPE40AyEa/7eDEftb7iO+/jBzegmoZyjQAep3280TrpEbLb + 7awsFgyUVhZQZXGWkZhGPE1owFOiN4ozlpJxvB0WBGSb+8hpevtQxH/OyHaOJdWa2N1ul+RajAxd + Hz/wLQ+d5Wj+fTl59Jc3bbvT7tjd5Y3jdCyIIMVnWVg2hqgRclOutiLgJYuiTNAQo1I6PyQgHMoo + kVnMMFrRFP4LywvbOGDSuDmzfZreT/zBtDKtEzbQIcGIvM71Yz6fzkDCVYPvVNGgfBEhL1TXeQMW + KR5JzZspsui5lDdYa4vrpHEQNUs2GT4+XFAZ3HWWjLGgwbLw/eEFi8FdZxGpbJCMZ08XHICqKVxS + wos/M6op1++fqrEP6f6ckcn9r+eB7xDAEcCTCkvewZGpcEr1BkrJ9zCCJMxN/oeqqopJYyb+cgTb + p2EfVwfNQWt/ubXaLqm0J2iOdF6uCFb7RXCGPcPeandadnd+4/Q+Oz3H/p2b1mZaRHwoYeDoupKZ + hmiqdMPa6fTa4LdQ1UiWKaphI71bFIk4gwiBv5KVPsgrTtyIa2pyQCLecCU0Z32sVcbxSRRFUyfQ + Rq/Y6rrRuawG0kxvJHAcPLhNxZEqL5ZLjrpjzKAwg+VBs8eDh6KWueAMMpIqotorhqLlFLBSeATK + IIt4rPOEUX7OejChL2+fT/NHPkFOFG5AYaSFDs2+FFCU61D+mUDVjsty4bPouRLrjYbpGCnOwwOi + WypCugo5RH2KPpIk2SoUKZS+FoHQbAzayoCuspCqQx/74xH2FsPBM1lMxv5gPnDz039qE0jgDrRZ + gLeThlcngqj6OM2SRCr9rYk4S5I0A3RJNTLlzpdrbl4uUj5d8P0PAAD//wMA2aScMu4GAAA= + headers: + Access-Control-Allow-Origin: ['*'] + Connection: [Keep-Alive] + Content-Encoding: [gzip] + Content-Language: [en] + Content-Type: [application/xml;charset=UTF-8] + Date: ['Wed, 19 Jul 2017 23:52:40 GMT'] + Keep-Alive: ['timeout=5, max=100'] + Server: [Apache] + Vary: [Accept-Encoding] + X-Frame-Options: [SAMEORIGIN] + status: {code: 200, message: '200'} +version: 1 diff --git a/siphon/tests/fixtures/cat_to_subset b/siphon/tests/fixtures/cat_to_subset new file mode 100644 index 000000000..4d6d2a650 --- /dev/null +++ b/siphon/tests/fixtures/cat_to_subset @@ -0,0 +1,123 @@ +interactions: +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + User-Agent: [Siphon (0.4.1+19.g2bea7f3.dirty)] + method: GET + uri: http://thredds.ucar.edu/thredds/catalog/satellite/VIS/EAST-CONUS_1km/current/catalog.xml + response: + body: + string: !!binary | + H4sIAAAAAAAAA7Rd227jOBJ9X2D/QdDTLDCRihJ1MxzPBkm6x8B20mgnM8C+BIqlTrRtS4YkJ5P9 + +iHp+BLn0mah6s2WqRJ1RFWdKpLHw9/+ms+ch7LtqqY+doUHrlPW06ao6rtj9/rq01Hq/jb65z+G + 07zPZ82do5rX3bF73/eLge8/Pj56y7oq1I/ecpq3Xlks/Tqfl90in5ad39+3ZVF0/rh+OF0Z8B/M + NYyZwV+zqv7xwthj6DXtnS+yLPPNr+6LznnCVZ1xnGFXtg/VtHT0tY7dfDZznedDV08LdeS0mS+a + ZV24zm3eqe+r016dWE+Lpuj2zr38en5xdvJ1fermJnTTU991/LdtPU73Df15OnllRLX6wMT8lYkv + b5iYf2Cinnb7Ni7Kflp8nyxvu7J/ZUy3f9/a71dXXyfqSNnu2dz9Yc/i92pWrn76qJfz/Sd2cfrl + P2/0bj5738qyKKZ7Vq7Pzk5fWdHt3rdSdc2ekfHk8pUN1ep9E9Ni3pbzpi/3R2Ex//Z8fM/c5oyN + 0aH/fOrqm36l1PNaX2DZtmWtnt747NidTM5P/fHZ2dEk78vZrFJG/hhP/POTydXR6eXF9eRG/Jj7 + 61PWPZ6Xfa6NOlV9X7bqpOLY7dtluW6wvakLdcmReqU2PTIHNq3yZX/fKANPI/Wyr976Zxcw9Le/ + bZrrHzQao8/fxmdDf/P1RYNPTTvP+9Hn8cV41eT5wLZRM13O1e3kvXIFjvEMAwXl9489kf7md2uU + bvTXm8C77/XgW9noq36m8N0g6ZxpiE4e8mqW385K5+q+bZZ39871yrSzHQKHdqouu6LqvLrJc++u + edh2p3urIxfnk7PxxNn2Z5HflR9ctDfjrFvO53n75I4uzNF85pzXD1Xb1KbhbGvuV3N/vzondeGM + 6+8GZG1m8jycf1ld/1/O57LpVpdQdp3LRdn+xHLn/PL58lydaQCsOkcN73yxKAunbxxlqi7ytnAW + bfO/cqotdZ560Lu38vEdttXdfd+5o09tWc6enHz9gN41Mm3LvG/azQF1SL9Ko4vLkxN/dZdDE6l2 + W0wbZWfaO8t2ZvcM1b1WH5/ybPq5/c7z9Pd7Olwsb2dVp17S/c47D800v13O1CM5ds/Gn9zR9enJ + N//6Ynx2cnVy+O28ek82N9AtF4um7f+93+Jlj193cHinxstCPYJ8dtooz68G7YvOK5v3XaOcw85R + 7XD6vO1HIvBEFkKo/I35/rJJ9f9yJDMP4lRK1UJ/fdFAdbXvRkV5p0ZGd2OuNPRXB3d64L/ZhWGZ + d/1j2e1f03TjSIShJ2Mpow86FnggwuTn/dIXeqtbb3RguFwUzWP9VpfAg/f7svrxnX78mL919b0r + Df0PHuOwr+blWw+3rIvRoi079Q6q+1Ffdn4rliu/MUqcIn/q1Nu6PrC95pt2hw95W+kXvHsx6N1d + 4+smzzFSxUB3p/HN6qjrmNtWntU/cUfjubpI6/xRdZVxHmsTO93ZXHgdOP115FwfeBmaX4bdG9WL + mwBEAonIboIwBO+uqitE4PYPNKxe7q95f6/eXTqjL4PzRI2pNYpfbp+U13NHIvECWIXqye6Q0yeU + zz57rrKI71VZuCN9iSNIjkR2FYSDMB0I+V9zcrkB+RlUDMZBxITxxjAlxmujB2EsYyzG4SCgxFhw + YSw4MBaHYxyryILEOJADSYkxJEwYbwxTYrw2egDGqScCJMYiGQSCEmMufwwc/hgO98epF2IxBuWP + CTEOIiZfsTVMiPHG6EEYS4HFmNYfB5ILY8mBsbTxx0mCwjgYRJFyF4QYh0z+eGuYEuPQxh/HEomx + TAYipcSYyR8HHPw4sODHqZfg+HGg+XEUE2LMxY8DDn4cWPBjhTF2HCt+LDNCjLn4ccDBjwMLfpx4 + gIt5wSCIaH0FFz8OOPhxYMWPMyzGih9LSl/BxY8DDn4cWPHjDMsrFD8OCTEWXPxYcPBjYcGPM7yv + IObHgosfCw5+LCz4ceIFuHEsND+mzKUFFz8WHPxYWPDjzBMpEmPFj0l9BRc/Fhz8WFjw48wLIiTG + ih+HISHGXPxYcPBjYcGPFcYZFmPljwn5seDix4KDHwsrfhzh6m5C82MgjHnAFfOAI+aBRczLvAyX + 58EgShTMhBhzxTzgiHlgFfMiXMwDUxMi9BXAFfOAI+aBVcyLcTUhMHOmhLVN4Ip5wBHzwCrmxTh/ + DCbmUfoKrnoFcNQrwKJeoTDGzZmCrlcIQn4MXPUK4KhXgEW9QmGMy/PA1Cvo6m4iY6pX7Bimw3hr + 9DCMsTEPwoGky0FExsTddgxTYmxVr0hDDMYiG0RS0TdCjJm4245hSoytuFuCqrspjBV3C4ASYx5/ + vGOYFGMbf4yba1IYK+5GmEuLjIm77RimxNiKuyUoXqExJp3PExlTvWLHMCXGVvUKXC6tMA6iQRAQ + YszEj3cMU2JsxY9TVG1TYaz5MSnGXP6YgR9vjR6GMaq2qTBW/Digy0FEysWPUw5+nFrxY9y8tMaY + NJcWKRc/Tjn4cWrBj1MPcPw4Nevd6OrHIuXixykHP06t+HGGqrspjBU/JlyLpXrN5I9TDn6cWvHj + FJVLi9TwY0pfwcWPUw5+nFrx4xRVE9IYh4OQkB+nXPw45eDHqR0/xuUgqdkPQlgTSrhiXsIR85LD + Y14AnsBhnOj5vIhuDYtIuGJewhHzEruaEGoeRC8oVBgT+uOEK+YlHDEvsasJ4XhFYmIeYW0z4Yp5 + CUfMS+zm83B5XqJjXkTIjxOumJdwxLzEKualWF+h13gT8uOEq16RcNQrEqt6RYQdx3o+j9Afx1z1 + ipijXhFb1Stwa1g0xuFAEs6DxFzcLebgbrHVfB5yrik283mE89IxF3eLObhbbMXdJG4cx5q7ASnG + TP445uBusRV3k7h5kNisP6b0x1zcLebgbrEVd5M4fhyb+TxKjLm4W8zB3WK79ce4mlBs1h9Txjyu + +byYYz4vtprPC7Hj2KQhlBhz+WMOfhxb8eMA6481PybM8yIufhxx8OPIih/jNJs0xiqXJpyXjrj4 + ccTBjyMrfhzi8rxIz+cBYW0z4op5EUfMi6xiXoCrV0S6Rk9Z25Rc41hyjGNptecGcP5YL9ochITj + WHLleZIjz5NWOiwS5yukWbdJyCskV54nOfI8aaXDgvTHknifqZBceZ7kyPOklQ5LiFvDIs28NKU/ + 5op5kiPmSSudQmSeJ03MI+THkivPkxx5nrTTKcTVNqXO8yJKX8GV50mOPE9a6bAg17tJ6n1NIVee + F3LkeaFFnpd4Ga5+LKnnQUIufhxy8OPQSqcQp18h9KT0IKAcx1z8OOTgx6EFP0bPS4dGh4VyHHPx + Yw4db2Gn451gx7Hek05YE+LS8RYcOt7CTscbuU7I6HgTaugJLh1vwaHjLWx0vCMvw+V5IfUaFi6N + acGhMS1sNKYT7BrvcAB6Swgdxlz6x4JD/1jY6B8nXozzx0b/OCRcU8ilzSs4tHmFjTYv9r8rxEqb + l5BXcOnGCg7dWGGjGxt7CS7PM7qxlOvoubQgBYcWpLDRgoy8CFevELpGT7lfmkunUHDoFAobnULp + xbhxbHQKI0qMuXwFh4aesNHQC70Utx/EaOhR7gcRXNxNcHA3YcHdAuy+f2HqboS8gkunUHDoFAob + ncLAE7gaPRhtXkJewaWhJzg09ISNhp7AzkuvNPQIeQVw+WPg8Mdg4Y8B+X9NCmPljynnTLn03QSH + vpuw0HfLPAix41j/XxNdTQi4tMeAQ3sMLLTHUi/Aaf5DZtZi0Y1j4NLFAg5dLLDQxdJ/UYjix0Ct + iwVcmk3AodkEFppNsRfj9ioojPWedLraJnDpsACHDgtY6LBEXoirH4P+sybKHAS4NEKAQyMELDRC + pJdGqNqmmfinrNEDl34FcOhXgIV+hfTiCOcrqPUrIGXibjuGKTE+nLtJlYJgxzGQ6ngDl0YIcGiE + gIVGSOgluH2mkJh9pnTr3YBLvwI49CvAQr9CYQy4mGf0K4Bu3SZwaSsAh7YCWGgrhJ7EaUGC0VYg + 3A8CXNoKwKGtABbaCqHK87DjWIc9Qoy59v0Dx75/sNj3r30Fzh/HJpemWycEXHvSgWNPOljsSQ+9 + VKDmpcHsSSf8D2+IuDCOODCODsc49gD3/3kQ/bx+vPtt6E/V51lzpz7/DQAA//8DAFS08U9ljwAA + headers: + Access-Control-Allow-Origin: ['*'] + Connection: [Keep-Alive] + Content-Encoding: [gzip] + Content-Language: [en] + Content-Type: [application/xml;charset=UTF-8] + Date: ['Wed, 19 Jul 2017 23:39:03 GMT'] + Keep-Alive: ['timeout=5, max=100'] + Server: [Apache] + Vary: [Accept-Encoding] + X-Frame-Options: [SAMEORIGIN] + status: {code: 200, message: '200'} +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + User-Agent: [Siphon (0.4.1+19.g2bea7f3.dirty)] + method: GET + uri: http://thredds.ucar.edu/thredds/ncss/satellite/VIS/EAST-CONUS_1km/current/EAST-CONUS_1km_VIS_20170719_2330.gini/dataset.xml + response: + body: + string: !!binary | + H4sIAAAAAAAAA8RX34/iNhB+r9T/wcpT+wC284OQVchpu72eTlptT1321LuXyCQG3EscZJu98N93 + nIQEuINl76FFkNiemW/G34wnIX5TlwV65kqLSs4cOiYO4jKrciFXM+dp/sdo6rxJfv4pXimR/84M + 09ygosqYafSxWSue5xrLTGusmeFFIQzHH98/4re3j/PR3Z8PT48p/VLibKsUl+ZkOQXN1CU0JCGN + UtfzyHglpHDQhpn1zLFXB9wjFLNaaCRZyWeOESV3kF6zDUyog8zODvJquyhg3SrOm5W51Wusrb0x + Siy2hncgRSVXqR066JkV2w4WaSEzjhawT5TDdhyEzwGkd1WlgCfQuu1c9lCN5/OmWymM7pVLzTPd + OaZRSEaEwndOyE3z/XwA1FjohI4DQgISEhq9pW6Mu+WGJ2z3/w1ldc9XQF1ynrJ3vPr7FZRtVPUP + z2wtoBplPR9Xb/1L+YMEN3Ge8oK0YcrMnBGNAgIUuZHv08nEpyEUCdCreAkV2FQ5DSLSfShMHCQ3 + Ru/JwZeY3F3P5KcfY3L3XzP56SyTvjcJx5PI92gQRpMoCI54HB0T6bnT6CUibRt5hBYyHGTYbt3z + NLDwW1V3i7BcClknNqvWIZlGbVYn0xg3kkGP1XXiuZB8LwopCaaeBz/QsuuHaLtkBLKxS4LQ813i + Ed9vwXZHYLsECJiCmutOOgKCBm2vFuPvRdyUy198edSu8PeFu7OS+kDSFMRcMXkgv2flgitzV8ll + pUpWHKhbljst6K8OyrnOZs77kq24Qh+FFk2tdmXcJ6Er5cXO9F3zqtI9xcXnjY9r9gHfXtROn6QW + K8nzoUWr7WUHOmMFT5csM5Xab0ivK2V6CHrRnuV5Wi2X8JT7vjW5HPDhOePDRmvg96QKsM1Rdzy6 + Q9HOhlTbtJ7NtdlrtN3mQ1+Gw1liCmwNZKbFsF7Skm024jSBRYueZnt4OxLZQbSnWAU8/80258BV + OpyAtFICnt49rhuMCbqEAqXUw2TQUhS4LjnEKdiAMopegIFeJXOm8tQKioIXVwfAocmtU8VysR2S + NfFC6rrRgWWMj5PSLt4zc3906r9ybaBLud6Y+NEkxs28k3FmZX4EjckFUTPtRLramnVC/bHnURLj + dtrJJNTeOgmi8dQnfozbaRvSsfvYvnE8bpjcWy44ZCKxb1YjEkLrnLvejUduaPQ5xq1sH5nMz6hZ + SevrCDy+zTK+Mfdi2MI7SNmt/lAJaQ4OR6OGcqE3Bds9tD2tLJwELvBIaKQvaqNflqLgvzZGqR1e + Z5npZyeBy9XavR8Yv8LPileNq/b+GpveYTt9hU/JTZYvnaS9v8bG3xv5J1Yx/jaDTVL/p0i6sjsq + tLZNdn9BYP4vAAAA//8DACaqOku6DAAA + headers: + Access-Control-Allow-Origin: ['*'] + Connection: [Keep-Alive] + Content-Encoding: [gzip] + Content-Type: [application/xml;charset=UTF-8] + Date: ['Wed, 19 Jul 2017 23:39:03 GMT'] + Keep-Alive: ['timeout=5, max=100'] + Server: [Apache] + Vary: [Accept-Encoding] + X-Frame-Options: [SAMEORIGIN] + status: {code: 200, message: '200'} +version: 1 diff --git a/siphon/tests/test_catalog_access.py b/siphon/tests/test_catalog_access.py new file mode 100644 index 000000000..7ce189c18 --- /dev/null +++ b/siphon/tests/test_catalog_access.py @@ -0,0 +1,117 @@ +# Copyright (c) 2017 University Corporation for Atmospheric Research/Unidata. +# Distributed under the terms of the MIT License. +# SPDX-License-Identifier: MIT +"""Test dataset access method helpers.""" + +import os +import tempfile + +import pytest + +from siphon.catalog import TDSCatalog +from siphon.ncss import NCSS +from siphon.testing import get_recorder + + +recorder = get_recorder(__file__) + + +@pytest.fixture +def nids_url(): + """Return the URL for accessing the test NIDS dataset.""" + return ('http://thredds.ucar.edu/thredds/catalog/nexrad/level3/NMD/FTG/20170719/catalog.' + 'xml?dataset=NWS/NEXRAD3/NMD/FTG/20170719/Level3_FTG_NMD_20170719_2337.nids') + + +@recorder.use_cassette('cat_to_subset') +def test_dataset_subset(): + """Test using the subset method to request NCSS access.""" + cat = TDSCatalog('http://thredds.ucar.edu/thredds/catalog/satellite/VIS/' + 'EAST-CONUS_1km/current/catalog.xml') + subset = cat.datasets[0].subset() + assert isinstance(subset, NCSS) + assert 'VIS' in subset.variables + + +@recorder.use_cassette('cat_to_open') +def test_dataset_remote_open(nids_url): + """Test using the remote_open method to request HTTP access.""" + cat = TDSCatalog(nids_url) + fobj = cat.datasets[0].remote_open() + assert fobj.read(8) == b'\x01\r\r\n562 ' + + +@recorder.use_cassette('cat_to_cdmr') +def test_dataset_remote_access_default(nids_url): + """Test using the remote_access method to request access using default method.""" + cat = TDSCatalog(nids_url) + ds = cat.datasets[0].remote_access() + assert ds.variables == {} + assert ds.title == 'Nexrad Level 3 Data' + + +@recorder.use_cassette('cat_to_cdmr') +def test_dataset_remote_access_cdmr(nids_url): + """Test using the remote_access method to request CDMR access.""" + cat = TDSCatalog(nids_url) + ds = cat.datasets[0].remote_access(service='CdmRemote') + assert ds.variables == {} + assert ds.title == 'Nexrad Level 3 Data' + + +@recorder.use_cassette('cat_to_opendap') +def test_dataset_remote_access_opendap(nids_url): + """Test using the remote_access method to request opendap access.""" + cat = TDSCatalog(nids_url) + ds = cat.datasets[0].remote_access(service='OPENDAP') + assert ds.variables == {} + assert ds.title == 'Nexrad Level 3 Data' + + +@recorder.use_cassette('cat_to_open') +def test_dataset_download(nids_url): + """Test using the download method to download entire dataset.""" + cat = TDSCatalog(nids_url) + temp = os.path.join(tempfile.gettempdir(), 'siphon-test.temp') + try: + assert not os.path.exists(temp) + cat.datasets[0].download(temp) + assert os.path.exists(temp) + finally: + os.remove(temp) + + +@recorder.use_cassette('cat_to_open') +def test_dataset_invalid_service_remote_access(nids_url): + """Test requesting an invalid service for remote_access gives a ValueError.""" + cat = TDSCatalog(nids_url) + with pytest.raises(ValueError) as err: + cat.datasets[0].remote_access('foobar') + assert 'not a valid service for' in str(err.value) + + +@recorder.use_cassette('cat_to_open') +def test_dataset_invalid_service_subset(nids_url): + """Test requesting an invalid service for subset gives a ValueError.""" + cat = TDSCatalog(nids_url) + with pytest.raises(ValueError) as err: + cat.datasets[0].subset('OPENDAP') + assert 'not a valid service for' in str(err.value) + + +@recorder.use_cassette('cat_to_open') +def test_dataset_unavailable_service(nids_url): + """Test requesting a service that isn't present gives a ValueError.""" + cat = TDSCatalog(nids_url) + with pytest.raises(ValueError) as err: + cat.datasets[0].access_with_service('NetcdfSubset') + assert 'not available' in str(err.value) + + +@recorder.use_cassette('cat_to_open') +def test_dataset_no_handler(nids_url): + """Test requesting a service that has no handler gives a ValueError.""" + cat = TDSCatalog(nids_url) + with pytest.raises(ValueError) as err: + cat.datasets[0].access_with_service('UDDC') + assert 'is not an access method supported' in str(err.value) From f11cb78c2d429c4215acf33ecb9db050d6d59404 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Thu, 20 Jul 2017 16:20:05 -0600 Subject: [PATCH 4/7] ENH: Add latest attribute to catalog This gives access to the latest dataset. Partially addresses #137. --- siphon/catalog.py | 58 ++++++++++-------------------------- siphon/tests/test_catalog.py | 9 ++++++ 2 files changed, 24 insertions(+), 43 deletions(-) diff --git a/siphon/catalog.py b/siphon/catalog.py index 068c53e02..0cb32b3f6 100644 --- a/siphon/catalog.py +++ b/siphon/catalog.py @@ -264,6 +264,15 @@ def _process_datasets(self): else: self.datasets.pop(dsName) + @property + def latest(self): + """Get the latest dataset, if available.""" + for service in self.services: + if service.is_resolver(): + latest_cat = self.catalog_url.replace('catalog.xml', 'latest.xml') + return TDSCatalog(latest_cat).datasets[0] + raise AttributeError('"latest" not available for this catalog') + class CatalogRef(object): """ @@ -580,8 +589,7 @@ class SimpleService(object): """ def __init__(self, service_node): - """ - Initialize the Dataset object. + """Initialize the Dataset object. Parameters ---------- @@ -594,6 +602,10 @@ def __init__(self, service_node): self.base = service_node.attrib['base'] self.access_urls = {} + def is_resolver(self): + """Return whether the service is a resolver service.""" + return self.service_type.lower() == 'resolver' + class CompoundService(object): """Hold information about compound services. @@ -644,29 +656,6 @@ def _find_base_tds_url(catalog_url): return catalog_url -def _get_latest_cat(catalog_url): - """Get the latest dataset catalog from the supplied top level dataset catalog url. - - Parameters - ---------- - catalog_url : str - The URL of a top level data catalog - - Returns - ------- - TDSCatalog - A TDSCatalog object containing the information from the latest dataset - - """ - cat = TDSCatalog(catalog_url) - for service in cat.services: - if service.service_type.lower() == 'resolver': - latest_cat = cat.catalog_url.replace('catalog.xml', 'latest.xml') - return TDSCatalog(latest_cat) - - log.error('ERROR: "latest" service not enabled for this catalog!') - - def get_latest_access_url(catalog_url, access_method): """Get the data access url to the latest data using a specified access method. @@ -688,21 +677,4 @@ def get_latest_access_url(catalog_url, access_method): but not always. """ - latest_cat = _get_latest_cat(catalog_url) - if latest_cat != '': - if len(list(latest_cat.datasets.keys())) > 0: - latest_ds = [] - for lds_name in latest_cat.datasets: - lds = latest_cat.datasets[lds_name] - if access_method in lds.access_urls: - latest_ds.append(lds.access_urls[access_method]) - if len(latest_ds) == 1: - latest_ds = latest_ds[0] - return latest_ds - else: - log.error('ERROR: More than one latest dataset found ' - 'this case is currently not suppored in ' - 'siphon.') - else: - log.error('ERROR: More than one access url matching the ' - 'requested access method...clearly this is an error') + return TDSCatalog(catalog_url).latest.access_urls[access_method] diff --git a/siphon/tests/test_catalog.py b/siphon/tests/test_catalog.py index b9194ed0a..cd3476e67 100644 --- a/siphon/tests/test_catalog.py +++ b/siphon/tests/test_catalog.py @@ -63,6 +63,15 @@ def test_get_latest(): assert latest_url +@recorder.use_cassette('latest_rap_catalog') +def test_latest_attribute(): + """Test using the catalog latest attribute.""" + url = ('http://thredds-test.unidata.ucar.edu/thredds/catalog/' + 'grib/NCEP/RAP/CONUS_13km/catalog.xml') + cat = TDSCatalog(url) + assert cat.latest.name == 'RR_CONUS_13km_20150527_0100.grib2' + + @recorder.use_cassette('top_level_cat') def test_tds_top_catalog(): """Test parsing top-level catalog.""" From 77eeea36e5a3b92c166bbec385b6828f6d7f2eff Mon Sep 17 00:00:00 2001 From: Ryan May Date: Thu, 20 Jul 2017 22:17:41 -0600 Subject: [PATCH 5/7] MNT: Update examples for API changes --- examples/Basic_Usage.py | 2 +- examples/Radar_Server_Level_3.py | 2 +- examples/ncss/NCSS_Cartopy_Example.py | 11 +++-------- examples/ncss/NCSS_Example.py | 11 +++-------- examples/ncss/NCSS_Timeseries_Examples.py | 11 +++-------- 5 files changed, 11 insertions(+), 26 deletions(-) diff --git a/examples/Basic_Usage.py b/examples/Basic_Usage.py index 254d2405a..f25a9c99f 100644 --- a/examples/Basic_Usage.py +++ b/examples/Basic_Usage.py @@ -19,4 +19,4 @@ ########################################### cat = TDSCatalog('http://thredds.ucar.edu/thredds/catalog.xml') -print(list(cat.catalog_refs.keys())) +print(list(cat.catalog_refs)) diff --git a/examples/Radar_Server_Level_3.py b/examples/Radar_Server_Level_3.py index 36dfaa8a1..a12751858 100644 --- a/examples/Radar_Server_Level_3.py +++ b/examples/Radar_Server_Level_3.py @@ -20,7 +20,7 @@ # First, point to the top-level thredds radar server accessor to find what datasets are # available. ds = get_radarserver_datasets('http://thredds.ucar.edu/thredds/') -print(list(ds.keys())) +print(list(ds)) ########################################### # Now create an instance of RadarServer to point to the appropriate diff --git a/examples/ncss/NCSS_Cartopy_Example.py b/examples/ncss/NCSS_Cartopy_Example.py index f8d6a1789..6d61cedf2 100644 --- a/examples/ncss/NCSS_Cartopy_Example.py +++ b/examples/ncss/NCSS_Cartopy_Example.py @@ -20,7 +20,6 @@ import numpy as np from siphon.catalog import TDSCatalog -from siphon.ncss import NCSS ########################################### # First we construct a `TDSCatalog` instance pointing to our dataset of interest, in @@ -32,13 +31,9 @@ print(list(best_gfs.datasets)) ########################################### -# We pull out this dataset and look at the access urls. -best_ds = list(best_gfs.datasets.values())[0] -print(best_ds.access_urls) - -########################################### -# Note the `NetcdfSubset` entry, which we will use with our NCSS class. -ncss = NCSS(best_ds.access_urls['NetcdfSubset']) +# We pull out this dataset and get the NCSS access point +best_ds = best_gfs.datasets[0] +ncss = best_ds.subset() ########################################### # We can then use the `ncss` object to create a new query object, which diff --git a/examples/ncss/NCSS_Example.py b/examples/ncss/NCSS_Example.py index 8e1c8b129..28aa046e2 100644 --- a/examples/ncss/NCSS_Example.py +++ b/examples/ncss/NCSS_Example.py @@ -13,7 +13,6 @@ import matplotlib.pyplot as plt from siphon.catalog import TDSCatalog -from siphon.ncss import NCSS ########################################### # First we construct a TDSCatalog instance pointing to our dataset of interest, in @@ -24,13 +23,9 @@ print(best_gfs.datasets) ########################################### -# We pull out this dataset and look at the access urls. -best_ds = list(best_gfs.datasets.values())[0] -print(best_ds.access_urls) - -########################################### -# Note the `NetcdfSubset` entry, which we will use with our NCSS class. -ncss = NCSS(best_ds.access_urls['NetcdfSubset']) +# We pull out this dataset and get the NCSS access point +best_ds = best_gfs.datasets[0] +ncss = best_ds.subset() ########################################### # We can then use the `ncss` object to create a new query object, which diff --git a/examples/ncss/NCSS_Timeseries_Examples.py b/examples/ncss/NCSS_Timeseries_Examples.py index a04d8b748..31c7bda9d 100644 --- a/examples/ncss/NCSS_Timeseries_Examples.py +++ b/examples/ncss/NCSS_Timeseries_Examples.py @@ -14,7 +14,6 @@ from netCDF4 import num2date from siphon.catalog import TDSCatalog -from siphon.ncss import NCSS ########################################### # First we construct a TDSCatalog instance pointing to our dataset of interest, in @@ -25,13 +24,9 @@ print(best_gfs.datasets) ########################################### -# We pull out this dataset and look at the access urls. -best_ds = list(best_gfs.datasets.values())[0] -print(best_ds.access_urls) - -########################################### -# Note the `NetcdfSubset` entry, which we will use with our NCSS class. -ncss = NCSS(best_ds.access_urls['NetcdfSubset']) +# We pull out this dataset and get the NCSS access point +best_ds = best_gfs.datasets[0] +ncss = best_ds.subset() ########################################### # We can then use the `ncss` object to create a new query object, which From a65193b938750c69b6df6b12c50a0dee4cdd5d2d Mon Sep 17 00:00:00 2001 From: Ryan May Date: Fri, 21 Jul 2017 10:02:54 -0600 Subject: [PATCH 6/7] MNT: Silence the log message on test failure Catchlog was dumping way to much output. --- .appveyor.yml | 2 +- .travis.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 4042c964d..d2aa57d30 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -35,7 +35,7 @@ install: build: off test_script: - - cmd: python setup.py test --addopts "-s --junitxml=tests.xml --flake8 --cov=siphon" + - cmd: python setup.py test --addopts "-s --junitxml=tests.xml --flake8 --cov=siphon --no-print-logs" #- cmd: cd docs #- cmd: make html #- cmd: cd .. diff --git a/.travis.yml b/.travis.yml index bcf8a73f4..c84bab3ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -98,7 +98,7 @@ script: fi; else flake8 --version; - python setup.py test --addopts "-s $TEST_OPTS"; + python setup.py test --addopts "-s $TEST_OPTS --no-print-logs"; fi after_script: From 3ada1e8c0b5ef0c4950e2b88d7011f8c3f4c6013 Mon Sep 17 00:00:00 2001 From: Ryan May Date: Fri, 21 Jul 2017 10:15:24 -0600 Subject: [PATCH 7/7] MNT: Update line length for prospector --- .prospector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.prospector.yaml b/.prospector.yaml index 47b3fc7ed..c7af77fd8 100644 --- a/.prospector.yaml +++ b/.prospector.yaml @@ -13,4 +13,4 @@ pylint: pep8: options: - max-line-length: 90 + max-line-length: 95