From b499253f748ec6cb97b2bd4ba89292b438917d4f Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Fri, 16 Aug 2024 17:48:11 -0500 Subject: [PATCH 01/16] first crack at changing the url --- dataretrieval/wqp.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index cad5e736..ac32c6e1 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -16,7 +16,7 @@ from .utils import BaseMetadata, query -def get_results(ssl_check=True, **kwargs): +def get_results(ssl_check=True, legacy=False, **kwargs): """Query the WQP for results. Any WQP API parameter can be passed as a keyword argument to this function. @@ -30,6 +30,12 @@ def get_results(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. + legacy: Boolean + Defaults to False and returns the new WQX3 Result profile. + If set to True, returns data using the legacy WQX version 2 profiles. + dataProfile: string + Describes the type of columns to return with the result dataset. + Includes 'fullPhysChem', 'biological', 'narrow', and 'basicPhysChem'. siteid: string Concatenate an agency code, a hyphen ("-"), and a site-identification number. @@ -63,10 +69,7 @@ def get_results(ssl_check=True, **kwargs): for available characteristic names) mimeType: string String specifying the output format which is 'csv' by default but can - be 'geojson' - zip: string - Parameter to stream compressed data, if 'yes', or uncompressed data - if 'no'. Default is 'no'. + be 'geojson' Returns ------- @@ -91,7 +94,19 @@ def get_results(ssl_check=True, **kwargs): _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - response = query(wqp_url('Result'), kwargs, delimiter=';', ssl_check=ssl_check) + + if legacy is True: + warnings.warn('Legacy profiles return stale USGS data as of ' + 'March 2024. Please set legacy=True to use the WQX3 ' + 'data profiles and obtain the latest USGS data.') + url = wqp_url('Result') + + else: + url = wqx3_url('Result') + if 'dataProfile' not in kwargs: + kwargs['dataProfile'] = 'basicPhysChem' + + response = query(url, kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') return df, WQP_Metadata(response) @@ -472,6 +487,11 @@ def wqp_url(service): base_url = 'https://www.waterqualitydata.us/data/' return f'{base_url}{service}/Search?' +def wqx3_url(service): + """Construct the WQP URL for a given WQX 3.0 service.""" + base_url = 'https://www.waterqualitydata.us/wqx3/' + return f'{base_url}{service}/search?' + class WQP_Metadata(BaseMetadata): """Metadata class for WQP service, derived from BaseMetadata. @@ -531,9 +551,6 @@ def _alter_kwargs(kwargs): user so they are aware of which are being hard-set. """ - if kwargs.get('zip', 'no') == 'yes': - warnings.warn('Compressed data not yet supported, zip set to no.') - kwargs['zip'] = 'no' if kwargs.get('mimeType', 'csv') == 'geojson': warnings.warn('GeoJSON not yet supported, mimeType set to csv.') From 41b3270c909535e18b286139dc6e7c242478c564 Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Tue, 20 Aug 2024 18:03:50 -0500 Subject: [PATCH 02/16] trying to avoid a legacy=True argument in favor of unique dataprofiles --- dataretrieval/wqp.py | 85 +++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 24 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index ac32c6e1..9982319a 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -15,8 +15,18 @@ from .utils import BaseMetadata, query - -def get_results(ssl_check=True, legacy=False, **kwargs): +result_profiles_wqx3 = ['fullPhysChem', 'narrow', 'basicPhysChem'] +result_profiles_legacy = ['resultPhysChem', 'biological', + 'narrowResult'] +activity_profiles_legacy = ['activityAll'] +services_wqx3 = ['Result', 'Station', 'Activity'] +services_legacy = ['Organization', 'Project', + 'ProjectMonitoringLocationWeighting', + 'ActivityMetric', + 'ResultDetectionQuantitationLimit', + 'BiologicalMetric'] + +def get_results(ssl_check=True, **kwargs): """Query the WQP for results. Any WQP API parameter can be passed as a keyword argument to this function. @@ -30,12 +40,11 @@ def get_results(ssl_check=True, legacy=False, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. - legacy: Boolean - Defaults to False and returns the new WQX3 Result profile. - If set to True, returns data using the legacy WQX version 2 profiles. dataProfile: string Describes the type of columns to return with the result dataset. - Includes 'fullPhysChem', 'biological', 'narrow', and 'basicPhysChem'. + Most recent WQX3 profiles include 'fullPhysChem', 'narrow', and + 'basicPhysChem'. Legacy profiles include 'resultPhysChem', + 'biological', and 'narrowResult'. siteid: string Concatenate an agency code, a hyphen ("-"), and a site-identification number. @@ -91,20 +100,23 @@ def get_results(ssl_check=True, legacy=False, **kwargs): >>> df, md = dataretrieval.wqp.get_results(bBox='-92.8,44.2,-88.9,46.0') """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - if legacy is True: - warnings.warn('Legacy profiles return stale USGS data as of ' - 'March 2024. Please set legacy=True to use the WQX3 ' - 'data profiles and obtain the latest USGS data.') + if 'dataProfile' not in kwargs: + # set to wqx3 full result profile + kwargs['dataProfile'] = 'fullPhysChem' + # check to ensure profile is available in either legacy + # or wqx3 + elif kwargs['dataProfile'] not in (result_profiles_legacy + result_profiles_wqx3): + raise TypeError('dataProfile not recognized') + + # warn user when selecting a legacy profile + if kwargs['dataProfile'] in result_profiles_legacy: + _warn_legacy_use('dataProfile') url = wqp_url('Result') - else: url = wqx3_url('Result') - if 'dataProfile' not in kwargs: - kwargs['dataProfile'] = 'basicPhysChem' response = query(url, kwargs, delimiter=';', ssl_check=ssl_check) @@ -146,11 +158,10 @@ def what_sites(ssl_check=True, **kwargs): ... ) """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - url = wqp_url('Station') + url = wqx3_url('Station') response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -190,7 +201,8 @@ def what_organizations(ssl_check=True, **kwargs): >>> df, md = dataretrieval.wqp.what_organizations() """ - _warn_v3_profiles_outage() + + _warn_legacy_use('service') kwargs = _alter_kwargs(kwargs) @@ -234,7 +246,7 @@ def what_projects(ssl_check=True, **kwargs): >>> df, md = dataretrieval.wqp.what_projects(huc='19') """ - _warn_v3_profiles_outage() + _warn_legacy_use('service') kwargs = _alter_kwargs(kwargs) @@ -260,6 +272,10 @@ def what_activities(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. + dataProfile: string + Describes the type of columns to return with the result dataset. + Currently, only the legacy services have a single data profile: + 'activityAll'. **kwargs: optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` @@ -281,11 +297,18 @@ def what_activities(ssl_check=True, **kwargs): ... ) """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - url = wqp_url('Activity') + if 'dataProfile' not in kwargs: + url = wqx3_url('Activity') + elif kwargs['dataProfile'] not in activity_profiles_legacy: + raise TypeError('dataProfile not recognized') + else: + _warn_legacy_use('service') + url = wqp_url('Activity') + + response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -332,7 +355,7 @@ def what_detection_limits(ssl_check=True, **kwargs): ... ) """ - _warn_v3_profiles_outage() + _warn_legacy_use('service') kwargs = _alter_kwargs(kwargs) @@ -376,7 +399,7 @@ def what_habitat_metrics(ssl_check=True, **kwargs): >>> df, md = dataretrieval.wqp.what_habitat_metrics(statecode='US:44') """ - _warn_v3_profiles_outage() + _warn_legacy_use('service') kwargs = _alter_kwargs(kwargs) @@ -423,7 +446,7 @@ def what_project_weights(ssl_check=True, **kwargs): ... ) """ - _warn_v3_profiles_outage() + _warn_legacy_use('service') kwargs = _alter_kwargs(kwargs) @@ -470,7 +493,7 @@ def what_activity_metrics(ssl_check=True, **kwargs): ... ) """ - _warn_v3_profiles_outage() + _warn_legacy_use('service') kwargs = _alter_kwargs(kwargs) @@ -574,3 +597,17 @@ def _warn_v3_profiles_outage(): 'If you have additional questions about these changes, ' 'email CompTools@usgs.gov.' ) + +def _warn_legacy_use(type): + """Private function for warning message about using legacy profiles + or services + """ + + warnings.warn( + f'The {type} you have selected is currently only ' + 'available in the legacy Water Quality Portal ' + 'profiles. This means that any USGS data served ' + 'are stale as of March 2024. Please review the ' + f'updated WQX3.0 {type}s in the dataretrieval-python ' + 'documentation.' + ) From c5839c1e8586ec6d1ba80b9cb68cdbde0ff08a8a Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Tue, 20 Aug 2024 18:12:26 -0500 Subject: [PATCH 03/16] check services --- dataretrieval/wqp.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 9982319a..ccd9dc99 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -20,7 +20,8 @@ 'narrowResult'] activity_profiles_legacy = ['activityAll'] services_wqx3 = ['Result', 'Station', 'Activity'] -services_legacy = ['Organization', 'Project', +services_legacy = ['Result', 'Station', 'Activity', + 'Organization', 'Project', 'ProjectMonitoringLocationWeighting', 'ActivityMetric', 'ResultDetectionQuantitationLimit', @@ -507,11 +508,15 @@ def what_activity_metrics(ssl_check=True, **kwargs): def wqp_url(service): """Construct the WQP URL for a given service.""" + if service not in services_legacy: + raise TypeError('Legacy service not recognized') base_url = 'https://www.waterqualitydata.us/data/' return f'{base_url}{service}/Search?' def wqx3_url(service): """Construct the WQP URL for a given WQX 3.0 service.""" + if service not in services_wqx3: + raise TypeError('WQX3 service not recognized') base_url = 'https://www.waterqualitydata.us/wqx3/' return f'{base_url}{service}/search?' From 9238e5085c5bc9d12694c882c27c57a7ffe5c508 Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Wed, 21 Aug 2024 16:38:24 -0500 Subject: [PATCH 04/16] add legacy back in, adjust all functions and tests --- dataretrieval/wqp.py | 179 +++++++++++++++++++++++++++++-------------- tests/wqp_test.py | 29 +++---- 2 files changed, 134 insertions(+), 74 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index ccd9dc99..db83b3c4 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -27,7 +27,7 @@ 'ResultDetectionQuantitationLimit', 'BiologicalMetric'] -def get_results(ssl_check=True, **kwargs): +def get_results(ssl_check=True, legacy=True, **kwargs): """Query the WQP for results. Any WQP API parameter can be passed as a keyword argument to this function. @@ -41,6 +41,10 @@ def get_results(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. + legacy: bool + If True, returns the legacy WQX data profile and warns the user of + the issues associated with it. If False, returns the new WQX3.0 + profile, if available. Defaults to True. dataProfile: string Describes the type of columns to return with the result dataset. Most recent WQX3 profiles include 'fullPhysChem', 'narrow', and @@ -101,31 +105,33 @@ def get_results(ssl_check=True, **kwargs): >>> df, md = dataretrieval.wqp.get_results(bBox='-92.8,44.2,-88.9,46.0') """ + _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - if 'dataProfile' not in kwargs: - # set to wqx3 full result profile - kwargs['dataProfile'] = 'fullPhysChem' - # check to ensure profile is available in either legacy - # or wqx3 - elif kwargs['dataProfile'] not in (result_profiles_legacy + result_profiles_wqx3): - raise TypeError('dataProfile not recognized') - - # warn user when selecting a legacy profile - if kwargs['dataProfile'] in result_profiles_legacy: - _warn_legacy_use('dataProfile') + if legacy is True: + if 'dataProfile' in kwargs: + if kwargs['dataProfile'] not in result_profiles_legacy: + raise TypeError('dataProfile is not a legacy profile name') + url = wqp_url('Result') + else: + if 'dataProfile' in kwargs: + if kwargs['dataProfile'] not in result_profiles_wqx3: + raise TypeError('dataProfile is not a WQX3.0 profile name') + else: + kwargs['dataProfile'] = 'fullPhysChem' + url = wqx3_url('Result') - + response = query(url, kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') return df, WQP_Metadata(response) -def what_sites(ssl_check=True, **kwargs): +def what_sites(ssl_check=True, legacy=True, **kwargs): """Search WQP for sites within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -139,6 +145,10 @@ def what_sites(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. + legacy: bool + If True, returns the legacy WQX data profile and warns the user of + the issues associated with it. If False, returns the new WQX3.0 + profile, if available. Defaults to True. **kwargs: optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` @@ -159,10 +169,15 @@ def what_sites(ssl_check=True, **kwargs): ... ) """ + _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - url = wqx3_url('Station') + if legacy is True: + url = wqp_url('Station') + else: + url = wqx3_url('Station') + response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -170,7 +185,7 @@ def what_sites(ssl_check=True, **kwargs): return df, WQP_Metadata(response) -def what_organizations(ssl_check=True, **kwargs): +def what_organizations(ssl_check=True, legacy=True, **kwargs): """Search WQP for organizations within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -184,6 +199,10 @@ def what_organizations(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. + legacy: bool + If True, returns the legacy WQX data profile and warns the user of + the issues associated with it. If False, returns the new WQX3.0 + profile, if available. Defaults to True. **kwargs: optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` @@ -202,12 +221,16 @@ def what_organizations(ssl_check=True, **kwargs): >>> df, md = dataretrieval.wqp.what_organizations() """ - - _warn_legacy_use('service') + _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - url = wqp_url('Organization') + if legacy is True: + url = wqp_url('Organization') + else: + print('No WQX3.0 profile currently available, returning legacy profile.') + url = wqp_url('Organization') + response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -215,7 +238,7 @@ def what_organizations(ssl_check=True, **kwargs): return df, WQP_Metadata(response) -def what_projects(ssl_check=True, **kwargs): +def what_projects(ssl_check=True, legacy=True, **kwargs): """Search WQP for projects within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -229,6 +252,10 @@ def what_projects(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. + legacy: bool + If True, returns the legacy WQX data profile and warns the user of + the issues associated with it. If False, returns the new WQX3.0 + profile, if available. Defaults to True. **kwargs: optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` @@ -247,11 +274,16 @@ def what_projects(ssl_check=True, **kwargs): >>> df, md = dataretrieval.wqp.what_projects(huc='19') """ - _warn_legacy_use('service') + _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - url = wqp_url('Project') + if legacy is True: + url = wqp_url('Project') + else: + print('No WQX3.0 profile currently available, returning legacy profile.') + url = wqp_url('Project') + response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -259,7 +291,7 @@ def what_projects(ssl_check=True, **kwargs): return df, WQP_Metadata(response) -def what_activities(ssl_check=True, **kwargs): +def what_activities(ssl_check=True, legacy=True, **kwargs): """Search WQP for activities within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -273,10 +305,10 @@ def what_activities(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. - dataProfile: string - Describes the type of columns to return with the result dataset. - Currently, only the legacy services have a single data profile: - 'activityAll'. + legacy: bool + If True, returns the legacy WQX data profile and warns the user of + the issues associated with it. If False, returns the new WQX3.0 + profile, if available. Defaults to True. **kwargs: optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` @@ -299,17 +331,15 @@ def what_activities(ssl_check=True, **kwargs): """ + _warn_v3_profiles_outage() + kwargs = _alter_kwargs(kwargs) - if 'dataProfile' not in kwargs: - url = wqx3_url('Activity') - elif kwargs['dataProfile'] not in activity_profiles_legacy: - raise TypeError('dataProfile not recognized') - else: - _warn_legacy_use('service') + if legacy is True: url = wqp_url('Activity') - - + else: + url = wqx3_url('Activity') + response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -317,7 +347,7 @@ def what_activities(ssl_check=True, **kwargs): return df, WQP_Metadata(response) -def what_detection_limits(ssl_check=True, **kwargs): +def what_detection_limits(ssl_check=True, legacy=True, **kwargs): """Search WQP for result detection limits within a region with specific data. @@ -332,6 +362,10 @@ def what_detection_limits(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. + legacy: bool + If True, returns the legacy WQX data profile and warns the user of + the issues associated with it. If False, returns the new WQX3.0 + profile, if available. Defaults to True. **kwargs: optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` @@ -356,11 +390,16 @@ def what_detection_limits(ssl_check=True, **kwargs): ... ) """ - _warn_legacy_use('service') + _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - url = wqp_url('ResultDetectionQuantitationLimit') + if legacy is True: + url = wqp_url('ResultDetectionQuantitationLimit') + else: + print('No WQX3.0 profile currently available, returning legacy profile.') + url = wqp_url('ResultDetectionQuantitationLimit') + response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -368,7 +407,7 @@ def what_detection_limits(ssl_check=True, **kwargs): return df, WQP_Metadata(response) -def what_habitat_metrics(ssl_check=True, **kwargs): +def what_habitat_metrics(ssl_check=True, legacy=True, **kwargs): """Search WQP for habitat metrics within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -382,6 +421,10 @@ def what_habitat_metrics(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. + legacy: bool + If True, returns the legacy WQX data profile and warns the user of + the issues associated with it. If False, returns the new WQX3.0 + profile, if available. Defaults to True. **kwargs: optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` @@ -400,11 +443,16 @@ def what_habitat_metrics(ssl_check=True, **kwargs): >>> df, md = dataretrieval.wqp.what_habitat_metrics(statecode='US:44') """ - _warn_legacy_use('service') + _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - url = wqp_url('BiologicalMetric') + if legacy is True: + url = wqp_url('BiologicalMetric') + else: + print('No WQX3.0 profile currently available, returning legacy profile.') + url = wqp_url('BiologicalMetric') + response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -412,7 +460,7 @@ def what_habitat_metrics(ssl_check=True, **kwargs): return df, WQP_Metadata(response) -def what_project_weights(ssl_check=True, **kwargs): +def what_project_weights(ssl_check=True, legacy=True, **kwargs): """Search WQP for project weights within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -426,6 +474,10 @@ def what_project_weights(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. + legacy: bool + If True, returns the legacy WQX data profile and warns the user of + the issues associated with it. If False, returns the new WQX3.0 + profile, if available. Defaults to True. **kwargs: optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` @@ -447,11 +499,16 @@ def what_project_weights(ssl_check=True, **kwargs): ... ) """ - _warn_legacy_use('service') + _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - url = wqp_url('ProjectMonitoringLocationWeighting') + if legacy is True: + url = wqp_url('ProjectMonitoringLocationWeighting') + else: + print('No WQX3.0 profile currently available, returning legacy profile.') + url = wqp_url('ProjectMonitoringLocationWeighting') + response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -459,7 +516,7 @@ def what_project_weights(ssl_check=True, **kwargs): return df, WQP_Metadata(response) -def what_activity_metrics(ssl_check=True, **kwargs): +def what_activity_metrics(ssl_check=True, legacy=True, **kwargs): """Search WQP for activity metrics within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -473,6 +530,10 @@ def what_activity_metrics(ssl_check=True, **kwargs): ssl_check: bool If True, check the SSL certificate. Default is True. If False, SSL certificate is not checked. + legacy: bool + If True, returns the legacy WQX data profile and warns the user of + the issues associated with it. If False, returns the new WQX3.0 + profile, if available. Defaults to True. **kwargs: optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` @@ -494,11 +555,16 @@ def what_activity_metrics(ssl_check=True, **kwargs): ... ) """ - _warn_legacy_use('service') + _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) - url = wqp_url('ActivityMetric') + if legacy is True: + url = wqp_url('ActivityMetric') + else: + print('No WQX3.0 profile currently available, returning legacy profile.') + url = wqp_url('ActivityMetric') + response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -508,6 +574,9 @@ def what_activity_metrics(ssl_check=True, **kwargs): def wqp_url(service): """Construct the WQP URL for a given service.""" + + _warn_legacy_use() + if service not in services_legacy: raise TypeError('Legacy service not recognized') base_url = 'https://www.waterqualitydata.us/data/' @@ -516,7 +585,7 @@ def wqp_url(service): def wqx3_url(service): """Construct the WQP URL for a given WQX 3.0 service.""" if service not in services_wqx3: - raise TypeError('WQX3 service not recognized') + raise TypeError('WQX3.0 service not recognized') base_url = 'https://www.waterqualitydata.us/wqx3/' return f'{base_url}{service}/search?' @@ -593,7 +662,7 @@ def _warn_v3_profiles_outage(): warnings.warn( 'USGS discrete water quality data availability ' - 'and format are changing. Beginning in March 2024 ' + 'and format are changing. As of March 2024, ' 'the data obtained from legacy profiles will not ' 'include new USGS data or recent updates to existing ' 'data. To view the status of changes in data ' @@ -603,16 +672,14 @@ def _warn_v3_profiles_outage(): 'email CompTools@usgs.gov.' ) -def _warn_legacy_use(type): +def _warn_legacy_use(): """Private function for warning message about using legacy profiles or services """ warnings.warn( - f'The {type} you have selected is currently only ' - 'available in the legacy Water Quality Portal ' - 'profiles. This means that any USGS data served ' - 'are stale as of March 2024. Please review the ' - f'updated WQX3.0 {type}s in the dataretrieval-python ' - 'documentation.' + 'This function call is currently returning the legacy WQX format. ' + 'This means that any USGS data served are stale as of March 2024. ' + 'Please review the dataretrieval-python documentation for more ' + 'information on updated WQX3.0 profiles.' ) diff --git a/tests/wqp_test.py b/tests/wqp_test.py index a4690183..8de4360a 100755 --- a/tests/wqp_test.py +++ b/tests/wqp_test.py @@ -15,7 +15,7 @@ def test_get_ratings(requests_mock): """Tests water quality portal ratings query""" request_url = "https://www.waterqualitydata.us/data/Result/Search?siteid=WIDNR_WQX-10032762" \ "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" \ - "&zip=no&mimeType=csv" + "&mimeType=csv" response_file_path = 'data/wqp_results.txt' mock_request(requests_mock, request_url, response_file_path) df, md = get_results(siteid='WIDNR_WQX-10032762', @@ -31,7 +31,7 @@ def test_get_ratings(requests_mock): def test_what_sites(requests_mock): """Tests Water quality portal sites query""" - request_url = "https://www.waterqualitydata.us/data/Station/Search?statecode=US%3A34&characteristicName=Chloride&zip=no" \ + request_url = "https://www.waterqualitydata.us/data/Station/Search?statecode=US%3A34&characteristicName=Chloride" \ "&mimeType=csv" response_file_path = 'data/wqp_sites.txt' mock_request(requests_mock, request_url, response_file_path) @@ -46,7 +46,7 @@ def test_what_sites(requests_mock): def test_what_organizations(requests_mock): """Tests Water quality portal organizations query""" - request_url = "https://www.waterqualitydata.us/data/Organization/Search?statecode=US%3A34&characteristicName=Chloride&zip=no" \ + request_url = "https://www.waterqualitydata.us/data/Organization/Search?statecode=US%3A34&characteristicName=Chloride" \ "&mimeType=csv" response_file_path = 'data/wqp_organizations.txt' mock_request(requests_mock, request_url, response_file_path) @@ -61,7 +61,7 @@ def test_what_organizations(requests_mock): def test_what_projects(requests_mock): """Tests Water quality portal projects query""" - request_url = "https://www.waterqualitydata.us/data/Project/Search?statecode=US%3A34&characteristicName=Chloride&zip=no" \ + request_url = "https://www.waterqualitydata.us/data/Project/Search?statecode=US%3A34&characteristicName=Chloride" \ "&mimeType=csv" response_file_path = 'data/wqp_projects.txt' mock_request(requests_mock, request_url, response_file_path) @@ -76,7 +76,7 @@ def test_what_projects(requests_mock): def test_what_activities(requests_mock): """Tests Water quality portal activities query""" - request_url = "https://www.waterqualitydata.us/data/Activity/Search?statecode=US%3A34&characteristicName=Chloride&zip=no" \ + request_url = "https://www.waterqualitydata.us/data/Activity/Search?statecode=US%3A34&characteristicName=Chloride" \ "&mimeType=csv" response_file_path = 'data/wqp_activities.txt' mock_request(requests_mock, request_url, response_file_path) @@ -91,7 +91,7 @@ def test_what_activities(requests_mock): def test_what_detection_limits(requests_mock): """Tests Water quality portal detection limits query""" - request_url = "https://www.waterqualitydata.us/data/ResultDetectionQuantitationLimit/Search?statecode=US%3A34&characteristicName=Chloride&zip=no" \ + request_url = "https://www.waterqualitydata.us/data/ResultDetectionQuantitationLimit/Search?statecode=US%3A34&characteristicName=Chloride" \ "&mimeType=csv" response_file_path = 'data/wqp_detection_limits.txt' mock_request(requests_mock, request_url, response_file_path) @@ -106,7 +106,7 @@ def test_what_detection_limits(requests_mock): def test_what_habitat_metrics(requests_mock): """Tests Water quality portal habitat metrics query""" - request_url = "https://www.waterqualitydata.us/data/BiologicalMetric/Search?statecode=US%3A34&characteristicName=Chloride&zip=no" \ + request_url = "https://www.waterqualitydata.us/data/BiologicalMetric/Search?statecode=US%3A34&characteristicName=Chloride" \ "&mimeType=csv" response_file_path = 'data/wqp_habitat_metrics.txt' mock_request(requests_mock, request_url, response_file_path) @@ -121,7 +121,7 @@ def test_what_habitat_metrics(requests_mock): def test_what_project_weights(requests_mock): """Tests Water quality portal project weights query""" - request_url = "https://www.waterqualitydata.us/data/ProjectMonitoringLocationWeighting/Search?statecode=US%3A34&characteristicName=Chloride&zip=no" \ + request_url = "https://www.waterqualitydata.us/data/ProjectMonitoringLocationWeighting/Search?statecode=US%3A34&characteristicName=Chloride" \ "&mimeType=csv" response_file_path = 'data/wqp_project_weights.txt' mock_request(requests_mock, request_url, response_file_path) @@ -136,7 +136,7 @@ def test_what_project_weights(requests_mock): def test_what_activity_metrics(requests_mock): """Tests Water quality portal activity metrics query""" - request_url = "https://www.waterqualitydata.us/data/ActivityMetric/Search?statecode=US%3A34&characteristicName=Chloride&zip=no" \ + request_url = "https://www.waterqualitydata.us/data/ActivityMetric/Search?statecode=US%3A34&characteristicName=Chloride" \ "&mimeType=csv" response_file_path = 'data/wqp_activity_metrics.txt' mock_request(requests_mock, request_url, response_file_path) @@ -157,16 +157,9 @@ def mock_request(requests_mock, request_url, file_path): class TestAlterKwargs: """Tests for keyword alteration. """ - def test_alter_kwargs_zip(self): - """Tests that zip kwarg is altered correctly and warning is thrown.""" - kwargs = {"zip": "yes", "mimeType": "csv"} - with pytest.warns(UserWarning): - kwargs = _alter_kwargs(kwargs) - assert kwargs == {"zip": "no", "mimeType": "csv"} - def test_alter_kwargs_mimetype(self): """Tests that mimetype kwarg is altered correctly and warning is thrown.""" - kwargs = {"zip": "no", "mimeType": "geojson"} + kwargs = {"mimeType": "geojson"} with pytest.warns(UserWarning): kwargs = _alter_kwargs(kwargs) - assert kwargs == {"zip": "no", "mimeType": "csv"} \ No newline at end of file + assert kwargs == {"mimeType": "csv"} \ No newline at end of file From 7c7ba2db7dbc03868afdc6e9268c858dc887d591 Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Wed, 4 Sep 2024 14:39:07 -0500 Subject: [PATCH 05/16] add test, remove old wqx warning, add links to beta page --- dataretrieval/wqp.py | 50 +++++++++++++++++++++++++++---------- tests/data/wqp3_results.txt | 6 +++++ tests/wqp_test.py | 21 ++++++++++++++-- 3 files changed, 62 insertions(+), 15 deletions(-) create mode 100644 tests/data/wqp3_results.txt diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index db83b3c4..587b11bb 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -33,6 +33,8 @@ def get_results(ssl_check=True, legacy=True, **kwargs): Any WQP API parameter can be passed as a keyword argument to this function. More information about the API can be found at: https://www.waterqualitydata.us/#advanced=true + or the beta version of the WQX3.0 API at: + https://www.waterqualitydata.us/beta/#mimeType=csv&providers=NWIS&providers=STORET or the Swagger documentation at: https://www.waterqualitydata.us/data/swagger-ui/index.html?docExpansion=none&url=/data/v3/api-docs#/ @@ -104,22 +106,28 @@ def get_results(ssl_check=True, legacy=True, **kwargs): >>> # Get results within a bounding box >>> df, md = dataretrieval.wqp.get_results(bBox='-92.8,44.2,-88.9,46.0') + >>> # Get results using a new WQX3.0 profile + >>> df, md = dataretrieval.wqp.get_results( + ... legacy=False, siteid='UTAHDWQ_WQX-4993795', dataProfile='narrow' + ... ) + """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) if legacy is True: if 'dataProfile' in kwargs: if kwargs['dataProfile'] not in result_profiles_legacy: - raise TypeError('dataProfile is not a legacy profile name') + raise TypeError(f'dataProfile {kwargs['dataProfile']} is not a legacy profile name. ' + 'Please choose from "resultPhysChem", "biological", or "narrowResult"') url = wqp_url('Result') else: if 'dataProfile' in kwargs: if kwargs['dataProfile'] not in result_profiles_wqx3: - raise TypeError('dataProfile is not a WQX3.0 profile name') + raise TypeError(f'dataProfile {kwargs['dataProfile']} is not a WQX3.0 profile name. ' + 'Please choose from "fullPhysChem", "narrow", or "basicPhysChem"') else: kwargs['dataProfile'] = 'fullPhysChem' @@ -137,6 +145,8 @@ def what_sites(ssl_check=True, legacy=True, **kwargs): Any WQP API parameter can be passed as a keyword argument to this function. More information about the API can be found at: https://www.waterqualitydata.us/#advanced=true + or the beta version of the WQX3.0 API at: + https://www.waterqualitydata.us/beta/#mimeType=csv&providers=NWIS&providers=STORET or the Swagger documentation at: https://www.waterqualitydata.us/data/swagger-ui/index.html?docExpansion=none&url=/data/v3/api-docs#/ @@ -169,7 +179,6 @@ def what_sites(ssl_check=True, legacy=True, **kwargs): ... ) """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) @@ -191,6 +200,8 @@ def what_organizations(ssl_check=True, legacy=True, **kwargs): Any WQP API parameter can be passed as a keyword argument to this function. More information about the API can be found at: https://www.waterqualitydata.us/#advanced=true + or the beta version of the WQX3.0 API at: + https://www.waterqualitydata.us/beta/#mimeType=csv&providers=NWIS&providers=STORET or the Swagger documentation at: https://www.waterqualitydata.us/data/swagger-ui/index.html?docExpansion=none&url=/data/v3/api-docs#/ @@ -221,7 +232,6 @@ def what_organizations(ssl_check=True, legacy=True, **kwargs): >>> df, md = dataretrieval.wqp.what_organizations() """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) @@ -244,6 +254,8 @@ def what_projects(ssl_check=True, legacy=True, **kwargs): Any WQP API parameter can be passed as a keyword argument to this function. More information about the API can be found at: https://www.waterqualitydata.us/#advanced=true + or the beta version of the WQX3.0 API at: + https://www.waterqualitydata.us/beta/#mimeType=csv&providers=NWIS&providers=STORET or the Swagger documentation at: https://www.waterqualitydata.us/data/swagger-ui/index.html?docExpansion=none&url=/data/v3/api-docs#/ @@ -274,7 +286,6 @@ def what_projects(ssl_check=True, legacy=True, **kwargs): >>> df, md = dataretrieval.wqp.what_projects(huc='19') """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) @@ -297,6 +308,8 @@ def what_activities(ssl_check=True, legacy=True, **kwargs): Any WQP API parameter can be passed as a keyword argument to this function. More information about the API can be found at: https://www.waterqualitydata.us/#advanced=true + or the beta version of the WQX3.0 API at: + https://www.waterqualitydata.us/beta/#mimeType=csv&providers=NWIS&providers=STORET or the Swagger documentation at: https://www.waterqualitydata.us/data/swagger-ui/index.html?docExpansion=none&url=/data/v3/api-docs#/ @@ -329,10 +342,16 @@ def what_activities(ssl_check=True, legacy=True, **kwargs): ... statecode='US:11', startDateLo='12-30-2019', startDateHi='01-01-2020' ... ) + >>> # Get activities within Washington D.C. + >>> # using the WQX3.0 profile during a specific time period + >>> df, md = dataretrieval.wqp.what_activities( + ... legacy=False, + ... statecode='US:11', + ... startDateLo='12-30-2019', + ... startDateHi='01-01-2020' + ... ) """ - _warn_v3_profiles_outage() - kwargs = _alter_kwargs(kwargs) if legacy is True: @@ -354,6 +373,8 @@ def what_detection_limits(ssl_check=True, legacy=True, **kwargs): Any WQP API parameter can be passed as a keyword argument to this function. More information about the API can be found at: https://www.waterqualitydata.us/#advanced=true + or the beta version of the WQX3.0 API at: + https://www.waterqualitydata.us/beta/#mimeType=csv&providers=NWIS&providers=STORET or the Swagger documentation at: https://www.waterqualitydata.us/data/swagger-ui/index.html?docExpansion=none&url=/data/v3/api-docs#/ @@ -390,7 +411,6 @@ def what_detection_limits(ssl_check=True, legacy=True, **kwargs): ... ) """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) @@ -413,6 +433,8 @@ def what_habitat_metrics(ssl_check=True, legacy=True, **kwargs): Any WQP API parameter can be passed as a keyword argument to this function. More information about the API can be found at: https://www.waterqualitydata.us/#advanced=true + or the beta version of the WQX3.0 API at: + https://www.waterqualitydata.us/beta/#mimeType=csv&providers=NWIS&providers=STORET or the Swagger documentation at: https://www.waterqualitydata.us/data/swagger-ui/index.html?docExpansion=none&url=/data/v3/api-docs#/ @@ -443,7 +465,6 @@ def what_habitat_metrics(ssl_check=True, legacy=True, **kwargs): >>> df, md = dataretrieval.wqp.what_habitat_metrics(statecode='US:44') """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) @@ -466,6 +487,8 @@ def what_project_weights(ssl_check=True, legacy=True, **kwargs): Any WQP API parameter can be passed as a keyword argument to this function. More information about the API can be found at: https://www.waterqualitydata.us/#advanced=true + or the beta version of the WQX3.0 API at: + https://www.waterqualitydata.us/beta/#mimeType=csv&providers=NWIS&providers=STORET or the Swagger documentation at: https://www.waterqualitydata.us/data/swagger-ui/index.html?docExpansion=none&url=/data/v3/api-docs#/ @@ -499,7 +522,6 @@ def what_project_weights(ssl_check=True, legacy=True, **kwargs): ... ) """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) @@ -522,6 +544,8 @@ def what_activity_metrics(ssl_check=True, legacy=True, **kwargs): Any WQP API parameter can be passed as a keyword argument to this function. More information about the API can be found at: https://www.waterqualitydata.us/#advanced=true + or the beta version of the WQX3.0 API at: + https://www.waterqualitydata.us/beta/#mimeType=csv&providers=NWIS&providers=STORET or the Swagger documentation at: https://www.waterqualitydata.us/data/swagger-ui/index.html?docExpansion=none&url=/data/v3/api-docs#/ @@ -555,7 +579,6 @@ def what_activity_metrics(ssl_check=True, legacy=True, **kwargs): ... ) """ - _warn_v3_profiles_outage() kwargs = _alter_kwargs(kwargs) @@ -681,5 +704,6 @@ def _warn_legacy_use(): 'This function call is currently returning the legacy WQX format. ' 'This means that any USGS data served are stale as of March 2024. ' 'Please review the dataretrieval-python documentation for more ' - 'information on updated WQX3.0 profiles.' + 'information on updated WQX3.0 profiles. When WQX3.0 profiles are ' + 'available, setting legacy=False will return the freshest data.' ) diff --git a/tests/data/wqp3_results.txt b/tests/data/wqp3_results.txt new file mode 100644 index 00000000..634144ca --- /dev/null +++ b/tests/data/wqp3_results.txt @@ -0,0 +1,6 @@ +Org_Identifier,Org_FormalName,Project_Identifier,Project_Name,Project_QAPPApproved,Project_QAPPApprovalAgency,ProjectAttachment_FileName,ProjectAttachment_FileType,Location_Identifier,Location_Name,Location_Type,Location_Description,Location_State,Location_CountryName,Location_CountyName,Location_CountryCode,Location_StatePostalCode,Location_CountyCode,Location_HUCEightDigitCode,Location_HUCTwelveDigitCode,Location_TribalLandIndicator,Location_TribalLand,Location_Latitude,Location_Longitude,Location_HorzCoordReferenceSystemDatum,Location_LatitudeStandardized,Location_LongitudeStandardized,Location_HorzCoordStandardizedDatum,AlternateLocation_IdentifierCount,Activity_ActivityIdentifier,Activity_ActivityIdentifierUserSupplied,Activity_TypeCode,Activity_Media,Activity_MediaSubdivisionName,Activity_BottomDepthSamplingComponent,ActivityBiological_AssemblageSampled,ActivityBiological_ToxicityTestType,Activity_ConductingOrganization,Activity_Comment,ActivityLocation_Latitude,ActivityLocation_Longitude,ActivityLocation_HorzCoordReferenceSystemDatum,ActivityLocation_SourceMapScale,ActivityLocation_LatitudeStandardized,ActivityLocation_LongitudeStandardized,ActivityLocation_HorzCoordStandardizedDatum,ActivityLocation_HorzAccuracyMeasure,ActivityLocation_HorzAccuracyMeasureUnit,ActivityLocation_HorizontalAccuracyHorzCollectionMethod,ActivityLocation_Description,Activity_StartDate,Activity_StartTime,Activity_StartTimeZone,Activity_EndDate,Activity_EndTime,Activity_EndTimeZone,Activity_DepthHeightMeasure,Activity_DepthHeightMeasureUnit,Activity_BottomDepthAltitudeReferencePoint,Activity_ActivityRelativeDepth,Activity_TopDepthMeasure,Activity_TopDepthMeasureUnit,Activity_BottomDepthMeasure,Activity_BottomDepthMeasureUnit,SampleCollectionMethod_Identifier,SampleCollectionMethod_IdentifierContext,SampleCollectionMethod_Name,SampleCollectionMethod_QualifierTypeName,SampleCollectionMethod_Description,SampleCollectionMethod_EquipmentName,SampleCollectionMethod_EquipmentComment,SamplePrepMethod_Identifier,SamplePrepMethod_IdentifierContext,SamplePrepMethod_Name,SamplePrepMethod_QualifierType,SamplePrepMethod_Description,SamplePrepMethod_ContainerLabel,SamplePrepMethod_ContainerType,SamplePrepMethod_ContainerColor,SamplePrepMethod_ChemicalPreservativeUsed,SamplePrepMethod_ThermalPreservativeUsed,SamplePrepMethod_TransportStorageDescription,Activity_HydrologicCondition,Activity_HydrologicEvent,ActivityAttachment_FileName,ActivityAttachment_FileType,ActivityAttachment_FileDownload,Result_DataLoggerLine,Result_ResultDetectionCondition,Result_Characteristic,Result_CharacteristicUserSupplied,Result_CASNumber,Result_MethodSpeciation,Result_SampleFraction,ResultBiological_Intent,ResultBiological_IndividualIdentifier,ResultBiological_Taxon,ResultBiological_TaxonUserSupplied,ResultBiological_TaxonUserSuppliedReference,ResultBiological_UnidentifiedSpeciesIdentifier,ResultBiological_SampleTissueAnatomy,ResultBiological_GroupSummaryCount,GroupSummaryWeight_Measure,GroupSummaryWeightMeasure_Unit,ResultDepthHeight_Measure,ResultDepthHeight_MeasureUnit,ResultDepthHeight_AltitudeReferencePoint,ResultDepthHeight_SamplingPointName,ResultDepthHeight_SamplingPointType,ResultDepthHeight_SamplingPointPlaceInSeries,ResultDepthHeight_SamplingPointComment,ResultDepthHeight_RecordIdentifierUserSupplied,Result_MeasureIdentifier,Result_Measure,Result_MeasureUnit,Result_MeasureQualifierCode,Result_MeasureStatusIdentifier,Result_StatisticalBase,Result_StatisticalNValue,Result_MeasureType,Result_WeightBasis,Result_TimeBasis,Result_MeasureTemperatureBasis,Result_MeasureParticleSizeBasis,DataQuality_PrecisionValue,DataQuality_BiasValue,DataQuality_ConfidenceIntervalValue,DataQuality_UpperConfidenceLimitValue,DataQuality_LowerConfidenceLimitValue,DataQuality_ResultComment,DetectionLimit_TypeA,DetectionLimit_MeasureA,DetectionLimit_MeasureUnitA,DetectionLimit_CommentA,DetectionLimit_TypeB,DetectionLimit_MeasureB,DetectionLimit_MeasureUnitB,DetectionLimit_CommentB,LabInfo_LabSampleSplitRatio,LabInfo_LabAccreditationIndicator,LabInfo_LabAccreditationAuthority,LabInfo_TaxonAccreditationIndicator,LabInfo_TaxonAccreditationAuthority,ResultAnalyticalMethod_Identifier,ResultAnalyticalMethod_IdentifierContext,ResultAnalyticalMethod_Name,ResultAnalyticalMethod_QualifierType,ResultAnalyticalMethod_Description,Result_ComparableMethodIdentifier,Result_ComparableMethodIdentifierContext,Result_ComparableMethodModification,LabInfo_Name,LabInfo_AnalysisStartDate,LabInfo_AnalysisStartTime,LabInfo_AnalysisStartTimeZone,LabInfo_AnalysisEndDate,LabInfo_AnalysisEndTime,LabInfo_AnalysisEndTimeZone,LabInfo_LaboratoryComment,LabSamplePrepMethod_Identifier,LabSamplePrepMethod_IdentifierContext,LabSamplePrepMethod_Name,LabSamplePrepMethod_QualifierType,LabSamplePrepMethod_Description,LabSamplePrepMethod_StartDate,LabSamplePrepMethod_StartTime,LabSamplePrepMethod_StartTimeZone,LabSamplePrepMethod_EndDate,LabSamplePrepMethod_EndTime,LabSamplePrepMethod_EndTimeZone,LabSamplePrepMethod_DilutionFactor,ResultAttachment_FileName,ResultAttachment_FileType,ResultAttachment_FileDownload,ProviderName,Result_CharacteristicComparable,Result_CharacteristicGroup,Org_Type,LastChangeDate,USGSpcode +WIDNR_WQX,Wisconsin Department of Natural Resources,"[""CBSM_URSS_Madison""]","[""Urban Road Salt Study - Madison Sites""]",,,,,WIDNR_WQX-10032762,North Branch Pheasant Branch downstream of pond at Deming Way,River/Stream,,Wisconsin,United States,Dane,US,WI,25,7090002,,,,43.102665,-89.51866,NAD83,43.102665,-89.51866,NAD83,0,WIDNR_WQX-49176537,,Field Msr/Obs,Water,,,,,WIDNR_WQX,,43.102665,-89.51866,NAD83,,43.102665,-89.51866,NAD83,,,Interpolation-Other,,2011-08-08,13:55:00,CDT,2011-08-08,14:05:00,CDT,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Specific conductance,,,,,,,,,,,,,,,,,,,,,,,STORET-113777847,471,uS/cm,,Final,,,Actual,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,STORET,,Physical,*State Government US,Fri Jun 15 09:44:33 GMT 2018, +WIDNR_WQX,Wisconsin Department of Natural Resources,"[""CBSM_URSS_Madison""]","[""Urban Road Salt Study - Madison Sites""]",,,,,WIDNR_WQX-10032762,North Branch Pheasant Branch downstream of pond at Deming Way,River/Stream,,Wisconsin,United States,Dane,US,WI,25,7090002,,,,43.102665,-89.51866,NAD83,43.102665,-89.51866,NAD83,0,WIDNR_WQX-47619240,,Field Msr/Obs,Water,,,,,WIDNR_WQX,,43.102665,-89.51866,NAD83,,43.102665,-89.51866,NAD83,,,Interpolation-Other,,2011-07-06,08:35:00,CDT,2011-07-06,08:45:00,CDT,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Specific conductance,,,,,,,,,,,,,,,,,,,,,,,STORET-113777841,860,uS/cm,,Final,,,Actual,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,STORET,,Physical,*State Government US,Fri Jun 15 09:44:33 GMT 2018, +WIDNR_WQX,Wisconsin Department of Natural Resources,"[""CBSM_URSS_Madison""]","[""Urban Road Salt Study - Madison Sites""]",,,,,WIDNR_WQX-10032762,North Branch Pheasant Branch downstream of pond at Deming Way,River/Stream,,Wisconsin,United States,Dane,US,WI,25,7090002,,,,43.102665,-89.51866,NAD83,43.102665,-89.51866,NAD83,0,WIDNR_WQX-45822640,,Field Msr/Obs,Water,,,,,WIDNR_WQX,,43.102665,-89.51866,NAD83,,43.102665,-89.51866,NAD83,,,Interpolation-Other,,2011-05-09,12:20:00,CDT,2011-05-09,12:30:00,CDT,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Specific conductance,,,,,,,,,,,,,,,,,,,,,,,STORET-113777835,1000,uS/cm,,Final,,,Actual,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,STORET,,Physical,*State Government US,Fri Jun 15 09:44:33 GMT 2018, +WIDNR_WQX,Wisconsin Department of Natural Resources,"[""CBSM_URSS_Madison""]","[""Urban Road Salt Study - Madison Sites""]",,,,,WIDNR_WQX-10032762,North Branch Pheasant Branch downstream of pond at Deming Way,River/Stream,,Wisconsin,United States,Dane,US,WI,25,7090002,,,,43.102665,-89.51866,NAD83,43.102665,-89.51866,NAD83,0,WIDNR_WQX-46495059,,Field Msr/Obs,Water,,,,,WIDNR_WQX,,43.102665,-89.51866,NAD83,,43.102665,-89.51866,NAD83,,,Interpolation-Other,,2011-06-05,14:45:00,CDT,2011-06-05,14:55:00,CDT,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Specific conductance,,,,,,,,,,,,,,,,,,,,,,,STORET-113777838,800,uS/cm,,Final,,,Actual,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,STORET,,Physical,*State Government US,Fri Jun 15 09:44:33 GMT 2018, +WIDNR_WQX,Wisconsin Department of Natural Resources,"[""CBSM_URSS_Madison""]","[""Urban Road Salt Study - Madison Sites""]",,,,,WIDNR_WQX-10032762,North Branch Pheasant Branch downstream of pond at Deming Way,River/Stream,,Wisconsin,United States,Dane,US,WI,25,7090002,,,,43.102665,-89.51866,NAD83,43.102665,-89.51866,NAD83,0,WIDNR_WQX-50689894,,Field Msr/Obs,Water,,,,,WIDNR_WQX,,43.102665,-89.51866,NAD83,,43.102665,-89.51866,NAD83,,,Interpolation-Other,,2011-09-11,16:10:00,CDT,2011-09-11,16:20:00,CDT,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,Specific conductance,,,,,,,,,,,,,,,,,,,,,,,STORET-113777850,750,uS/cm,,Final,,,Actual,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,STORET,,Physical,*State Government US,Fri Jun 15 09:44:33 GMT 2018, diff --git a/tests/wqp_test.py b/tests/wqp_test.py index 8de4360a..fd34ae4b 100755 --- a/tests/wqp_test.py +++ b/tests/wqp_test.py @@ -11,8 +11,8 @@ _alter_kwargs) -def test_get_ratings(requests_mock): - """Tests water quality portal ratings query""" +def test_get_results(requests_mock): + """Tests water quality portal results query""" request_url = "https://www.waterqualitydata.us/data/Result/Search?siteid=WIDNR_WQX-10032762" \ "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" \ "&mimeType=csv" @@ -28,6 +28,23 @@ def test_get_ratings(requests_mock): assert md.header == {"mock_header": "value"} assert md.comment is None +def test_get_results_WQX3(requests_mock): + """Tests water quality portal results query with new WQX3.0 profile""" + request_url = "https://www.waterqualitydata.us/wqx3/Result/search?siteid=WIDNR_WQX-10032762" \ + "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" \ + "&mimeType=csv&dataProfile=fullPhysChem" + response_file_path = 'data/wqp3_results.txt' + mock_request(requests_mock, request_url, response_file_path) + df, md = get_results(legacy=False, siteid='WIDNR_WQX-10032762', + characteristicName = 'Specific conductance', + startDateLo='05-01-2011', startDateHi='09-30-2011') + assert type(df) is DataFrame + assert df.size == 900 + assert md.url == request_url + assert isinstance(md.query_time, datetime.timedelta) + assert md.header == {"mock_header": "value"} + assert md.comment is None + def test_what_sites(requests_mock): """Tests Water quality portal sites query""" From 13bb200b04f952c8b31d570714b4260c58f03538 Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Wed, 4 Sep 2024 14:50:39 -0500 Subject: [PATCH 06/16] fix type error syntax --- dataretrieval/wqp.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 587b11bb..22a55e6d 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -118,16 +118,20 @@ def get_results(ssl_check=True, legacy=True, **kwargs): if legacy is True: if 'dataProfile' in kwargs: if kwargs['dataProfile'] not in result_profiles_legacy: - raise TypeError(f'dataProfile {kwargs['dataProfile']} is not a legacy profile name. ' - 'Please choose from "resultPhysChem", "biological", or "narrowResult"') + raise TypeError(f""" + dataProfile "{kwargs['dataProfile']}" is not a legacy profile name. + Please choose from "resultPhysChem", "biological", or "narrowResult" + """) url = wqp_url('Result') else: if 'dataProfile' in kwargs: if kwargs['dataProfile'] not in result_profiles_wqx3: - raise TypeError(f'dataProfile {kwargs['dataProfile']} is not a WQX3.0 profile name. ' - 'Please choose from "fullPhysChem", "narrow", or "basicPhysChem"') + raise TypeError(f""" + dataProfile "{kwargs['dataProfile']}" is not a WQX3.0 profile name. + Please choose from "fullPhysChem", "narrow", or "basicPhysChem" + """) else: kwargs['dataProfile'] = 'fullPhysChem' From 9061f4c18486c0c9067dddd11d611cb273d176cd Mon Sep 17 00:00:00 2001 From: thodson Date: Mon, 16 Sep 2024 11:42:02 -0500 Subject: [PATCH 07/16] Docstrings and linting --- dataretrieval/wqp.py | 367 +++++++++++++++++++++---------------------- 1 file changed, 178 insertions(+), 189 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 22a55e6d..5609fa26 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -8,6 +8,9 @@ - implement other services like Organization, Activity, etc. """ +from __future__ import annotations +from typing import TYPE_CHECKING + import warnings from io import StringIO @@ -15,19 +18,32 @@ from .utils import BaseMetadata, query +if TYPE_CHECKING: + from pandas import DataFrame + + result_profiles_wqx3 = ['fullPhysChem', 'narrow', 'basicPhysChem'] -result_profiles_legacy = ['resultPhysChem', 'biological', - 'narrowResult'] +result_profiles_legacy = ['resultPhysChem', 'biological', 'narrowResult'] activity_profiles_legacy = ['activityAll'] services_wqx3 = ['Result', 'Station', 'Activity'] -services_legacy = ['Result', 'Station', 'Activity', - 'Organization', 'Project', - 'ProjectMonitoringLocationWeighting', - 'ActivityMetric', - 'ResultDetectionQuantitationLimit', - 'BiologicalMetric'] - -def get_results(ssl_check=True, legacy=True, **kwargs): +services_legacy = [ + 'Activity', + 'ActivityMetric', + 'BiologicalMetric', + 'Organization', + 'Project', + 'ProjectMonitoringLocationWeighting', + 'Result', + 'ResultDetectionQuantitationLimit', + 'Station', + ] + + +def get_results( + ssl_check=True, + legacy=True, + **kwargs, +) -> tuple[DataFrame, WQP_Metadata]: """Query the WQP for results. Any WQP API parameter can be passed as a keyword argument to this function. @@ -40,52 +56,46 @@ def get_results(ssl_check=True, legacy=True, **kwargs): Parameters ---------- - ssl_check: bool - If True, check the SSL certificate. Default is True. If False, SSL - certificate is not checked. - legacy: bool - If True, returns the legacy WQX data profile and warns the user of - the issues associated with it. If False, returns the new WQX3.0 - profile, if available. Defaults to True. - dataProfile: string - Describes the type of columns to return with the result dataset. - Most recent WQX3 profiles include 'fullPhysChem', 'narrow', and - 'basicPhysChem'. Legacy profiles include 'resultPhysChem', - 'biological', and 'narrowResult'. - siteid: string - Concatenate an agency code, a hyphen ("-"), and a site-identification - number. - statecode: string - Concatenate 'US', a colon (":"), and a FIPS numeric code - (Example: Illinois is US:17) - countycode: string - A FIPS county code - huc: string - One or more eight-digit hydrologic units, delimited by semicolons. - bBox: string - Bounding box (Example: bBox=-92.8,44.2,-88.9,46.0) - lat: string - Latitude for radial search, expressed in decimal degrees, WGS84 - long: string - Longitude for radial search - within: string - Distance for a radial search, expressed in decimal miles - pCode: string - One or more five-digit USGS parameter codes, separated by semicolons. + ssl_check : bool, optional + Check the SSL certificate. + legacy : bool, optional + Return the legacy WQX data profile. Default is True. + dataProfile : string, optional + Specifies the data fields returned by the query. + WQX3.0 profiles include 'fullPhysChem', 'narrow', and 'basicPhysChem'. + Legacy profiles include 'resultPhysChem','biological', and + 'narrowResult'. Default is 'fullPhysChem'. + siteid : string + Monitoring location identified by agency code, a hyphen, and + identification number (Example: "USGS-05586100"). + statecode : string + US state FIPS code (Example: Illinois is "US:17"). + countycode : string + US county FIPS code. + huc : string + Eight-digit hydrologic unit (HUC), delimited by semicolons. + bBox : string + Search bounding box (Example: bBox=-92.8,44.2,-88.9,46.0) + lat : string + Radial-search central latitude in WGS84 decimal degrees. + long : string + Radial-search central longitude in WGS84 decimal degrees. + within : string + Radial-search distance in decimal miles. + pCode : string + Five-digit USGS parameter code, delimited by semicolons. NWIS only. - startDateLo: string + startDateLo : string Date of earliest desired data-collection activity, expressed as 'MM-DD-YYYY' - startDateHi: string + startDateHi : string Date of last desired data-collection activity, expressed as 'MM-DD-YYYY' - characteristicName: string + characteristicName : string One or more case-sensitive characteristic names, separated by - semicolons. (See https://www.waterqualitydata.us/public_srsnames/ - for available characteristic names) - mimeType: string - String specifying the output format which is 'csv' by default but can - be 'geojson' + semicolons (https://www.waterqualitydata.us/public_srsnames/). + mimeType : string + Output format. Only 'csv' is supported at this time. Returns ------- @@ -116,34 +126,38 @@ def get_results(ssl_check=True, legacy=True, **kwargs): kwargs = _alter_kwargs(kwargs) if legacy is True: - if 'dataProfile' in kwargs: - if kwargs['dataProfile'] not in result_profiles_legacy: - raise TypeError(f""" - dataProfile "{kwargs['dataProfile']}" is not a legacy profile name. - Please choose from "resultPhysChem", "biological", or "narrowResult" - """) - - url = wqp_url('Result') - + if "dataProfile" in kwargs: + if kwargs["dataProfile"] not in result_profiles_legacy: + raise TypeError( + f"dataProfile '{kwargs["dataProfile"]}' is not a legacy profile.", + f"Please choose from {result_profiles_legacy}.", + ) + + url = wqp_url("Result") + else: if 'dataProfile' in kwargs: if kwargs['dataProfile'] not in result_profiles_wqx3: - raise TypeError(f""" - dataProfile "{kwargs['dataProfile']}" is not a WQX3.0 profile name. - Please choose from "fullPhysChem", "narrow", or "basicPhysChem" - """) + raise TypeError( + f"dataProfile '{kwargs["dataProfile"]}' is not a valid WQX3.0" + f"profile. Please choose {result_profiles_wqx3}.", + ) else: - kwargs['dataProfile'] = 'fullPhysChem' - + kwargs["dataProfile"] = "fullPhysChem" + url = wqx3_url('Result') - response = query(url, kwargs, delimiter=';', ssl_check=ssl_check) + response = query(url, kwargs, delimiter=";", ssl_check=ssl_check) - df = pd.read_csv(StringIO(response.text), delimiter=',') + df = pd.read_csv(StringIO(response.text), delimiter=",") return df, WQP_Metadata(response) -def what_sites(ssl_check=True, legacy=True, **kwargs): +def what_sites( + ssl_check=True, + legacy=True, + **kwargs, +) -> tuple[DataFrame, WQP_Metadata]: """Search WQP for sites within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -156,10 +170,9 @@ def what_sites(ssl_check=True, legacy=True, **kwargs): Parameters ---------- - ssl_check: bool - If True, check the SSL certificate. Default is True. If False, SSL - certificate is not checked. - legacy: bool + ssl_check : bool, optional + Check the SSL certificate. Default is True. + legacy : bool, optional If True, returns the legacy WQX data profile and warns the user of the issues associated with it. If False, returns the new WQX3.0 profile, if available. Defaults to True. @@ -190,7 +203,7 @@ def what_sites(ssl_check=True, legacy=True, **kwargs): url = wqp_url('Station') else: url = wqx3_url('Station') - + response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) df = pd.read_csv(StringIO(response.text), delimiter=',') @@ -198,7 +211,11 @@ def what_sites(ssl_check=True, legacy=True, **kwargs): return df, WQP_Metadata(response) -def what_organizations(ssl_check=True, legacy=True, **kwargs): +def what_organizations( + ssl_check=True, + legacy=True, + **kwargs, +) -> tuple[DataFrame, WQP_Metadata]: """Search WQP for organizations within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -211,21 +228,18 @@ def what_organizations(ssl_check=True, legacy=True, **kwargs): Parameters ---------- - ssl_check: bool - If True, check the SSL certificate. Default is True. If False, SSL - certificate is not checked. - legacy: bool - If True, returns the legacy WQX data profile and warns the user of - the issues associated with it. If False, returns the new WQX3.0 - profile, if available. Defaults to True. - **kwargs: optional + ssl_check : bool, optional + Check the SSL certificate. Default is True. + legacy : bool, optional + Return the legacy WQX data profile. Default is True. + **kwargs : optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` Returns ------- - df: ``pandas.DataFrame`` + df : ``pandas.DataFrame`` Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.Metadata` + md : :obj:`dataretrieval.utils.Metadata` Custom metadata object pertaining to the query. Examples @@ -265,21 +279,18 @@ def what_projects(ssl_check=True, legacy=True, **kwargs): Parameters ---------- - ssl_check: bool - If True, check the SSL certificate. Default is True. If False, SSL - certificate is not checked. - legacy: bool - If True, returns the legacy WQX data profile and warns the user of - the issues associated with it. If False, returns the new WQX3.0 - profile, if available. Defaults to True. - **kwargs: optional + ssl_check : bool, optional + Check the SSL certificate. Default is True. + legacy : bool, optional + Return the legacy WQX data profile. Default is True. + **kwargs : optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` Returns ------- - df: ``pandas.DataFrame`` + df : ``pandas.DataFrame`` Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.Metadata` + md : :obj:`dataretrieval.utils.Metadata` Custom metadata object pertaining to the query. Examples @@ -306,7 +317,11 @@ def what_projects(ssl_check=True, legacy=True, **kwargs): return df, WQP_Metadata(response) -def what_activities(ssl_check=True, legacy=True, **kwargs): +def what_activities( + ssl_check=True, + legacy=True, + **kwargs, +) -> tuple[DataFrame, WQP_Metadata]: """Search WQP for activities within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -319,21 +334,18 @@ def what_activities(ssl_check=True, legacy=True, **kwargs): Parameters ---------- - ssl_check: bool - If True, check the SSL certificate. Default is True. If False, SSL - certificate is not checked. - legacy: bool - If True, returns the legacy WQX data profile and warns the user of - the issues associated with it. If False, returns the new WQX3.0 - profile, if available. Defaults to True. - **kwargs: optional + ssl_check : bool, optional + Check the SSL certificate. Default is True. + legacy : bool, optional + Return the legacy WQX data profile. Default is True. + **kwargs : optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` Returns ------- - df: ``pandas.DataFrame`` + df : ``pandas.DataFrame`` Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.Metadata` + md : :obj:`dataretrieval.utils.Metadata` Custom metadata object pertaining to the query. Examples @@ -359,18 +371,22 @@ def what_activities(ssl_check=True, legacy=True, **kwargs): kwargs = _alter_kwargs(kwargs) if legacy is True: - url = wqp_url('Activity') + url = wqp_url("Activity") else: - url = wqx3_url('Activity') + url = wqx3_url("Activity") - response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) + response = query(url, payload=kwargs, delimiter=";", ssl_check=ssl_check) - df = pd.read_csv(StringIO(response.text), delimiter=',') + df = pd.read_csv(StringIO(response.text), delimiter=",") return df, WQP_Metadata(response) -def what_detection_limits(ssl_check=True, legacy=True, **kwargs): +def what_detection_limits( + ssl_check=True, + legacy=True, + **kwargs, +) -> tuple[DataFrame, WQP_Metadata]: """Search WQP for result detection limits within a region with specific data. @@ -384,21 +400,18 @@ def what_detection_limits(ssl_check=True, legacy=True, **kwargs): Parameters ---------- - ssl_check: bool - If True, check the SSL certificate. Default is True. If False, SSL - certificate is not checked. - legacy: bool - If True, returns the legacy WQX data profile and warns the user of - the issues associated with it. If False, returns the new WQX3.0 - profile, if available. Defaults to True. - **kwargs: optional + ssl_check : bool + Check the SSL certificate. Default is True. + legacy : bool + Return the legacy WQX data profile. Default is True. + **kwargs : optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` Returns ------- - df: ``pandas.DataFrame`` + df : ``pandas.DataFrame`` Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.Metadata` + md : :obj:`dataretrieval.utils.Metadata` Custom metadata object pertaining to the query. Examples @@ -431,7 +444,11 @@ def what_detection_limits(ssl_check=True, legacy=True, **kwargs): return df, WQP_Metadata(response) -def what_habitat_metrics(ssl_check=True, legacy=True, **kwargs): +def what_habitat_metrics( + ssl_check=True, + legacy=True, + **kwargs, +) -> tuple[DataFrame, WQP_Metadata]: """Search WQP for habitat metrics within a region with specific data. Any WQP API parameter can be passed as a keyword argument to this function. @@ -444,21 +461,18 @@ def what_habitat_metrics(ssl_check=True, legacy=True, **kwargs): Parameters ---------- - ssl_check: bool - If True, check the SSL certificate. Default is True. If False, SSL - certificate is not checked. - legacy: bool - If True, returns the legacy WQX data profile and warns the user of - the issues associated with it. If False, returns the new WQX3.0 - profile, if available. Defaults to True. - **kwargs: optional + ssl_check : bool + Check the SSL certificate. Default is True. + legacy : bool + Return the legacy WQX data profile. Default is True. + **kwargs : optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` Returns ------- - df: ``pandas.DataFrame`` + df : ``pandas.DataFrame`` Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.Metadata` + md : :obj:`dataretrieval.utils.Metadata` Custom metadata object pertaining to the query. Examples @@ -498,21 +512,18 @@ def what_project_weights(ssl_check=True, legacy=True, **kwargs): Parameters ---------- - ssl_check: bool - If True, check the SSL certificate. Default is True. If False, SSL - certificate is not checked. - legacy: bool - If True, returns the legacy WQX data profile and warns the user of - the issues associated with it. If False, returns the new WQX3.0 - profile, if available. Defaults to True. - **kwargs: optional + ssl_check : bool + Check the SSL certificate. Default is True. + legacy : bool + Retrun the legacy WQX data profile. Default is True. + **kwargs : optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` Returns ------- - df: ``pandas.DataFrame`` + df : ``pandas.DataFrame`` Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.Metadata` + md : :obj:`dataretrieval.utils.Metadata` Custom metadata object pertaining to the query. Examples @@ -555,21 +566,18 @@ def what_activity_metrics(ssl_check=True, legacy=True, **kwargs): Parameters ---------- - ssl_check: bool - If True, check the SSL certificate. Default is True. If False, SSL - certificate is not checked. - legacy: bool - If True, returns the legacy WQX data profile and warns the user of - the issues associated with it. If False, returns the new WQX3.0 - profile, if available. Defaults to True. - **kwargs: optional + ssl_check : bool + Check the SSL certificate. Default is True. + legacy : bool + Return the legacy WQX data profile. Default is True. + **kwargs : optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` Returns ------- - df: ``pandas.DataFrame`` + df : ``pandas.DataFrame`` Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.Metadata` + md : :obj:`dataretrieval.utils.Metadata` Custom metadata object pertaining to the query. Examples @@ -609,6 +617,7 @@ def wqp_url(service): base_url = 'https://www.waterqualitydata.us/data/' return f'{base_url}{service}/Search?' + def wqx3_url(service): """Construct the WQP URL for a given WQX 3.0 service.""" if service not in services_wqx3: @@ -643,12 +652,12 @@ def __init__(self, response, **parameters) -> None: response: Response Response object from requests module - parameters: unpacked dictionary + parameters: dict Unpacked dictionary of the parameters supplied in the request Returns ------- - md: :obj:`dataretrieval.wqp.WQP_Metadata` + md : :obj:`dataretrieval.wqp.WQP_Metadata` A ``dataretrieval`` custom :obj:`dataretrieval.wqp.WQP_Metadata` object. """ @@ -659,12 +668,12 @@ def __init__(self, response, **parameters) -> None: @property def site_info(self): - if 'sites' in self._parameters: - return what_sites(sites=parameters['sites']) - elif 'site' in self._parameters: - return what_sites(sites=parameters['site']) - elif 'site_no' in self._parameters: - return what_sites(sites=parameters['site_no']) + if "sites" in self._parameters: + return what_sites(sites=parameters["sites"]) + elif "site" in self._parameters: + return what_sites(sites=parameters["site"]) + elif "site_no" in self._parameters: + return what_sites(sites=parameters["site_no"]) def _alter_kwargs(kwargs): @@ -673,41 +682,21 @@ def _alter_kwargs(kwargs): Not all query parameters are currently supported by ``dataretrieval``, so this function is used to set some of them and raise warnings to the user so they are aware of which are being hard-set. - """ - if kwargs.get('mimeType', 'csv') == 'geojson': - warnings.warn('GeoJSON not yet supported, mimeType set to csv.') - kwargs['mimeType'] = 'csv' + if kwargs.get("mimeType", "csv") == "geojson": + warnings.warn("GeoJSON not yet supported, mimeType set to csv.") + kwargs["mimeType"] = "csv" return kwargs -def _warn_v3_profiles_outage(): - """Private function for warning message about WQX 3.0 profiles - """ - - warnings.warn( - 'USGS discrete water quality data availability ' - 'and format are changing. As of March 2024, ' - 'the data obtained from legacy profiles will not ' - 'include new USGS data or recent updates to existing ' - 'data. To view the status of changes in data ' - 'availability and code functionality, visit: ' - 'https://doi-usgs.github.io/dataRetrieval/articles/Status.html. ' - 'If you have additional questions about these changes, ' - 'email CompTools@usgs.gov.' - ) - def _warn_legacy_use(): - """Private function for warning message about using legacy profiles - or services - """ - - warnings.warn( - 'This function call is currently returning the legacy WQX format. ' - 'This means that any USGS data served are stale as of March 2024. ' - 'Please review the dataretrieval-python documentation for more ' - 'information on updated WQX3.0 profiles. When WQX3.0 profiles are ' - 'available, setting legacy=False will return the freshest data.' + message = ( + "This function call will return the legacy WQX format, " + "which means USGS data have not been updated since March 2024. " + "Please review the dataretrieval-python documentation for more " + "information on updated WQX3.0 profiles. Setting `legacy=False` " + "will remove this warning." ) + warnings.warn(message) From 85f279141e6d772e137559f452a8c2cbb007a801 Mon Sep 17 00:00:00 2001 From: thodson Date: Mon, 16 Sep 2024 16:29:27 -0500 Subject: [PATCH 08/16] Add wqx3 warning --- dataretrieval/wqp.py | 101 ++++++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 5609fa26..0ff0992a 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING import warnings +from warnings import DeprecationWarning, UserWarning from io import StringIO import pandas as pd @@ -99,9 +100,9 @@ def get_results( Returns ------- - df: ``pandas.DataFrame`` + df : ``pandas.DataFrame`` Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.Metadata` + md : :obj:`dataretrieval.utils.Metadata` Custom ``dataretrieval`` metadata object pertaining to the query. Examples @@ -123,14 +124,14 @@ def get_results( """ - kwargs = _alter_kwargs(kwargs) + _check_mimetype(kwargs) if legacy is True: if "dataProfile" in kwargs: if kwargs["dataProfile"] not in result_profiles_legacy: raise TypeError( f"dataProfile '{kwargs["dataProfile"]}' is not a legacy profile.", - f"Please choose from {result_profiles_legacy}.", + f"Valid options are {result_profiles_legacy}.", ) url = wqp_url("Result") @@ -140,7 +141,7 @@ def get_results( if kwargs['dataProfile'] not in result_profiles_wqx3: raise TypeError( f"dataProfile '{kwargs["dataProfile"]}' is not a valid WQX3.0" - f"profile. Please choose {result_profiles_wqx3}.", + f"profile. Valid options are {result_profiles_wqx3}.", ) else: kwargs["dataProfile"] = "fullPhysChem" @@ -176,14 +177,14 @@ def what_sites( If True, returns the legacy WQX data profile and warns the user of the issues associated with it. If False, returns the new WQX3.0 profile, if available. Defaults to True. - **kwargs: optional + **kwargs : optional Accepts the same parameters as :obj:`dataretrieval.wqp.get_results` Returns ------- - df: ``pandas.DataFrame`` + df : ``pandas.DataFrame`` Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.Metadata` + md : :obj:`dataretrieval.utils.Metadata` Custom metadata object pertaining to the query. Examples @@ -197,7 +198,7 @@ def what_sites( """ - kwargs = _alter_kwargs(kwargs) + _check_mimetype(kwargs) if legacy is True: url = wqp_url('Station') @@ -251,12 +252,12 @@ def what_organizations( """ - kwargs = _alter_kwargs(kwargs) + _check_mimetype(kwargs) if legacy is True: url = wqp_url('Organization') else: - print('No WQX3.0 profile currently available, returning legacy profile.') + print('WQX3.0 profile not available, returning legacy profile.') url = wqp_url('Organization') response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) @@ -302,12 +303,12 @@ def what_projects(ssl_check=True, legacy=True, **kwargs): """ - kwargs = _alter_kwargs(kwargs) + _check_mimetype(kwargs) if legacy is True: url = wqp_url('Project') else: - print('No WQX3.0 profile currently available, returning legacy profile.') + print('WQX3.0 profile not available, returning legacy profile.') url = wqp_url('Project') response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) @@ -368,7 +369,7 @@ def what_activities( ... ) """ - kwargs = _alter_kwargs(kwargs) + _check_mimetype(kwargs) if legacy is True: url = wqp_url("Activity") @@ -429,12 +430,12 @@ def what_detection_limits( """ - kwargs = _alter_kwargs(kwargs) + _check_mimetype(kwargs) if legacy is True: url = wqp_url('ResultDetectionQuantitationLimit') else: - print('No WQX3.0 profile currently available, returning legacy profile.') + print('WQX3.0 profile not available, returning legacy profile.') url = wqp_url('ResultDetectionQuantitationLimit') response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) @@ -484,12 +485,12 @@ def what_habitat_metrics( """ - kwargs = _alter_kwargs(kwargs) + _check_mimetype(kwargs) if legacy is True: url = wqp_url('BiologicalMetric') else: - print('No WQX3.0 profile currently available, returning legacy profile.') + print('WQX3.0 profile not available, returning legacy profile.') url = wqp_url('BiologicalMetric') response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) @@ -538,12 +539,12 @@ def what_project_weights(ssl_check=True, legacy=True, **kwargs): """ - kwargs = _alter_kwargs(kwargs) + _check_mimetype(kwargs) if legacy is True: url = wqp_url('ProjectMonitoringLocationWeighting') else: - print('No WQX3.0 profile currently available, returning legacy profile.') + print('WQX3.0 profile not available, returning legacy profile.') url = wqp_url('ProjectMonitoringLocationWeighting') response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) @@ -592,12 +593,12 @@ def what_activity_metrics(ssl_check=True, legacy=True, **kwargs): """ - kwargs = _alter_kwargs(kwargs) + _check_mimetype(kwargs) if legacy is True: url = wqp_url('ActivityMetric') else: - print('No WQX3.0 profile currently available, returning legacy profile.') + print('WQX3.0 profile not available, returning legacy profile.') url = wqp_url('ActivityMetric') response = query(url, payload=kwargs, delimiter=';', ssl_check=ssl_check) @@ -610,19 +611,30 @@ def what_activity_metrics(ssl_check=True, legacy=True, **kwargs): def wqp_url(service): """Construct the WQP URL for a given service.""" + base_url = 'https://www.waterqualitydata.us/data/' _warn_legacy_use() if service not in services_legacy: - raise TypeError('Legacy service not recognized') - base_url = 'https://www.waterqualitydata.us/data/' + raise TypeError( + 'Legacy service not recognized. Valid options are', + f'{services_legacy}.', + ) + return f'{base_url}{service}/Search?' def wqx3_url(service): """Construct the WQP URL for a given WQX 3.0 service.""" - if service not in services_wqx3: - raise TypeError('WQX3.0 service not recognized') + base_url = 'https://www.waterqualitydata.us/wqx3/' + _warn_wqx3_use() + + if service not in services_wqx3: + raise TypeError( + 'WQX3.0 service not recognized. Valid options are', + f'{services_wqx3}.', + ) + return f'{base_url}{service}/search?' @@ -633,13 +645,13 @@ class WQP_Metadata(BaseMetadata): ---------- url : str Response url - query_time: datetme.timedelta + query_time : datetme.timedelta Response elapsed time - header: requests.structures.CaseInsensitiveDict + header : requests.structures.CaseInsensitiveDict Response headers - comments: None + comments : None Metadata comments. WQP does not return comments. - site_info: tuple[pd.DataFrame, NWIS_Metadata] | None + site_info : tuple[pd.DataFrame, NWIS_Metadata] | None Site information if the query included `sites`, `site` or `site_no`. """ @@ -649,10 +661,10 @@ def __init__(self, response, **parameters) -> None: Parameters ---------- - response: Response + response : Response Response object from requests module - parameters: dict + parameters : dict Unpacked dictionary of the parameters supplied in the request Returns @@ -676,19 +688,20 @@ def site_info(self): return what_sites(sites=parameters["site_no"]) -def _alter_kwargs(kwargs): - """Private function to manipulate **kwargs. +def _check_mimetype(kwargs): + mimetype = kwargs.get("mimeType") + if mimetype == "geojson": + raise NotImplementedError("GeoJSON not yet supported. Set 'mimeType=csv'.") + elif mimetype != "csv": + raise ValueError("Invalid mimeType. Set 'mimeType=csv'.") - Not all query parameters are currently supported by ``dataretrieval``, - so this function is used to set some of them and raise warnings to the - user so they are aware of which are being hard-set. - """ - - if kwargs.get("mimeType", "csv") == "geojson": - warnings.warn("GeoJSON not yet supported, mimeType set to csv.") - kwargs["mimeType"] = "csv" - return kwargs +def _warn_wqx3_use(): + message = ( + "Support for the WQX3.0 profiles is experimental. " + "Queries may be slow or fail intermitttently." + ) + warnings.warn(message, UserWarning) def _warn_legacy_use(): @@ -699,4 +712,4 @@ def _warn_legacy_use(): "information on updated WQX3.0 profiles. Setting `legacy=False` " "will remove this warning." ) - warnings.warn(message) + warnings.warn(message, DeprecationWarning) From 9bbae87fac01394cdf94957c96cf3d812dea3b15 Mon Sep 17 00:00:00 2001 From: thodson Date: Mon, 16 Sep 2024 17:11:31 -0500 Subject: [PATCH 09/16] Quick fix --- dataretrieval/wqp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 0ff0992a..2d68d7e1 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING import warnings -from warnings import DeprecationWarning, UserWarning from io import StringIO import pandas as pd @@ -692,7 +691,7 @@ def _check_mimetype(kwargs): mimetype = kwargs.get("mimeType") if mimetype == "geojson": raise NotImplementedError("GeoJSON not yet supported. Set 'mimeType=csv'.") - elif mimetype != "csv": + elif mimetype != "csv" and mimetype is not None: raise ValueError("Invalid mimeType. Set 'mimeType=csv'.") From 83444be68c007fd6f9ca07d5eb9964a53a2f41ff Mon Sep 17 00:00:00 2001 From: thodson Date: Mon, 16 Sep 2024 17:15:27 -0500 Subject: [PATCH 10/16] Reorder services --- dataretrieval/wqp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 2d68d7e1..501cb4da 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -22,10 +22,10 @@ from pandas import DataFrame -result_profiles_wqx3 = ['fullPhysChem', 'narrow', 'basicPhysChem'] +result_profiles_wqx3 = ['basicPhysChem', 'fullPhysChem', 'narrow'] result_profiles_legacy = ['resultPhysChem', 'biological', 'narrowResult'] activity_profiles_legacy = ['activityAll'] -services_wqx3 = ['Result', 'Station', 'Activity'] +services_wqx3 = ['Activity', 'Result', 'Station'] services_legacy = [ 'Activity', 'ActivityMetric', From c2b4d12d865d4cfc3f1b7034cdaac3ca41f384eb Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Tue, 17 Sep 2024 09:13:48 -0500 Subject: [PATCH 11/16] try to fix linting issue --- dataretrieval/wqp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 501cb4da..383b7546 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -129,7 +129,7 @@ def get_results( if "dataProfile" in kwargs: if kwargs["dataProfile"] not in result_profiles_legacy: raise TypeError( - f"dataProfile '{kwargs["dataProfile"]}' is not a legacy profile.", + f"dataProfile {kwargs["dataProfile"]} is not a legacy profile.", f"Valid options are {result_profiles_legacy}.", ) @@ -139,7 +139,7 @@ def get_results( if 'dataProfile' in kwargs: if kwargs['dataProfile'] not in result_profiles_wqx3: raise TypeError( - f"dataProfile '{kwargs["dataProfile"]}' is not a valid WQX3.0" + f"dataProfile {kwargs["dataProfile"]} is not a valid WQX3.0" f"profile. Valid options are {result_profiles_wqx3}.", ) else: From 089354ccf8c99e5e6b97daaed32d3787c0f80fbd Mon Sep 17 00:00:00 2001 From: thodson Date: Tue, 17 Sep 2024 09:16:17 -0500 Subject: [PATCH 12/16] Fix mimetype test --- tests/wqp_test.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/wqp_test.py b/tests/wqp_test.py index fd34ae4b..7170ddb0 100755 --- a/tests/wqp_test.py +++ b/tests/wqp_test.py @@ -4,11 +4,18 @@ from pandas import DataFrame -from dataretrieval.wqp import (get_results, what_sites, what_organizations, - what_projects, what_activities, - what_detection_limits, what_habitat_metrics, - what_project_weights, what_activity_metrics, - _alter_kwargs) +from dataretrieval.wqp import ( + get_results, + what_sites, + what_organizations, + what_projects, + what_activities, + what_detection_limits, + what_habitat_metrics, + what_project_weights, + what_activity_metrics, + _check_mimetype, +) def test_get_results(requests_mock): @@ -171,12 +178,11 @@ def mock_request(requests_mock, request_url, file_path): requests_mock.get(request_url, text=text.read(), headers={"mock_header": "value"}) -class TestAlterKwargs: - """Tests for keyword alteration. - """ - def test_alter_kwargs_mimetype(self): - """Tests that mimetype kwarg is altered correctly and warning is thrown.""" - kwargs = {"mimeType": "geojson"} - with pytest.warns(UserWarning): - kwargs = _alter_kwargs(kwargs) - assert kwargs == {"mimeType": "csv"} \ No newline at end of file +def test_check_mimetype(self): + """Tests that correct errors are raised for invalid mimetypes.""" + kwargs = {"mimeType": "geojson"} + with pytest.raises(NotImplementedError): + kwargs = _check_mimetype(kwargs) + kwargs = {"mimeType": "foo"} + with pytest.raises(ValueError): + kwargs = _check_mimetype(kwargs) From 3a8c818cd20aad7d741220e1af9e6ca98dde2adf Mon Sep 17 00:00:00 2001 From: Elise Hinman Date: Tue, 17 Sep 2024 09:35:31 -0500 Subject: [PATCH 13/16] another fix --- dataretrieval/wqp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 383b7546..9fe51ff8 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -129,7 +129,7 @@ def get_results( if "dataProfile" in kwargs: if kwargs["dataProfile"] not in result_profiles_legacy: raise TypeError( - f"dataProfile {kwargs["dataProfile"]} is not a legacy profile.", + f"dataProfile {kwargs['dataProfile']} is not a legacy profile.", f"Valid options are {result_profiles_legacy}.", ) @@ -139,7 +139,7 @@ def get_results( if 'dataProfile' in kwargs: if kwargs['dataProfile'] not in result_profiles_wqx3: raise TypeError( - f"dataProfile {kwargs["dataProfile"]} is not a valid WQX3.0" + f"dataProfile {kwargs['dataProfile']} is not a valid WQX3.0" f"profile. Valid options are {result_profiles_wqx3}.", ) else: From ec3699734123f4a254d2182ac65b4dd75da1ac69 Mon Sep 17 00:00:00 2001 From: thodson Date: Tue, 17 Sep 2024 15:25:28 -0500 Subject: [PATCH 14/16] Fix test --- tests/wqp_test.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/tests/wqp_test.py b/tests/wqp_test.py index 7170ddb0..16ae668f 100755 --- a/tests/wqp_test.py +++ b/tests/wqp_test.py @@ -21,8 +21,7 @@ def test_get_results(requests_mock): """Tests water quality portal results query""" request_url = "https://www.waterqualitydata.us/data/Result/Search?siteid=WIDNR_WQX-10032762" \ - "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" \ - "&mimeType=csv" + "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" response_file_path = 'data/wqp_results.txt' mock_request(requests_mock, request_url, response_file_path) df, md = get_results(siteid='WIDNR_WQX-10032762', @@ -39,7 +38,7 @@ def test_get_results_WQX3(requests_mock): """Tests water quality portal results query with new WQX3.0 profile""" request_url = "https://www.waterqualitydata.us/wqx3/Result/search?siteid=WIDNR_WQX-10032762" \ "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" \ - "&mimeType=csv&dataProfile=fullPhysChem" + "&dataProfile=fullPhysChem" response_file_path = 'data/wqp3_results.txt' mock_request(requests_mock, request_url, response_file_path) df, md = get_results(legacy=False, siteid='WIDNR_WQX-10032762', @@ -55,8 +54,7 @@ def test_get_results_WQX3(requests_mock): def test_what_sites(requests_mock): """Tests Water quality portal sites query""" - request_url = "https://www.waterqualitydata.us/data/Station/Search?statecode=US%3A34&characteristicName=Chloride" \ - "&mimeType=csv" + request_url = "https://www.waterqualitydata.us/data/Station/Search?statecode=US%3A34&characteristicName=Chloride" response_file_path = 'data/wqp_sites.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_sites(statecode="US:34", characteristicName="Chloride") @@ -70,8 +68,7 @@ def test_what_sites(requests_mock): def test_what_organizations(requests_mock): """Tests Water quality portal organizations query""" - request_url = "https://www.waterqualitydata.us/data/Organization/Search?statecode=US%3A34&characteristicName=Chloride" \ - "&mimeType=csv" + request_url = "https://www.waterqualitydata.us/data/Organization/Search?statecode=US%3A34&characteristicName=Chloride" response_file_path = 'data/wqp_organizations.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_organizations(statecode="US:34", characteristicName="Chloride") @@ -85,8 +82,7 @@ def test_what_organizations(requests_mock): def test_what_projects(requests_mock): """Tests Water quality portal projects query""" - request_url = "https://www.waterqualitydata.us/data/Project/Search?statecode=US%3A34&characteristicName=Chloride" \ - "&mimeType=csv" + request_url = "https://www.waterqualitydata.us/data/Project/Search?statecode=US%3A34&characteristicName=Chloride" response_file_path = 'data/wqp_projects.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_projects(statecode="US:34", characteristicName="Chloride") @@ -100,8 +96,7 @@ def test_what_projects(requests_mock): def test_what_activities(requests_mock): """Tests Water quality portal activities query""" - request_url = "https://www.waterqualitydata.us/data/Activity/Search?statecode=US%3A34&characteristicName=Chloride" \ - "&mimeType=csv" + request_url = "https://www.waterqualitydata.us/data/Activity/Search?statecode=US%3A34&characteristicName=Chloride" response_file_path = 'data/wqp_activities.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_activities(statecode="US:34", characteristicName="Chloride") @@ -115,8 +110,7 @@ def test_what_activities(requests_mock): def test_what_detection_limits(requests_mock): """Tests Water quality portal detection limits query""" - request_url = "https://www.waterqualitydata.us/data/ResultDetectionQuantitationLimit/Search?statecode=US%3A34&characteristicName=Chloride" \ - "&mimeType=csv" + request_url = "https://www.waterqualitydata.us/data/ResultDetectionQuantitationLimit/Search?statecode=US%3A34&characteristicName=Chloride" response_file_path = 'data/wqp_detection_limits.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_detection_limits(statecode="US:34", characteristicName="Chloride") @@ -130,8 +124,7 @@ def test_what_detection_limits(requests_mock): def test_what_habitat_metrics(requests_mock): """Tests Water quality portal habitat metrics query""" - request_url = "https://www.waterqualitydata.us/data/BiologicalMetric/Search?statecode=US%3A34&characteristicName=Chloride" \ - "&mimeType=csv" + request_url = "https://www.waterqualitydata.us/data/BiologicalMetric/Search?statecode=US%3A34&characteristicName=Chloride" response_file_path = 'data/wqp_habitat_metrics.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_habitat_metrics(statecode="US:34", characteristicName="Chloride") @@ -145,8 +138,7 @@ def test_what_habitat_metrics(requests_mock): def test_what_project_weights(requests_mock): """Tests Water quality portal project weights query""" - request_url = "https://www.waterqualitydata.us/data/ProjectMonitoringLocationWeighting/Search?statecode=US%3A34&characteristicName=Chloride" \ - "&mimeType=csv" + request_url = "https://www.waterqualitydata.us/data/ProjectMonitoringLocationWeighting/Search?statecode=US%3A34&characteristicName=Chloride" response_file_path = 'data/wqp_project_weights.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_project_weights(statecode="US:34", characteristicName="Chloride") @@ -160,8 +152,7 @@ def test_what_project_weights(requests_mock): def test_what_activity_metrics(requests_mock): """Tests Water quality portal activity metrics query""" - request_url = "https://www.waterqualitydata.us/data/ActivityMetric/Search?statecode=US%3A34&characteristicName=Chloride" \ - "&mimeType=csv" + request_url = "https://www.waterqualitydata.us/data/ActivityMetric/Search?statecode=US%3A34&characteristicName=Chloride" response_file_path = 'data/wqp_activity_metrics.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_activity_metrics(statecode="US:34", characteristicName="Chloride") @@ -178,7 +169,7 @@ def mock_request(requests_mock, request_url, file_path): requests_mock.get(request_url, text=text.read(), headers={"mock_header": "value"}) -def test_check_mimetype(self): +def test_check_mimetype(): """Tests that correct errors are raised for invalid mimetypes.""" kwargs = {"mimeType": "geojson"} with pytest.raises(NotImplementedError): From ef09e3d0829b364763a785208ff0965e5322d93f Mon Sep 17 00:00:00 2001 From: thodson Date: Tue, 17 Sep 2024 16:01:02 -0500 Subject: [PATCH 15/16] Return csv to kwargs --- dataretrieval/wqp.py | 26 ++++++++++++-------- tests/wqp_test.py | 56 +++++++++++++++++++------------------------- 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 9fe51ff8..b006a0ec 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -123,7 +123,7 @@ def get_results( """ - _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) if legacy is True: if "dataProfile" in kwargs: @@ -197,7 +197,7 @@ def what_sites( """ - _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) if legacy is True: url = wqp_url('Station') @@ -251,7 +251,7 @@ def what_organizations( """ - _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) if legacy is True: url = wqp_url('Organization') @@ -302,7 +302,7 @@ def what_projects(ssl_check=True, legacy=True, **kwargs): """ - _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) if legacy is True: url = wqp_url('Project') @@ -368,7 +368,7 @@ def what_activities( ... ) """ - _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) if legacy is True: url = wqp_url("Activity") @@ -429,7 +429,7 @@ def what_detection_limits( """ - _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) if legacy is True: url = wqp_url('ResultDetectionQuantitationLimit') @@ -484,7 +484,7 @@ def what_habitat_metrics( """ - _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) if legacy is True: url = wqp_url('BiologicalMetric') @@ -538,7 +538,7 @@ def what_project_weights(ssl_check=True, legacy=True, **kwargs): """ - _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) if legacy is True: url = wqp_url('ProjectMonitoringLocationWeighting') @@ -592,7 +592,7 @@ def what_activity_metrics(ssl_check=True, legacy=True, **kwargs): """ - _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) if legacy is True: url = wqp_url('ActivityMetric') @@ -687,12 +687,18 @@ def site_info(self): return what_sites(sites=parameters["site_no"]) -def _check_mimetype(kwargs): +def _check_kwargs(kwargs): + """Private function to check kwargs for unsupported parameters. + """ mimetype = kwargs.get("mimeType") if mimetype == "geojson": raise NotImplementedError("GeoJSON not yet supported. Set 'mimeType=csv'.") elif mimetype != "csv" and mimetype is not None: raise ValueError("Invalid mimeType. Set 'mimeType=csv'.") + else: + kwargs["mimeType"] = "csv" + + return kwargs def _warn_wqx3_use(): diff --git a/tests/wqp_test.py b/tests/wqp_test.py index 16ae668f..01fc7f39 100755 --- a/tests/wqp_test.py +++ b/tests/wqp_test.py @@ -14,14 +14,15 @@ what_habitat_metrics, what_project_weights, what_activity_metrics, - _check_mimetype, + _check_kwargs, ) -def test_get_results(requests_mock): - """Tests water quality portal results query""" +def test_get_ratings(requests_mock): + """Tests water quality portal ratings query""" request_url = "https://www.waterqualitydata.us/data/Result/Search?siteid=WIDNR_WQX-10032762" \ - "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" + "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" \ + "&mimeType=csv" response_file_path = 'data/wqp_results.txt' mock_request(requests_mock, request_url, response_file_path) df, md = get_results(siteid='WIDNR_WQX-10032762', @@ -34,27 +35,11 @@ def test_get_results(requests_mock): assert md.header == {"mock_header": "value"} assert md.comment is None -def test_get_results_WQX3(requests_mock): - """Tests water quality portal results query with new WQX3.0 profile""" - request_url = "https://www.waterqualitydata.us/wqx3/Result/search?siteid=WIDNR_WQX-10032762" \ - "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" \ - "&dataProfile=fullPhysChem" - response_file_path = 'data/wqp3_results.txt' - mock_request(requests_mock, request_url, response_file_path) - df, md = get_results(legacy=False, siteid='WIDNR_WQX-10032762', - characteristicName = 'Specific conductance', - startDateLo='05-01-2011', startDateHi='09-30-2011') - assert type(df) is DataFrame - assert df.size == 900 - assert md.url == request_url - assert isinstance(md.query_time, datetime.timedelta) - assert md.header == {"mock_header": "value"} - assert md.comment is None - def test_what_sites(requests_mock): """Tests Water quality portal sites query""" - request_url = "https://www.waterqualitydata.us/data/Station/Search?statecode=US%3A34&characteristicName=Chloride" + request_url = "https://www.waterqualitydata.us/data/Station/Search?statecode=US%3A34&characteristicName=Chloride" \ + "&mimeType=csv" response_file_path = 'data/wqp_sites.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_sites(statecode="US:34", characteristicName="Chloride") @@ -68,7 +53,8 @@ def test_what_sites(requests_mock): def test_what_organizations(requests_mock): """Tests Water quality portal organizations query""" - request_url = "https://www.waterqualitydata.us/data/Organization/Search?statecode=US%3A34&characteristicName=Chloride" + request_url = "https://www.waterqualitydata.us/data/Organization/Search?statecode=US%3A34&characteristicName=Chloride" \ + "&mimeType=csv" response_file_path = 'data/wqp_organizations.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_organizations(statecode="US:34", characteristicName="Chloride") @@ -82,7 +68,8 @@ def test_what_organizations(requests_mock): def test_what_projects(requests_mock): """Tests Water quality portal projects query""" - request_url = "https://www.waterqualitydata.us/data/Project/Search?statecode=US%3A34&characteristicName=Chloride" + request_url = "https://www.waterqualitydata.us/data/Project/Search?statecode=US%3A34&characteristicName=Chloride" \ + "&mimeType=csv" response_file_path = 'data/wqp_projects.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_projects(statecode="US:34", characteristicName="Chloride") @@ -96,7 +83,8 @@ def test_what_projects(requests_mock): def test_what_activities(requests_mock): """Tests Water quality portal activities query""" - request_url = "https://www.waterqualitydata.us/data/Activity/Search?statecode=US%3A34&characteristicName=Chloride" + request_url = "https://www.waterqualitydata.us/data/Activity/Search?statecode=US%3A34&characteristicName=Chloride" \ + "&mimeType=csv" response_file_path = 'data/wqp_activities.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_activities(statecode="US:34", characteristicName="Chloride") @@ -110,7 +98,8 @@ def test_what_activities(requests_mock): def test_what_detection_limits(requests_mock): """Tests Water quality portal detection limits query""" - request_url = "https://www.waterqualitydata.us/data/ResultDetectionQuantitationLimit/Search?statecode=US%3A34&characteristicName=Chloride" + request_url = "https://www.waterqualitydata.us/data/ResultDetectionQuantitationLimit/Search?statecode=US%3A34&characteristicName=Chloride" \ + "&mimeType=csv" response_file_path = 'data/wqp_detection_limits.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_detection_limits(statecode="US:34", characteristicName="Chloride") @@ -124,7 +113,8 @@ def test_what_detection_limits(requests_mock): def test_what_habitat_metrics(requests_mock): """Tests Water quality portal habitat metrics query""" - request_url = "https://www.waterqualitydata.us/data/BiologicalMetric/Search?statecode=US%3A34&characteristicName=Chloride" + request_url = "https://www.waterqualitydata.us/data/BiologicalMetric/Search?statecode=US%3A34&characteristicName=Chloride" \ + "&mimeType=csv" response_file_path = 'data/wqp_habitat_metrics.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_habitat_metrics(statecode="US:34", characteristicName="Chloride") @@ -138,7 +128,8 @@ def test_what_habitat_metrics(requests_mock): def test_what_project_weights(requests_mock): """Tests Water quality portal project weights query""" - request_url = "https://www.waterqualitydata.us/data/ProjectMonitoringLocationWeighting/Search?statecode=US%3A34&characteristicName=Chloride" + request_url = "https://www.waterqualitydata.us/data/ProjectMonitoringLocationWeighting/Search?statecode=US%3A34&characteristicName=Chloride" \ + "&mimeType=csv" response_file_path = 'data/wqp_project_weights.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_project_weights(statecode="US:34", characteristicName="Chloride") @@ -152,7 +143,8 @@ def test_what_project_weights(requests_mock): def test_what_activity_metrics(requests_mock): """Tests Water quality portal activity metrics query""" - request_url = "https://www.waterqualitydata.us/data/ActivityMetric/Search?statecode=US%3A34&characteristicName=Chloride" + request_url = "https://www.waterqualitydata.us/data/ActivityMetric/Search?statecode=US%3A34&characteristicName=Chloride" \ + "&mimeType=csv" response_file_path = 'data/wqp_activity_metrics.txt' mock_request(requests_mock, request_url, response_file_path) df, md = what_activity_metrics(statecode="US:34", characteristicName="Chloride") @@ -169,11 +161,11 @@ def mock_request(requests_mock, request_url, file_path): requests_mock.get(request_url, text=text.read(), headers={"mock_header": "value"}) -def test_check_mimetype(): +def test_check_kwargs(): """Tests that correct errors are raised for invalid mimetypes.""" kwargs = {"mimeType": "geojson"} with pytest.raises(NotImplementedError): - kwargs = _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) kwargs = {"mimeType": "foo"} with pytest.raises(ValueError): - kwargs = _check_mimetype(kwargs) + kwargs = _check_kwargs(kwargs) From 40a2b46eb9619b17b0bb9c8c47af7acee9adbec8 Mon Sep 17 00:00:00 2001 From: thodson Date: Tue, 17 Sep 2024 16:22:16 -0500 Subject: [PATCH 16/16] Return wqx3 test --- tests/wqp_test.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/wqp_test.py b/tests/wqp_test.py index 01fc7f39..affb0966 100755 --- a/tests/wqp_test.py +++ b/tests/wqp_test.py @@ -18,7 +18,7 @@ ) -def test_get_ratings(requests_mock): +def test_get_results(requests_mock): """Tests water quality portal ratings query""" request_url = "https://www.waterqualitydata.us/data/Result/Search?siteid=WIDNR_WQX-10032762" \ "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" \ @@ -36,6 +36,25 @@ def test_get_ratings(requests_mock): assert md.comment is None +def test_get_results_WQX3(requests_mock): + """Tests water quality portal results query with new WQX3.0 profile""" + request_url = "https://www.waterqualitydata.us/wqx3/Result/search?siteid=WIDNR_WQX-10032762" \ + "&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011" \ + "&mimeType=csv" \ + "&dataProfile=fullPhysChem" + response_file_path = 'data/wqp3_results.txt' + mock_request(requests_mock, request_url, response_file_path) + df, md = get_results(legacy=False, siteid='WIDNR_WQX-10032762', + characteristicName = 'Specific conductance', + startDateLo='05-01-2011', startDateHi='09-30-2011') + assert type(df) is DataFrame + assert df.size == 900 + assert md.url == request_url + assert isinstance(md.query_time, datetime.timedelta) + assert md.header == {"mock_header": "value"} + assert md.comment is None + + def test_what_sites(requests_mock): """Tests Water quality portal sites query""" request_url = "https://www.waterqualitydata.us/data/Station/Search?statecode=US%3A34&characteristicName=Chloride" \