From 494865c361a6480bd10b2975cd6fe7500991ebb0 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Wed, 28 Jun 2023 12:22:42 -0700 Subject: [PATCH 1/7] style(black): format covidcast_rows and logger --- src/common/covidcast_row.py | 22 ++++++++++++++++------ src/common/logger.py | 17 ++++++----------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/common/covidcast_row.py b/src/common/covidcast_row.py index 23e19eb57..253201db4 100644 --- a/src/common/covidcast_row.py +++ b/src/common/covidcast_row.py @@ -25,6 +25,7 @@ "value_updated_timestamp": "Int64", } + @dataclass class CovidcastRow: """A container for the values of a single covidcast database row. @@ -62,7 +63,11 @@ class CovidcastRow: # Classvars. _db_row_ignore_fields: ClassVar = [] _api_row_ignore_fields: ClassVar = ["epimetric_id", "value_updated_timestamp"] - _api_row_compatibility_ignore_fields: ClassVar = _api_row_ignore_fields + ["source", "time_type", "geo_type"] + _api_row_compatibility_ignore_fields: ClassVar = _api_row_ignore_fields + [ + "source", + "time_type", + "geo_type", + ] _pandas_dtypes: ClassVar = PANDAS_DTYPES @@ -72,14 +77,16 @@ def as_dict(self, ignore_fields: Optional[List[str]] = None) -> dict: for key in ignore_fields: del d[key] return d - + def as_api_row_dict(self, ignore_fields: Optional[List[str]] = None) -> dict: """Returns a dict view into the row with the fields returned by the API server.""" return self.as_dict(ignore_fields=self._api_row_ignore_fields + (ignore_fields or [])) def as_api_compatibility_row_dict(self, ignore_fields: Optional[List[str]] = None) -> dict: """Returns a dict view into the row with the fields returned by the old API server (the PHP server).""" - return self.as_dict(ignore_fields=self._api_row_compatibility_ignore_fields + (ignore_fields or [])) + return self.as_dict( + ignore_fields=self._api_row_compatibility_ignore_fields + (ignore_fields or []) + ) def as_db_row_dict(self, ignore_fields: Optional[List[str]] = None) -> dict: """Returns a dict view into the row with the fields returned by the database.""" @@ -95,9 +102,13 @@ def as_api_row_df(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFra """Returns a dataframe view into the row with the fields returned by the API server.""" return self.as_dataframe(ignore_fields=self._api_row_ignore_fields + (ignore_fields or [])) - def as_api_compatibility_row_df(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFrame: + def as_api_compatibility_row_df( + self, ignore_fields: Optional[List[str]] = None + ) -> pd.DataFrame: """Returns a dataframe view into the row with the fields returned by the old API server (the PHP server).""" - return self.as_dataframe(ignore_fields=self._api_row_compatibility_ignore_fields + (ignore_fields or [])) + return self.as_dataframe( + ignore_fields=self._api_row_compatibility_ignore_fields + (ignore_fields or []) + ) def as_db_row_df(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFrame: """Returns a dataframe view into the row with the fields returned by an all-field database query.""" @@ -113,7 +124,6 @@ def time_pair(self): return f"{self.time_type}:{self.time_value}" - def check_valid_dtype(dtype): try: pd.api.types.pandas_dtype(dtype) diff --git a/src/common/logger.py b/src/common/logger.py index 4d7d7d257..909e47fb7 100644 --- a/src/common/logger.py +++ b/src/common/logger.py @@ -4,11 +4,12 @@ import threading import structlog + def handle_exceptions(logger): """Handle exceptions using the provided logger.""" + def exception_handler(etype, value, traceback): - logger.exception("Top-level exception occurred", - exc_info=(etype, value, traceback)) + logger.exception("Top-level exception occurred", exc_info=(etype, value, traceback)) def multithread_exception_handler(args): exception_handler(args.exc_type, args.exc_value, args.exc_traceback) @@ -17,9 +18,7 @@ def multithread_exception_handler(args): threading.excepthook = multithread_exception_handler -def get_structured_logger(name=__name__, - filename=None, - log_exceptions=True): +def get_structured_logger(name=__name__, filename=None, log_exceptions=True): """Create a new structlog logger. Use the logger returned from this in indicator code using the standard @@ -49,11 +48,7 @@ def get_structured_logger(name=__name__, else: log_level = logging.INFO - logging.basicConfig( - format="%(message)s", - level=log_level, - handlers=handlers - ) + logging.basicConfig(format="%(message)s", level=log_level, handlers=handlers) def add_pid(logger, method_name, event_dict): """ @@ -85,7 +80,7 @@ def add_pid(logger, method_name, event_dict): # Decode unicode characters structlog.processors.UnicodeDecoder(), # Render as JSON - structlog.processors.JSONRenderer() + structlog.processors.JSONRenderer(), ], # Use a dict class for keeping track of data. context_class=dict, From a1a1434d8fffbb340768613053ecbe64ef0798ac Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Wed, 28 Jun 2023 12:24:23 -0700 Subject: [PATCH 2/7] style(tests): format test covidcast_nowcast with black --- .../covidcast_nowcast/test_load_sensors.py | 64 ++++++++++--------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/tests/acquisition/covidcast_nowcast/test_load_sensors.py b/tests/acquisition/covidcast_nowcast/test_load_sensors.py index 9b0c5181a..efa21bcc7 100644 --- a/tests/acquisition/covidcast_nowcast/test_load_sensors.py +++ b/tests/acquisition/covidcast_nowcast/test_load_sensors.py @@ -13,35 +13,41 @@ from delphi.epidata.acquisition.covidcast_nowcast.load_sensors import main, load_and_prepare_file # py3tester coverage target -__test_target__ = 'delphi.epidata.acquisition.covidcast_nowcast.load_sensors' +__test_target__ = "delphi.epidata.acquisition.covidcast_nowcast.load_sensors" class UpdateTests(unittest.TestCase): - - @mock.patch('time.time', mock.MagicMock(return_value=12345)) - def test_load_and_prepare_file(self): - - test_attributes = PathDetails( - 20210102, - 3, - "test_source", - "test_signal", - "test_time_type", - 20201231, - "test_geo_type", - ) - - test_df = load_and_prepare_file(StringIO("sensor_name,geo_value,value\ntestname,01001,1.5"), test_attributes) - pd.testing.assert_frame_equal(test_df, - pd.DataFrame({"sensor_name": ["testname"], - "geo_value": ["01001"], - "value": [1.5], - "source": ["test_source"], - "signal": ["test_signal"], - "time_type": ["test_time_type"], - "geo_type": ["test_geo_type"], - "time_value": [20201231], - "issue": [20210102], - "lag": [3], - "value_updated_timestamp": [12345]}) - ) + @mock.patch("time.time", mock.MagicMock(return_value=12345)) + def test_load_and_prepare_file(self): + + test_attributes = PathDetails( + 20210102, + 3, + "test_source", + "test_signal", + "test_time_type", + 20201231, + "test_geo_type", + ) + + test_df = load_and_prepare_file( + StringIO("sensor_name,geo_value,value\ntestname,01001,1.5"), test_attributes + ) + pd.testing.assert_frame_equal( + test_df, + pd.DataFrame( + { + "sensor_name": ["testname"], + "geo_value": ["01001"], + "value": [1.5], + "source": ["test_source"], + "signal": ["test_signal"], + "time_type": ["test_time_type"], + "geo_type": ["test_geo_type"], + "time_value": [20201231], + "issue": [20210102], + "lag": [3], + "value_updated_timestamp": [12345], + } + ), + ) From 8b808a9a12c8c90ada3f9b80e7f6aaab42b29df6 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Wed, 28 Jun 2023 12:24:47 -0700 Subject: [PATCH 3/7] style(tests): format test flusurv with black --- tests/acquisition/flusurv/test_flusurv.py | 28 +++++++++---------- .../flusurv/test_flusurv_update.py | 10 +++---- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/acquisition/flusurv/test_flusurv.py b/tests/acquisition/flusurv/test_flusurv.py index f75efaabc..e0c5acaad 100644 --- a/tests/acquisition/flusurv/test_flusurv.py +++ b/tests/acquisition/flusurv/test_flusurv.py @@ -8,26 +8,26 @@ from delphi.epidata.acquisition.flusurv.flusurv import fetch_json # py3tester coverage target -__test_target__ = 'delphi.epidata.acquisition.flusurv.flusurv' +__test_target__ = "delphi.epidata.acquisition.flusurv.flusurv" class FunctionTests(unittest.TestCase): - """Tests each function individually.""" + """Tests each function individually.""" - def test_fetch_json(self): - """Run through a successful flow.""" + def test_fetch_json(self): + """Run through a successful flow.""" - path = 'path' - payload = None + path = "path" + payload = None - response_object = MagicMock() - response_object.status_code = 200 - response_object.headers = {'Content-Type': 'application/json'} - response_object.json.return_value = sentinel.expected + response_object = MagicMock() + response_object.status_code = 200 + response_object.headers = {"Content-Type": "application/json"} + response_object.json.return_value = sentinel.expected - requests_impl = MagicMock() - requests_impl.get.return_value = response_object + requests_impl = MagicMock() + requests_impl.get.return_value = response_object - actual = fetch_json(path, payload, requests_impl=requests_impl) + actual = fetch_json(path, payload, requests_impl=requests_impl) - self.assertEqual(actual, sentinel.expected) + self.assertEqual(actual, sentinel.expected) diff --git a/tests/acquisition/flusurv/test_flusurv_update.py b/tests/acquisition/flusurv/test_flusurv_update.py index 1908bcf69..66500d210 100644 --- a/tests/acquisition/flusurv/test_flusurv_update.py +++ b/tests/acquisition/flusurv/test_flusurv_update.py @@ -4,12 +4,12 @@ import unittest # py3tester coverage target -__test_target__ = 'delphi.epidata.acquisition.flusurv.flusurv_update' +__test_target__ = "delphi.epidata.acquisition.flusurv.flusurv_update" class FunctionTests(unittest.TestCase): - """Tests each function individually.""" + """Tests each function individually.""" - def test_syntax(self): - """This no-op test ensures that syntax is valid.""" - pass + def test_syntax(self): + """This no-op test ensures that syntax is valid.""" + pass From e7beb32b567fa258b34bf35960d8b755c0ee90b3 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Wed, 28 Jun 2023 12:25:03 -0700 Subject: [PATCH 4/7] style(tests): format test fluview with black --- .../fluview/test_impute_missing_values.py | 238 +++++++++--------- 1 file changed, 121 insertions(+), 117 deletions(-) diff --git a/tests/acquisition/fluview/test_impute_missing_values.py b/tests/acquisition/fluview/test_impute_missing_values.py index 37aab2555..96338d467 100644 --- a/tests/acquisition/fluview/test_impute_missing_values.py +++ b/tests/acquisition/fluview/test_impute_missing_values.py @@ -7,125 +7,129 @@ # first party from delphi.utils.geo.locations import Locations -from delphi.epidata.acquisition.fluview.impute_missing_values import get_argument_parser, \ - get_lag_and_ili, impute_missing_values, StatespaceException +from delphi.epidata.acquisition.fluview.impute_missing_values import ( + get_argument_parser, + get_lag_and_ili, + impute_missing_values, + StatespaceException, +) # py3tester coverage target -__test_target__ = 'delphi.epidata.acquisition.fluview.impute_missing_values' +__test_target__ = "delphi.epidata.acquisition.fluview.impute_missing_values" class FunctionTests(unittest.TestCase): - """Tests each function individually.""" - - def test_get_argument_parser(self): - """An ArgumentParser is returned.""" - self.assertIsInstance(get_argument_parser(), argparse.ArgumentParser) - - def test_get_lag_and_ili(self): - samples = ( - ((201740, 201730, 5, 10), (10, 50)), - ((201740, 201740, 1, 1), (0, 100)), - ((201501, 201452, 0, 0), (2, 0)), - ) - - for args, expected in samples: - with self.subTest(args=args): - actual = get_lag_and_ili(*args) - self.assertEqual(actual, expected) - - def test_impute_missing_values(self): - """Atoms are imputed and stored.""" - - unknown_set = set(['pa', 'tx']) - known_set = set(['nat', 'hhs6'] + Locations.atom_list) - unknown_set - known_data = {} - for loc in known_set: - n = len(Locations.region_map[loc]) - known_data[loc] = (n, n, n) - - db = MagicMock() - db.count_rows.return_value = 123 - db.find_missing_rows.return_value = [(201740, 201740)] - db.get_known_values.return_value = known_data - - impute_missing_values(db, test_mode=True) - - self.assertEqual(db.connect.call_count, 1) - self.assertTrue(db.count_rows.call_count >= 1) - self.assertTrue(db.find_missing_rows.call_count >= 1) - self.assertEqual(db.add_imputed_values.call_count, 1) - self.assertEqual(db.close.call_count, 1) - self.assertFalse(db.close.call_args[0][0]) - - imputed = db.add_imputed_values.call_args[0][-1] - self.assertTrue(unknown_set <= set(imputed.keys())) - for loc, (lag, n_ili, n_pat, n_prov, ili) in imputed.items(): - with self.subTest(loc=loc): - num = len(Locations.region_map[loc]) - self.assertEqual(lag, 0) - self.assertEqual(n_ili, num) - self.assertEqual(n_pat, num) - self.assertEqual(n_prov, num) - self.assertEqual(ili, 100) - - def test_impute_missing_values_vipr(self): - """PR and VI are imputed only when appropriate.""" - - unknown_set = set(['vi', 'pr']) - known_set = set(['nat'] + Locations.atom_list) - unknown_set - known_data = {} - for loc in known_set: - n = len(Locations.region_map[loc]) - known_data[loc] = (n, n, n) - - db = MagicMock() - db.count_rows.return_value = 123 - db.get_known_values.return_value = known_data - - db.find_missing_rows.return_value = [(201340, 201340)] - with self.assertRaises(Exception): - impute_missing_values(db, test_mode=True) - - db.find_missing_rows.return_value = [(201339, 201339)] - impute_missing_values(db, test_mode=True) - - imputed = db.add_imputed_values.call_args[0][-1] - self.assertIn('pr', set(imputed.keys())) - - def test_impute_missing_values_regions(self): - """Regions are imputed in addition to atoms.""" - - known_set = set(Locations.atom_list) - known_data = {} - for loc in known_set: - known_data[loc] = (1, 2, 3) - - db = MagicMock() - db.count_rows.return_value = 123 - db.find_missing_rows.return_value = [(201740, 201740)] - db.get_known_values.return_value = known_data - - impute_missing_values(db, test_mode=True) - imputed = db.add_imputed_values.call_args[0][-1] - self.assertIn('nat', set(imputed.keys())) - self.assertIn('hhs2', set(imputed.keys())) - self.assertIn('cen3', set(imputed.keys())) - self.assertIn('ny', set(imputed.keys())) - - def test_impute_missing_values_underdetermined(self): - """Fail when the system is underdetermined.""" - - unknown_set = set(['pa', 'tx']) - known_set = set(Locations.atom_list) - unknown_set - known_data = {} - for loc in known_set: - n = len(Locations.region_map[loc]) - known_data[loc] = (n, n, n) - - db = MagicMock() - db.count_rows.return_value = 123 - db.find_missing_rows.return_value = [(201740, 201740)] - db.get_known_values.return_value = known_data - - with self.assertRaises(StatespaceException): - impute_missing_values(db, test_mode=True) + """Tests each function individually.""" + + def test_get_argument_parser(self): + """An ArgumentParser is returned.""" + self.assertIsInstance(get_argument_parser(), argparse.ArgumentParser) + + def test_get_lag_and_ili(self): + samples = ( + ((201740, 201730, 5, 10), (10, 50)), + ((201740, 201740, 1, 1), (0, 100)), + ((201501, 201452, 0, 0), (2, 0)), + ) + + for args, expected in samples: + with self.subTest(args=args): + actual = get_lag_and_ili(*args) + self.assertEqual(actual, expected) + + def test_impute_missing_values(self): + """Atoms are imputed and stored.""" + + unknown_set = set(["pa", "tx"]) + known_set = set(["nat", "hhs6"] + Locations.atom_list) - unknown_set + known_data = {} + for loc in known_set: + n = len(Locations.region_map[loc]) + known_data[loc] = (n, n, n) + + db = MagicMock() + db.count_rows.return_value = 123 + db.find_missing_rows.return_value = [(201740, 201740)] + db.get_known_values.return_value = known_data + + impute_missing_values(db, test_mode=True) + + self.assertEqual(db.connect.call_count, 1) + self.assertTrue(db.count_rows.call_count >= 1) + self.assertTrue(db.find_missing_rows.call_count >= 1) + self.assertEqual(db.add_imputed_values.call_count, 1) + self.assertEqual(db.close.call_count, 1) + self.assertFalse(db.close.call_args[0][0]) + + imputed = db.add_imputed_values.call_args[0][-1] + self.assertTrue(unknown_set <= set(imputed.keys())) + for loc, (lag, n_ili, n_pat, n_prov, ili) in imputed.items(): + with self.subTest(loc=loc): + num = len(Locations.region_map[loc]) + self.assertEqual(lag, 0) + self.assertEqual(n_ili, num) + self.assertEqual(n_pat, num) + self.assertEqual(n_prov, num) + self.assertEqual(ili, 100) + + def test_impute_missing_values_vipr(self): + """PR and VI are imputed only when appropriate.""" + + unknown_set = set(["vi", "pr"]) + known_set = set(["nat"] + Locations.atom_list) - unknown_set + known_data = {} + for loc in known_set: + n = len(Locations.region_map[loc]) + known_data[loc] = (n, n, n) + + db = MagicMock() + db.count_rows.return_value = 123 + db.get_known_values.return_value = known_data + + db.find_missing_rows.return_value = [(201340, 201340)] + with self.assertRaises(Exception): + impute_missing_values(db, test_mode=True) + + db.find_missing_rows.return_value = [(201339, 201339)] + impute_missing_values(db, test_mode=True) + + imputed = db.add_imputed_values.call_args[0][-1] + self.assertIn("pr", set(imputed.keys())) + + def test_impute_missing_values_regions(self): + """Regions are imputed in addition to atoms.""" + + known_set = set(Locations.atom_list) + known_data = {} + for loc in known_set: + known_data[loc] = (1, 2, 3) + + db = MagicMock() + db.count_rows.return_value = 123 + db.find_missing_rows.return_value = [(201740, 201740)] + db.get_known_values.return_value = known_data + + impute_missing_values(db, test_mode=True) + imputed = db.add_imputed_values.call_args[0][-1] + self.assertIn("nat", set(imputed.keys())) + self.assertIn("hhs2", set(imputed.keys())) + self.assertIn("cen3", set(imputed.keys())) + self.assertIn("ny", set(imputed.keys())) + + def test_impute_missing_values_underdetermined(self): + """Fail when the system is underdetermined.""" + + unknown_set = set(["pa", "tx"]) + known_set = set(Locations.atom_list) - unknown_set + known_data = {} + for loc in known_set: + n = len(Locations.region_map[loc]) + known_data[loc] = (n, n, n) + + db = MagicMock() + db.count_rows.return_value = 123 + db.find_missing_rows.return_value = [(201740, 201740)] + db.get_known_values.return_value = known_data + + with self.assertRaises(StatespaceException): + impute_missing_values(db, test_mode=True) From fb41ff22b90de755a09a300bbdb80552f564a78b Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Wed, 28 Jun 2023 12:25:15 -0700 Subject: [PATCH 5/7] style(tests): format test py client with black --- tests/client/test_delphi_epidata.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/client/test_delphi_epidata.py b/tests/client/test_delphi_epidata.py index ad60f3461..1c1a2e386 100644 --- a/tests/client/test_delphi_epidata.py +++ b/tests/client/test_delphi_epidata.py @@ -4,13 +4,13 @@ import unittest # py3tester coverage target -__test_target__ = 'delphi.epidata.client.delphi_epidata' +__test_target__ = "delphi.epidata.client.delphi_epidata" class UnitTests(unittest.TestCase): - """Basic unit tests.""" + """Basic unit tests.""" - # TODO: Unit tests still need to be written. This no-op test will pass unless - # the target file can't be loaded. In effect, it's a syntax checker. - def test_syntax(self): - pass + # TODO: Unit tests still need to be written. This no-op test will pass unless + # the target file can't be loaded. In effect, it's a syntax checker. + def test_syntax(self): + pass From 7114b2c85b80b1a3ec840f397b27e4cbb6f7e60a Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Wed, 28 Jun 2023 12:25:32 -0700 Subject: [PATCH 6/7] style(tests): format test covidcast_row with black --- tests/common/test_covidcast_row.py | 69 ++++++++++++++---------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/tests/common/test_covidcast_row.py b/tests/common/test_covidcast_row.py index 834a7852d..ab957250d 100644 --- a/tests/common/test_covidcast_row.py +++ b/tests/common/test_covidcast_row.py @@ -7,49 +7,47 @@ from delphi.epidata.common.covidcast_row import CovidcastRow, set_df_dtypes from delphi.epidata.acquisition.covidcast.test_utils import ( CovidcastTestRow, - covidcast_rows_as_api_compatibility_row_df, - covidcast_rows_as_api_row_df, - covidcast_rows_from_args, - transpose_dict, + covidcast_rows_as_api_compatibility_row_df, + covidcast_rows_as_api_row_df, + covidcast_rows_from_args, + transpose_dict, ) from delphi.epidata.acquisition.covidcast.test_utils import MSA # py3tester coverage target (equivalent to `import *`) -__test_target__ = 'delphi.epidata.common.covidcast_row' +__test_target__ = "delphi.epidata.common.covidcast_row" class TestCovidcastRows(unittest.TestCase): - expected_df = set_df_dtypes(DataFrame({ - "source": ["src"] * 10, - "signal": ["sig_base"] * 5 + ["sig_other"] * 5, - "time_type": ["day"] * 10, - "geo_type": ["msa"] * 10, - "time_value": [2021_05_01 + i for i in range(5)] * 2, - "geo_value": [MSA[0]] * 10, - "value": range(10), - "stderr": [10.0] * 10, - "sample_size": [10.0] * 10, - "missing_value": [Nans.NOT_MISSING] * 10, - "missing_stderr": [Nans.NOT_MISSING] * 10, - "missing_sample_size": [Nans.NOT_MISSING] * 10, - "issue": [2021_05_01 + i for i in range(5)] * 2, - "lag": [0] * 10, - "direction": [None] * 10 - }), CovidcastRow._pandas_dtypes) - - def test_transpose_dict(self): - assert transpose_dict( + expected_df = set_df_dtypes( + DataFrame( { - "a": [2, 4, 6], - "b": [3, 5, 7], - "c": [10, 20, 30] + "source": ["src"] * 10, + "signal": ["sig_base"] * 5 + ["sig_other"] * 5, + "time_type": ["day"] * 10, + "geo_type": ["msa"] * 10, + "time_value": [2021_05_01 + i for i in range(5)] * 2, + "geo_value": [MSA[0]] * 10, + "value": range(10), + "stderr": [10.0] * 10, + "sample_size": [10.0] * 10, + "missing_value": [Nans.NOT_MISSING] * 10, + "missing_stderr": [Nans.NOT_MISSING] * 10, + "missing_sample_size": [Nans.NOT_MISSING] * 10, + "issue": [2021_05_01 + i for i in range(5)] * 2, + "lag": [0] * 10, + "direction": [None] * 10, } - ) == [ - {"a": 2, "b": 3, "c": 10}, - {"a": 4, "b": 5, "c": 20}, - {"a": 6, "b": 7, "c": 30} - ] + ), + CovidcastRow._pandas_dtypes, + ) + def test_transpose_dict(self): + assert transpose_dict({"a": [2, 4, 6], "b": [3, 5, 7], "c": [10, 20, 30]}) == [ + {"a": 2, "b": 3, "c": 10}, + {"a": 4, "b": 5, "c": 20}, + {"a": 6, "b": 7, "c": 30}, + ] def test_CovidcastRow(self): df = CovidcastTestRow.make_default_row( @@ -70,13 +68,12 @@ def test_CovidcastRow(self): expected_df = self.expected_df.iloc[0:1][df.columns] assert_frame_equal(df, expected_df) - def test_covidcast_rows(self): covidcast_rows = covidcast_rows_from_args( signal=["sig_base"] * 5 + ["sig_other"] * 5, time_value=[2021_05_01 + i for i in range(5)] * 2, value=list(range(10)), - sanitize_fields = True + sanitize_fields=True, ) df = covidcast_rows_as_api_row_df(covidcast_rows) expected_df = self.expected_df @@ -86,7 +83,7 @@ def test_covidcast_rows(self): signal=["sig_base"] * 5 + ["sig_other"] * 5, time_value=[2021_05_01 + i for i in range(5)] * 2, value=list(range(10)), - sanitize_fields = True + sanitize_fields=True, ) df = covidcast_rows_as_api_compatibility_row_df(covidcast_rows) expected_df = self.expected_df[df.columns] From 36123e71444d403a51365cf3eafeb6586e56c799 Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Mon, 10 Jul 2023 12:32:40 -0700 Subject: [PATCH 7/7] style: revert one format change and rewrap a few long comments --- src/common/covidcast_row.py | 30 +++++++++++++++++++----------- tests/common/test_covidcast_row.py | 8 +++++++- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/common/covidcast_row.py b/src/common/covidcast_row.py index 253201db4..fc81d3a6c 100644 --- a/src/common/covidcast_row.py +++ b/src/common/covidcast_row.py @@ -36,7 +36,8 @@ class CovidcastRow: - converting from and to formats (dict, csv, df, kwargs) - creating consistent views, with consistent data types (dict, csv, df) - The rows are specified in 'v4_schema.sql'. The datatypes are made to match database. When writing to Pandas, the dtypes match the JIT model.py schema. + The rows are specified in 'v4_schema.sql'. The datatypes are made to match + database. When writing to Pandas, the dtypes match the JIT model.py schema. """ # Arguments. @@ -55,7 +56,8 @@ class CovidcastRow: missing_sample_size: int issue: int lag: int - # The following three fields are only the database, but are not ingested at acquisition and not returned by the API. + # The following three fields are only in the database, but are not ingested at + # acquisition and not returned by the API. epimetric_id: Optional[int] = None direction: Optional[int] = None value_updated_timestamp: Optional[int] = 0 @@ -79,17 +81,20 @@ def as_dict(self, ignore_fields: Optional[List[str]] = None) -> dict: return d def as_api_row_dict(self, ignore_fields: Optional[List[str]] = None) -> dict: - """Returns a dict view into the row with the fields returned by the API server.""" + """Returns a dict view into the row with the fields returned by the API + server.""" return self.as_dict(ignore_fields=self._api_row_ignore_fields + (ignore_fields or [])) def as_api_compatibility_row_dict(self, ignore_fields: Optional[List[str]] = None) -> dict: - """Returns a dict view into the row with the fields returned by the old API server (the PHP server).""" + """Returns a dict view into the row with the fields returned by the old + API server (the PHP server).""" return self.as_dict( ignore_fields=self._api_row_compatibility_ignore_fields + (ignore_fields or []) ) def as_db_row_dict(self, ignore_fields: Optional[List[str]] = None) -> dict: - """Returns a dict view into the row with the fields returned by the database.""" + """Returns a dict view into the row with the fields returned by the + database.""" return self.as_dict(ignore_fields=self._db_row_ignore_fields + (ignore_fields or [])) def as_dataframe(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFrame: @@ -99,19 +104,22 @@ def as_dataframe(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFram return df def as_api_row_df(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFrame: - """Returns a dataframe view into the row with the fields returned by the API server.""" + """Returns a dataframe view into the row with the fields returned by the + API server.""" return self.as_dataframe(ignore_fields=self._api_row_ignore_fields + (ignore_fields or [])) - def as_api_compatibility_row_df( - self, ignore_fields: Optional[List[str]] = None - ) -> pd.DataFrame: - """Returns a dataframe view into the row with the fields returned by the old API server (the PHP server).""" + # fmt: off + def as_api_compatibility_row_df(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFrame: + """Returns a dataframe view into the row with the fields returned by the + old API server (the PHP server).""" + # fmt: on return self.as_dataframe( ignore_fields=self._api_row_compatibility_ignore_fields + (ignore_fields or []) ) def as_db_row_df(self, ignore_fields: Optional[List[str]] = None) -> pd.DataFrame: - """Returns a dataframe view into the row with the fields returned by an all-field database query.""" + """Returns a dataframe view into the row with the fields returned by an + all-field database query.""" return self.as_dataframe(ignore_fields=self._db_row_ignore_fields + (ignore_fields or [])) def signal_pair(self): diff --git a/tests/common/test_covidcast_row.py b/tests/common/test_covidcast_row.py index ab957250d..273596ebb 100644 --- a/tests/common/test_covidcast_row.py +++ b/tests/common/test_covidcast_row.py @@ -43,11 +43,17 @@ class TestCovidcastRows(unittest.TestCase): ) def test_transpose_dict(self): - assert transpose_dict({"a": [2, 4, 6], "b": [3, 5, 7], "c": [10, 20, 30]}) == [ + # fmt: off + assert transpose_dict({ + "a": [2, 4, 6], + "b": [3, 5, 7], + "c": [10, 20, 30] + }) == [ {"a": 2, "b": 3, "c": 10}, {"a": 4, "b": 5, "c": 20}, {"a": 6, "b": 7, "c": 30}, ] + # fmt: on def test_CovidcastRow(self): df = CovidcastTestRow.make_default_row(