Merge pull request #1015 from OSOceanAcoustics/dev

Release/v0.7.0
OSOceanAcoustics · Mar 25, 2023 · 3320bd1 · 3320bd1
2 parents d021796 + 2736ee9
commit 3320bd1
Show file tree

Hide file tree

Showing 51 changed files with 3,402 additions and 970 deletions.
diff --git a/.ci_helpers/run-test.py b/.ci_helpers/run-test.py
@@ -25,15 +25,17 @@
 
 MODULES_TO_TEST = {
     "root": {},  # This is to test the root folder.
-    "convert": {},
     "calibrate": {},
+    "clean": {},
+    "commongrid": {},
+    "consolidate": {},
+    "convert": {},
     "echodata": {},
+    "mask": {},
+    "metrics": {},
     "preprocess": {},
     "utils": {},
     "visualize": {},
-    "metrics": {},
-    "mask": {},
-    "consolidate": {},
 }
 
 if __name__ == "__main__":

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -39,4 +39,4 @@ repos:
       - id: codespell
         # Checks spelling in `docs/source` and `echopype` dirs ONLY
         # Ignores `.ipynb` files and `_build` folders
-        args: ["--skip=*.ipynb,docs/source/_build", "-w", "docs/source", "echopype"]
+        args: ["--skip=*.ipynb,docs/source/_build,echopype/test_data", "-w", "docs/source", "echopype"]
diff --git a/docs/source/_toc.yml b/docs/source/_toc.yml
@@ -20,6 +20,8 @@ parts:
     - file: data-format-processed
     - file: data-format-5to6
   - file: viz
+  - file: processing-levels
+    title: Processing levels
 - caption: Help & reference
   chapters:
   - file: api

diff --git a/docs/source/processing-levels.md b/docs/source/processing-levels.md
@@ -0,0 +1,63 @@
+# Proposed Echosounder Data Processing Levels (DRAFT)
+
+The decades-long experience from the satellite remote sensing community has shown that a set of robust and well-articulated definitions of "data processing levels" [1]–[5] can lead directly to broad and highly productive use of data. Processing level designations also provide important context for data interpretation [6]. However, no such community agreement exists for active acoustic data. The ambiguity associated with the interoperability and inter-comparability of processed sonar data products has hindered efficient collaboration and integrative use of the rapidly growing data archive across research institutions and agencies.
+
+The `echopype` team is developing a clearly defined progression of data processing levels for active ocean sonar data. The development leverages the collective experience from remote sensing and large-scale, long-term ocean and ecological observatories [7]–[10]. Data processing functions in `echopype` are clearly associated with specific Processing Level inputs and outputs, and when appropriate, will generate a `processing_level` dataset global attribute with entries such as "Level 1A", "Level 2B", "Level 4", etc.
+
+
+## Processing Levels and Sub-levels
+
+### Level 0 (L0)
+
+**Description:**  Raw data in vendor sensor format.
+
+- raw binary files. Associated metadata may be found in external files.
+
+### Level 1 (L1)
+
+**Description:** Raw data packaged with ancillary information and converted and standardized to an open convention and standard data formats. May be distributed in the following two forms:
+  - as sets of individual converted files as originally segmented into arbitrary time ranges during sensor file creation, or
+  - compiled into larger granules corresponding to logical deployment intervals.
+
+- **L1A**: Raw L0 data converted to a standardized, open format with geographic coordinates (latitude & longitude) included. Includes other ancillary information extracted from sensor-generated L0 data or other external sources. May include environmental information such as temperature, salinity and pressure. Use of the SONAR-netcDF4 v1 convention is strongly recommended.
+- **L1B**: L1A data with quality-control steps applied, such as time-coordinate corrections that enforce strictly increasing, non-duplicate timestamps.
+
+### Level 2 (L2)
+
+**Description:** Calibrated acoustic quantities at raw data resolution, with spatial coordinates included (latitude, longitude and depth)
+
+- **L2A**: Volume backscattering strength (`Sv`) with interpolated latitude, longitude and depth coordinates. May incorporate addition information, such as  split beam angle
+- **L2B**: `Sv` L2A data with noise removal or other data filtering applied, including seafloor bottom removal.
+
+### Level 3 (L3)
+
+**Description:** Calibrated acoustic quantities regridded or aggregated to a common grid across channels. May include noise removal or other filtering.
+
+- `Sv` resampled to a common, high-resolution grid across channels
+- Mean Volume Backscattering Strength (`MVBS`)
+- Masks applied to `Sv` based on different scattering source classification criteria and methods, such as frequency difference across two channels.
+- **L3A**: The above variables computed on L2A data
+- **L3B**: The above variables computed on L2B (filtered) data
+
+### Level 4 (L4)
+
+**Description:** Acoustically derived biological features, involving further processing of L3 data that may include data reduction or incorporation of external sources of data.
+
+- Nautical Area Backscattering Coefficients (`NASC`), potentially partitioned into inferred sources of scattering
+- Summary statistics of echogram features (center_of_mass, dispersion, etc)
+- Taxon or species-level data labels (classification). May originate from a variety of methods, including frequency difference thresholds.
+- Estimated biomass, combined or partitioned into different taxonomic sources.
+
+
+## References
+
+- [1] Parkinson, C. L., A. Ward, and M. D. King (eds.). 2006. Earth science reference handbook: A guide to NASA’s Earth Science Program and Earth Observing Satellite Missions. National Aeronautics and Space Administration. https://atrain.nasa.gov/publications/2006ReferenceHandbook.pdf
+- [2] NASA. 2021. Data Processing Levels | Earthdata. Last viewed Mar. 24, 2023. https://earthdata.nasa.gov/collaborate/open-data-services-and-software/data-information-policy/data-levels/
+- [3] Ramapriyan, H. K., and P. J. T. Leonard. 2021. Data Product Development Guide (DPDG) for Data Producers version1.1. NASA Earth Science Data and Information System Standards Office, 21 October 2021. https://doi.org/10.5067/DOC/ESO/RFC-041VERSION1
+- [4] Robinson, I. 2006. Satellite Measurements for Operational Ocean Models, pp. 147-189. In: Chassignet, E.P. and Verron, J. (eds). Ocean Weather Forecasting: An Integrated View of Oceanography. Springer, New York, NY. https://doi.org/10.1007/1-4020-4028-8_6
+- [5] Weaver, R. 2014. Processing Levels, pp. 517-520. In: Njoku, E.G. (ed). Encyclopedia of Remote  Sensing. Encyclopedia of Earth Sciences Series. Springer, New York, NY.  https://doi.org/10.1007/978-0-387-36699-9_36
+- [6] Hills, D. J., R. R. Downs, R. Duerr, J. C. Goldstein, M. A. Parsons, and H. K. Ramapriyan. 2015. The importance of data set provenance for science. Eos, 96, Published on 4 December 2015. https://doi.org/10.1029/2015EO040557
+- [7] Heaney, K., B. Martin, J. Miksis-Olds, T. Moore, J. Warren, and M. Ainslie. 2020. ADEON data processing specification. Version 1.0. Technical report by Applied Ocean Sciences for Prime Contract No. M16PC00003, Apr. 2020. https://adeon.unh.edu/sites/default/files/user-uploads/ADEON_Data%20Processing_Specification_FINAL.pdf
+- [8] IFREMER. 2019. Processing Levels - Oceanographic Data. Last viewed Mar. 24, 2023. http://en.data.ifremer.fr/All-about-data/Data-management/Processing-levels
+- [9] NEON. 2023 Data Processing | NSF NEON | Open Data to Understand our Ecosystems. Last viewed Mar. 24, 2023. https://www.neonscience.org/data-samples/data-management/data-processing
+- [10] OOI. 2023. Ocean Observatories Initiative: Glossary - Data Terminology. Ocean Observatories Initiative. Last viewed Mar. 24, 2023. https://oceanobservatories.org/glossary/#DATATERMS
diff --git a/docs/source/whats-new.md b/docs/source/whats-new.md
@@ -4,6 +4,45 @@ What's new
 See [GitHub releases page](https://github.com/OSOceanAcoustics/echopype/releases) for the complete history.
 
 
+# v0.7.0 (2023 March 25)
+
+## Overview
+
+This release includes new features to interface with Echoview ECS files for computing Sv, reorganization of computing functions into new subpackages, addition of data processing level attributes to data products, and other improvements and bug fixes.
+
+## New features and major changes
+- Allow using ECS for calibrating Simrad echosounders (#996, #1004)
+  - This functionality is in a **beta** testing stage
+  - Details of implementation may change and bugs are possible
+  - Expand ECS parser to accept frequency-dependent values in EK80 ECS files
+- Overhaul `env_params` to ensure correct intake for calibration (#985)
+  - Now allows using `env_params` entries that are xr.DataArrays
+- Move functions previously in `preprocess` subpackage to new subpackages (#993). Calling these functions from `preprocess` is deprecated and will be removed in v0.7.1.
+  - `clean`: `remove_noise`, `estimate_noise`
+  - `commongrid`: `compute_MVBS`, `compute_MVBS_index_binning`
+- Add `commongrid.compute_NASC` (#1005)
+  - The current implementation uses brute force looping for mean Sv computation, this will be refactored and optimized together with other functions requiring the same pattern in an upcoming release
+- Add global attributes for data processing levels (#1001).
+  - This functionality is in a **beta** testing stage
+  - See [data processing level specifications](https://echopype.readthedocs.io/en/stable/processing-levels.html) for functions and conditions under which such attributes are added
+- Expand `mask.apply_mask` to handle multi-channel Sv datasets (#1010)
+- Standardize sonar metadata for EK80 data (#992)
+    - `sonar_serial_number` is now an empty global attribute, no longer a variable, as in the EK60 case
+    - `transducer_name`, `transducer_serial_number`, `transceiver_serial_number` based on parser parameter `transducer_name`, `transducer_serial_number`, and `serial_number`, respectively
+
+## Bug fixes
+- Fix scaling bug for `beamwidth_alongship` and `beamwidth_athwartship` from CW-based parameters to values corresponding to center frequency of broadband transmit signals (#998)
+
+## Tests and infrastructure
+- Add more comprehensive tests for `add_location` (#1000)
+- Add test for splitbeam angle `ek80_CW_power` case (#994)
+- Add unit and integration tests for `env_params` intake for calibration (#985)
+- Exclude `test_data` folder in `codespell` pre-commit hook (#1016)
+
+
+
+
+
 # v0.6.4.1 (2023 March 15)
 
 ## Overview

diff --git a/echopype/__init__.py b/echopype/__init__.py
@@ -2,7 +2,7 @@
 
 from _echopype_version import version as __version__  # noqa
 
-from . import calibrate, consolidate, mask, preprocess, utils
+from . import calibrate, clean, commongrid, consolidate, mask, preprocess, utils
 from .convert.api import open_raw
 from .echodata.api import open_converted
 from .echodata.combine import combine_echodata
@@ -15,12 +15,14 @@
 init_ep_dir()
 
 __all__ = [
-    "open_raw",
-    "open_converted",
-    "combine_echodata",
     "calibrate",
+    "clean",
+    "combine_echodata",
+    "commongrid",
     "consolidate",
     "mask",
+    "open_converted",
+    "open_raw",
     "preprocess",
     "utils",
     "verbose",

diff --git a/echopype/calibrate/api.py b/echopype/calibrate/api.py
@@ -24,6 +24,7 @@ def _compute_cal(
     echodata: EchoData,
     env_params=None,
     cal_params=None,
+    ecs_file=None,
     waveform_mode=None,
     encode_mode=None,
 ):
@@ -49,6 +50,7 @@ def _compute_cal(
         echodata,
         env_params=env_params,
         cal_params=cal_params,
+        ecs_file=ecs_file,
         waveform_mode=waveform_mode,
         encode_mode=encode_mode,
     )

diff --git a/echopype/calibrate/cal_params.py b/echopype/calibrate/cal_params.py
@@ -49,10 +49,6 @@
 }
 
 
-# TODO: need a function (something like "_check_param_freq_dep")
-# to check user input cal_params and env_params
-
-
 def param2da(p_val: Union[int, float, list], channel: Union[list, xr.DataArray]) -> xr.DataArray:
     """
     Organize individual parameter in scalar or list to xr.DataArray with channel coordinate.
@@ -88,7 +84,7 @@ def param2da(p_val: Union[int, float, list], channel: Union[list, xr.DataArray])
 
 def sanitize_user_cal_dict(
     sonar_type: Literal["EK60", "EK80", "AZFP"],
-    user_dict: Dict[str, Union[int, float, xr.DataArray]],
+    user_dict: Dict[str, Union[int, float, list, xr.DataArray]],
     channel: Union[List, xr.DataArray],
 ) -> Dict[str, Union[int, float, xr.DataArray]]:
     """
@@ -126,7 +122,7 @@ def sanitize_user_cal_dict(
 
     # Screen parameters: only retain those defined in CAL_PARAMS
     #  -- transform params in scalar or list to xr.DataArray
-    #  -- directly pass through those that are xr.DataArray
+    #  -- directly pass through those that are xr.DataArray and pass the check for coordinates
     out_dict = dict.fromkeys(CAL_PARAMS[sonar_type])
     for p_name, p_val in user_dict.items():
         if p_name in out_dict:
@@ -470,14 +466,17 @@ def _get_fs():
                         if p in [
                             "angle_sensitivity_alongship",
                             "angle_sensitivity_athwartship",
+                        ]:
+                            BB_factor = freq_center / beam["frequency_nominal"]
+                        elif p in [
                             "beamwidth_alongship",
                             "beamwidth_athwartship",
                         ]:
-                            BB_factor = freq_center / beam["frequency_nominal"]
+                            BB_factor = beam["frequency_nominal"] / freq_center
                         else:
                             BB_factor = 1
 
-                        p_beam = PARAM_BEAM_NAME_MAP[p]
+                        p_beam = PARAM_BEAM_NAME_MAP[p]  # Beam_groupX data variable name
                         out_dict[p] = _get_interp_da(
                             da_param=None if p not in vend else vend[p],
                             freq_center=freq_center,

diff --git a/echopype/calibrate/calibrate_azfp.py b/echopype/calibrate/calibrate_azfp.py
@@ -8,14 +8,20 @@
 
 
 class CalibrateAZFP(CalibrateBase):
-    def __init__(self, echodata: EchoData, env_params=None, cal_params=None, **kwargs):
-        super().__init__(echodata, env_params, cal_params)
+    def __init__(
+        self, echodata: EchoData, env_params=None, cal_params=None, ecs_file=None, **kwargs
+    ):
+        super().__init__(echodata, env_params, cal_params, ecs_file)
 
         # Set sonar_type
         self.sonar_type = "AZFP"
 
+        # Screen for ECS file: currently not support
+        if self.ecs_file is not None:
+            raise ValueError("Using ECS file for calibration is not currently supported for AZFP!")
+
         # load env and cal parameters
-        self.env_params = get_env_params_AZFP(echodata=self.echodata, user_env_dict=self.env_params)
+        self.env_params = get_env_params_AZFP(echodata=self.echodata, user_dict=self.env_params)
         self.cal_params = get_cal_params_AZFP(
             beam=self.echodata["Sonar/Beam_group1"],
             vend=self.echodata["Vendor_specific"],

diff --git a/echopype/calibrate/calibrate_base.py b/echopype/calibrate/calibrate_base.py
@@ -1,28 +1,50 @@
 import abc
 
 from ..echodata import EchoData
+from ..utils.log import _init_logger
+from .ecs import ECSParser
+
+logger = _init_logger(__name__)
 
 
 class CalibrateBase(abc.ABC):
     """Class to handle calibration for all sonar models."""
 
-    def __init__(self, echodata: EchoData, env_params=None, cal_params=None):
+    def __init__(self, echodata: EchoData, env_params=None, cal_params=None, ecs_file=None):
         self.echodata = echodata
         self.sonar_type = None
-
-        if env_params is None:
+        self.ecs_file = ecs_file
+        self.ecs_dict = {}
+
+        # Set ECS to overwrite user-provided dict
+        if self.ecs_file is not None:
+            if env_params is not None or cal_params is not None:
+                logger.warning(
+                    "The ECS file takes precedence when it is provided. "
+                    "Parameter values provided in 'env_params' and 'cal_params' will not be used!"
+                )
+
+            # Parse ECS file to a dict
+            ecs = ECSParser(self.ecs_file)
+            ecs.parse()
+            self.ecs_dict = ecs.get_cal_params()  # apply ECS hierarchy
             self.env_params = {}
-        elif isinstance(env_params, dict):
-            self.env_params = env_params
-        else:
-            raise ValueError("'env_params' has to be None or a dict")
-
-        if cal_params is None:
             self.cal_params = {}
-        elif isinstance(cal_params, dict):
-            self.cal_params = cal_params
+
         else:
-            raise ValueError("'cal_params' has to be None or a dict")
+            if env_params is None:
+                self.env_params = {}
+            elif isinstance(env_params, dict):
+                self.env_params = env_params
+            else:
+                raise ValueError("'env_params' has to be None or a dict")
+
+            if cal_params is None:
+                self.cal_params = {}
+            elif isinstance(cal_params, dict):
+                self.cal_params = cal_params
+            else:
+                raise ValueError("'cal_params' has to be None or a dict")
 
         # range_meter is computed in compute_Sv/TS in child class
         self.range_meter = None