From 7e7e6c9b43d410b8f19cebc9de9b1ff925967e62 Mon Sep 17 00:00:00 2001 From: Roland Proud Date: Mon, 13 Dec 2021 17:16:38 +0000 Subject: [PATCH] Compatibility with ES70, ES80 and SEAT files 1.) Added ES70 compatibility to simrad_parsers (same as ES60). 2.) Added SEAT file compatibility as new instrument module (Historic ES80 files that have been converted to EK60 data format i.e. RAW0 format, via SEAT software - resulting in files that have a mixture of EK80 and EK60 datagrams). For processing historic data only. 3.) Checked compatibility with ES80 files (works well). NB - missing metadata capture in some ES70 and EK60 files: TrawlUpperDepthValid TrawlOpeningValid TrawlUpperDepth TrawlOpening --- echolab2/instruments/SEAT.py | 2763 +++++++++++++++++++ echolab2/instruments/util/simrad_parsers.py | 43 +- 2 files changed, 2800 insertions(+), 6 deletions(-) create mode 100644 echolab2/instruments/SEAT.py diff --git a/echolab2/instruments/SEAT.py b/echolab2/instruments/SEAT.py new file mode 100644 index 0000000..ed581fc --- /dev/null +++ b/echolab2/instruments/SEAT.py @@ -0,0 +1,2763 @@ +# coding=utf-8 + +# National Oceanic and Atmospheric Administration +# Alaskan Fisheries Science Center +# Resource Assessment and Conservation Engineering +# Midwater Assessment and Conservation Engineering + +# THIS SOFTWARE AND ITS DOCUMENTATION ARE CONSIDERED TO BE IN THE PUBLIC DOMAIN +# AND THUS ARE AVAILABLE FOR UNRESTRICTED PUBLIC USE. THEY ARE FURNISHED "AS +# IS." THE AUTHORS, THE UNITED STATES GOVERNMENT, ITS INSTRUMENTALITIES, +# OFFICERS, EMPLOYEES, AND AGENTS MAKE NO WARRANTY, EXPRESS OR IMPLIED, AS TO +# THE USEFULNESS OF THE SOFTWARE AND DOCUMENTATION FOR ANY PURPOSE. THEY +# ASSUME NO RESPONSIBILITY (1) FOR THE USE OF THE SOFTWARE AND DOCUMENTATION; +# OR (2) TO PROVIDE TECHNICAL SUPPORT TO USERS. + +''' +.. module:: echolab2.instruments.SEAT + + :synopsis: A high-level interface for reading SIMRAD ".raw" formatted + files written by the Simrad ES80 WBES system that have been + formatted by the SEAT software + + +| Developed by: Rick Towler +| National Oceanic and Atmospheric Administration (NOAA) +| Alaska Fisheries Science Center (AFSC) +| Midwater Assesment and Conservation Engineering Group (MACE) +| +| Authors: +| Rick Towler +| Maintained by: +| Rick Towler +| Modified by: +| Roland Proud + +$Id$ +''' + +import os +import numpy as np +from scipy.interpolate import interp1d +from .util.simrad_calibration import calibration +from .util.simrad_raw_file import RawSimradFile, SimradEOF +from .util.nmea_data import nmea_data +from .util.motion_data import motion_data +from .util.annotation_data import annotation_data +from .util import simrad_signal_proc +from .util import date_conversion +from .util import simrad_parsers +from ..ping_data import ping_data +from ..processing.processed_data import processed_data +from ..processing import line + +class SEAT(object): + """This class is the 'file reader' class for Simrad ES80 instrument files + that include raw data in the EK60 data format (converted via SEAT software). + This class is for processing historical data only. + + The SEAT class can read in one or more SEAT files and generates a raw_data + class instance for each unique channel ID in the files. The class also + contains numerous methods used to extract the raw sample data from one + channel, or create processed_data objects containing. + + Attributes: + start_time: Start_time is a datetime object that defines the start + time of the data within the SEAT class + end_time: End_time is a datetime object that defines the end time of + the data within the SEAT class + start_ping: Start_ping is an integer that defines the first ping of the + data within the SEAT class. + end_ping: End_ping is an integer that defines the last ping of + the data within the SEAT class. + n_pings: Integer value representing the total number of pings. + frequencies: List of frequencies read from raw data files. + channel_ids: List of stings identifying the unique channel IDs read + from the instrument files. + n_channels: Integer value representing the total number of + channels in the class instance. + raw_data: A dictionary that stores the raw_data objects, one for each + unique channel in the files. The dictionary keys are the channel + numbers. + nmea_data: reference to a NmeaData class instance that will contain + the NMEA data from the data files. + read_incremental: Boolean value controlling whether files are read + incrementally or all at once. The default value is False. + store_angles: Boolean control variable to set whether or not to store + angle data generated by EK60 GPTs or WBTs operated in CW + reduced mode. + store_power: Boolean control variable to set whether or not to store + the power data generated by EK60 GPTs or WBTs operated in CW + reduced mode. + store_complex: Boolean control variable to set whether or not to store + the complex data generated by EK80 WBTs. + read_max_sample_count: Integer value to specify the max sample count + to read. This property can be used to limit the number of samples + read (and memory used) when your data of interest is less than + the total number of samples contained in the EK80 files. + read_start_time: Datetime64 object containing timestamp of the first + ping to read if not reading all pings based on time. + read_end_time: Datetime64 object containing timestamp of last ping to + read if not reading all ping based on time. + read_start_ping: Integer ping number of first ping to read if not + reading all pings based on ping number. + read_end_ping: Integer ping number of last ping to read if not + reading all pings based on ping number. + read_start_sample: Integer sample number of first sample to read if not + reading all samples based on sample number. + read_end_sample: Integer sample number of last sample to read if not + reading all samples based on sample number. + read_frequencies: List of floats (i.e. 18000.0) specifying the + frequencies to read. An empty list will result in all frequencies + being read. + read_channel_ids = List of strings specifying the channel_ids + (i,e, 'GPT 38 kHz 009072033fa2 1-1 ES38B') of the channels to + read. An empty list will result in all channels being read. + """ + + def __init__(self): + """Initializes SEAT class object. + + Creates and sets several internal properties used to store information + about data and control operation of file reading with SEAT object + instance. Code is heavily commented to facilitate use. + """ + + # Define EK80 properties. + + # These properties control what data are stored when reading + # EK80 raw files. These properties can be manipulated directly or by + # specifying them when calling the read and append methods. + + # Set store_angles to true to store angle data if any of the data files + # contain data recorded from EK60 GPTs or WBT data saved in the reduced + # power/angle format. + self.store_angles = True + + # Set store_power to true to store power data if any of the data files + # contain data recorded from EK60 GPTs or WBT data saved in the reduced + # power/angle format. + self.store_power = True + + # Set store_complex to true to store complex data if any of the data files + # contain data recorded from EK80 WBTs saved as complex samples. + self.store_complex = True + + # Specify the maximum sample count to read. This property can be used + # to limit the number of samples read (and memory used). Samples beyond + # the maximum will be dropped. + self.read_max_sample_count = None + + # These params store any limits set on storing of data contained in + # the raw files read. These parameters can be set directly, or through + # many of the methods where you can specify start/end time or ping + # and sample number. + self.read_start_time = None + self.read_end_time = None + self.read_start_ping = None + self.read_end_ping = None + self.read_start_sample = None + self.read_end_sample = None + + # read_frequencies can be set to a list of floats specifying the + # frequencies to read. An empty list will result in all frequencies + # being read. + self.read_frequencies = [] + + # read_channel_ids can be set to a list of strings specifying the + # channel_ids of the channels to read. An empty list will result in all + # channels being read. + self.read_channel_ids = [] + + # initialize the data arrays + self._init() + + + def _init(self): + '''_init is an internal method that initializes the data fields of + the SEAT class. + ''' + + # The start_time and end_time will define the time span of the + # data within the SEAT class. + self.start_time = None + self.end_time = None + + # The start_ping and end_ping will define the ping span of the data + # within the SEAT class. + self.start_ping = None + self.end_ping = None + + # n_pings stores the total number of pings read. + self.n_pings = 0 + + # n_files stores the total number of raw files read. + self.n_files = 0 + + # A list of stings identifying the channel IDs that have been read. + self.channel_ids = [] + + # n_channels stores the total number of channels in the object. + self.n_channels = 0 + + # A dictionary to store the raw_data objects. The dictionary is keyed + # by channel ID and each value is a list of raw_data objects associated + # with that channel. + self.raw_data = {} + + # frequency_map maps frequency in Hz to channel ID. + self.frequency_map = {} + + # channel_map maps channel number to channel ID. Channel numbers + # are based on the order each channel is added to the ER60 object. + self.channel_map = {} + + # file_channel_map maps channel number to channel ID for the current file. + self._file_channel_map = {} + + # nmea_data stores the util.nmea_data object which will contain + # the NMEA data read from the raw file. This object has methods + # to extract, parse, and interpolate the NMEA data. + self.nmea_data = nmea_data() + + # motion_data is the util.motion_data object which will + # contain data from the MRU datagrams contained in the raw file. + # This object has methods to extract and interpolate the motion data. + self.motion_data = motion_data() + + # annotations stores the contents of the TAG0 aka "annotation" + # datagrams. + self.annotations = annotation_data() + + # data_array_dims contains the dimensions of the sample and angle or + # complex data arrays specified as [n_pings, n_samples]. Values of + # -1 specifythat the arrays will resize appropriately to contain all + # pings and all samples read from the data source. Setting values > 0 + # will create arrays that are fixed to that size. Any samples beyond + # the number specified as n_samples will be dropped. When a ping is + # added and the data arrays are full, the sample data will be rolled + # such that the oldest data is dropped and the new ping is added. + self._data_array_dims = [-1, -1] + + # initialize the ping time - used to group "pings" + self._this_ping_time = np.datetime64('1000-02') + + # _last_progress stores the last reported progess through the current + # file in percent as integer. + self._last_progress = -1 + + # initialize some internal attributes + self._config = None + self._filters = {} + self._tx_params = {} + self._environment = None + + + def read_raw(self, *args, **kwargs): + """Reads one or more Simrad SEAT .raw files into memory. Overwrites + existing data (if any). The data are stored in an SEAT.raw_data object. + + + Args: + raw_files (list): List containing full paths to data files to be + read. + power (bool): Controls whether power data is stored + angles (bool): Controls whether angle data is stored + complex (bool): Controls whether complex data is stored + max_sample_count (int): Specify the max sample count to read + if your data of interest is less than the total number of + samples contained in the instrument files. + start_time (datetime64): Specify a start time if you do not want + to start reading from the first ping. + end_time (datetime64): Specify an end time if you do not want to read + to the last ping. + start_ping (int): Specify starting ping number if you do not want + to start reading at first ping. + end_ping (int): Specify end ping number if you do not want + to read all pings. + frequencies (list): List of floats (i.e. 18000.0) if you + only want to read specific frequencies. + channel_ids (list): A list of strings that contain the unique + channel IDs to read. If no list is supplied, all channels are + read. + start_sample (int): Specify starting sample number if not + reading from first sample. + end_sample (int): Specify ending sample number if not + reading to last sample. + """ + + # initialize the data arrays - this discards any previously read data. + self._init() + + # now call append_raw, passing on all arguments + self.append_raw(*args, **kwargs) + + + def append_raw(self, raw_files, power=None, angles=None, complex=None, + max_sample_count=None, start_time=None, end_time=None, + start_ping=None, end_ping=None, frequencies=None, + channel_ids=None, incremental=None, start_sample=None, + end_sample=None, progress_callback=None): + """Reads one or more Simrad SEAT .raw files and appends the data to any + existing data. The data are ordered as read. + + + Args: + raw_files (list): List containing full paths to data files to be + read. + power (bool): Controls whether power data is stored + angles (bool): Controls whether angle data is stored + complex (bool): Controls whether complex data is stored + max_sample_count (int): Specify the max sample count to read + if your data of interest is less than the total number of + samples contained in the instrument files. + start_time (datetime64): Specify a start time if you do not want + to start reading from the first ping. + end_time (datetime64): Specify an end time if you do not want to read + to the last ping. + start_ping (int): Specify starting ping number if you do not want + to start reading at first ping. + end_ping (int): Specify end ping number if you do not want + to read all pings. + frequencies (list): List of floats (i.e. 18000.0) if you + only want to read specific frequencies. + channel_ids (list): A list of strings that contain the unique + channel IDs to read. If no list is supplied, all channels are + read. + start_sample (int): Specify starting sample number if not + reading from first sample. + end_sample (int): Specify ending sample number if not + reading to last sample. + progress_callback (function reference): Pass a reference to a + function that accepts three arguments: + (filename, cumulative_pct, cumulative_bytes) + This function will be periodicaly called, passing the + current file name that is being read, the percent read, + and the bytes read. + """ + + # Update the reader state variables. + if start_time: + self.read_start_time = start_time + if end_time: + self.read_end_time = end_time + if start_ping: + self.read_start_ping = start_ping + if end_ping: + self.read_end_ping = end_ping + if start_sample: + self.read_start_sample = start_sample + if end_sample: + self.read_end_sample = end_sample + if power: + self.store_power = power + if complex: + self.store_complex = complex + if angles: + self.store_angles = angles + if max_sample_count: + self.read_max_sample_count = max_sample_count + if frequencies: + self.read_frequencies = frequencies + if channel_ids: + self.read_channel_ids = channel_ids + + # Ensure that the raw_files argument is a list. + if isinstance(raw_files, str): + raw_files = [raw_files] + + # Iterate through the list of .raw files to read. + for filename in raw_files: + # reset attributes that store the most recent XML configuration datagram + # FIL filter datagrams, XML environment datagrams and XML parameter datagrams + self._config = None + self._file_channel_map = {} + self._file_channels = 0 + self._filters = {} + self._tx_params = {} + self._environment = None + + # get the total file size + total_bytes = os.stat(filename).st_size + + # normalize the file path and split out the parts + filename = os.path.normpath(filename) + file_parts = filename.split(os.path.sep) + current_filename = file_parts[-1] + current_filepath = os.path.sep.join(file_parts[0:-1]) + + # open the raw file + fid = RawSimradFile(filename, 'r') + + # read the configuration datagram - this will always be the + # first datagram in an EK80 raw file. + _config = self._read_config(fid) + + # extract the per channel data into our internal config attribute + # and add some additional fields to each channel + self._config = _config['configuration'] + for channel in self._config: + self._config[channel]['file_name'] = current_filename + self._config[channel]['file_path'] = current_filepath + self._config[channel]['file_bytes'] = total_bytes + self._config[channel]['start_time'] = _config['timestamp'] + self._config[channel]['start_ping'] = self.n_pings + + # set the cumulative_bytes var and start time + cumulative_bytes = _config['bytes_read'] + + # create a variable to track when we're done reading. + finished = False + + # and read datagrams until we're done + while not finished: + # read a datagram - method returns some basic info + dg_info = self._read_datagram(fid) + + # call progress callback if supplied + if (progress_callback): + # determine the progress as an integer percent. + cumulative_bytes += dg_info['bytes_read'] + cumulative_pct = round(cumulative_bytes / total_bytes * 100.) + + # call the callback when the percent changes + if cumulative_pct != self._last_progress: + # call the provided callback - the callback has 3 args, + # the first is the full path to the current file and the + # second is the percent read and the third is the bytes read. + progress_callback(filename, cumulative_pct, cumulative_bytes) + self._last_progress = cumulative_pct + + # update our finished state var - the file will be finished when + # the reader hits the end of the file or if end time/ping are + # set and we hit that point. + finished = dg_info['finished'] + + # close the file + fid.close() + + # Trim excess data from arrays after reading. + for channel_id in self.channel_ids: + for raw in self.raw_data[channel_id]: + raw.trim() + self.nmea_data.trim() + self.motion_data.trim() + self.annotations.trim() + + + def _read_config(self, fid): + ''' + _read_config reads the raw file configuration datagram. It then checks if + if there are any new channels it should be storing and if so, creates + raw_data objects for those channels and updates some attributes. + ''' + + # read the configuration datagram - this is the first datagram + # in an EK80 file. + config_datagram = fid.read(1) + config_datagram['timestamp'] = \ + np.datetime64(config_datagram['timestamp'], '[ms]') + + # check for any new channels and add them if required + + file_channels = config_datagram['configuration'].keys() + + for channel_id in file_channels: + # Increment the per file channel total + self._file_channels += 1 + + # check if we're reading this channel + if (not self.read_channel_ids or channel_id in self.read_channel_ids): + # check if we're reading this frequency + frequency = config_datagram['configuration'][channel_id]['transducer_frequency'] + if (self.read_frequencies and frequency not in self.read_frequencies): + # There are specific frequencies specified and this + # is NOT one of them. Remove this channel from the config_datagram + # dictionary and continue. + config_datagram['configuration'].pop(channel_id) + continue + + if channel_id not in self._file_channel_map: + + # populate the _file_channel_map which maps channel + # number to channel id *for this file only*. + self._file_channel_map[self._file_channels] = channel_id + + # Set the file channel number in the configuration - we need this + # in certain cases when writing data. + config_datagram['configuration'][channel_id]['channel_number'] = self._file_channels + + # we're reading this channel - check if it's new to us + if channel_id not in self.raw_data: + # This is a new channel, create a list to store this + # channel's raw_data objects + self.raw_data[channel_id] = [] + + # add the channel to our list of channel IDs + self.channel_ids.append(channel_id) + + # and increment the channel counter + self.n_channels += 1 + + # and populate the frequency map + if frequency in self.frequency_map: + self.frequency_map[frequency].append(channel_id) + else: + self.frequency_map[frequency] = [channel_id] + + # check if we need to create an entry for this channel's filters + if channel_id not in self._filters: + # and an empty dict to store this channel's filters + self._filters[channel_id] = {} + + # Return the configuration datagram dict + return config_datagram + + + def _read_datagram(self, fid): + """Reads the next raw file datagram + + This method reads the next datagram from the file, storing the + data contained in the datagram (if applicable), and returns a dict + containing the number of bytes read, the datagramn timestamp, datagram + type, and whether the reader is "finished". The finished flag will be set + if end_time and/or end_ping is set in the reader and the datagram time + is after the end time or the internal "ping" counter matches the + specified end ping. + + Args: + fid (file object): Pointer to currently open file object. This is a + RawSimradFile file object and not the standard Python file + object. + """ + # create the return dict that provides feedback on progress + result = {'bytes_read':0, 'timestamp':None, 'type':None, 'finished':False} + + # attempt to read the next datagram + try: + new_datagram = fid.read(1) + except SimradEOF: + # we're at the end of the file + result['finished'] = True + return result + + # check if this is a valid datagram, otherwise skip + if type(new_datagram) == bytes: + #print(new_datagram) + return result + + # Convert the timestamp to a datetime64 object. + # Check for NULL datagram date/time which is returned as datetime.datetime(1601, 1, 1, 0, 0) + if new_datagram['timestamp'].year < 1900: + # This datagram has NULL date/time values + new_datagram['timestamp'] = np.datetime64("NaT") + else: + # We have a plausible date/time value + new_datagram['timestamp'] = \ + np.datetime64(new_datagram['timestamp'], '[ms]') + + # update the return dict properties + result['timestamp'] = new_datagram['timestamp'] + result['bytes_read'] = new_datagram['bytes_read'] + result['type'] = new_datagram['type'] + + # We have to process all XML parameter and environment datagrams + # regardless of time/ping bounds. This ensures all pings have fresh + # references to these data. + if new_datagram['type'].startswith('XML'): + if new_datagram['subtype'] == 'parameter': + # update the most recent parameter attribute for this channel + self._tx_params[new_datagram[new_datagram['subtype']]['channel_id']] = \ + new_datagram[new_datagram['subtype']] + elif new_datagram['subtype'] == 'environment': + # update the most recent environment attribute + self._environment = new_datagram[new_datagram['subtype']] + + return result + + # Check if data should be stored based on time bounds. + if self.read_start_time is not None: + if new_datagram['timestamp'] < self.read_start_time: + # we have a start time but this data comes before it + # so we return without doing anything else + return result + if self.read_end_time is not None: + if new_datagram['timestamp'] > self.read_end_time: + # we have a end time and this data comes after it + # so we are actually done reading - set the finished + # field in our return dict and return + result['finished'] = True + return result + + # update the ping counter + if new_datagram['type'].startswith('RAW'): + if self._this_ping_time != new_datagram['timestamp']: + self.n_pings += 1 + self._this_ping_time = new_datagram['timestamp'] + + # check if we're storing this channel + if new_datagram['channel'] not in self._file_channel_map.keys(): + # no, it's not in the list - just return + return result + else: + # add the channel_id to our datagram + new_datagram['channel_id'] = self._file_channel_map[new_datagram['channel']] + + # Check if we should store this data based on ping bounds. + if self.read_start_ping is not None: + if self.n_pings < self.read_start_ping: + # we have a start ping but this data comes before it + # so we return without doing anything else + return result + if self.read_end_ping is not None: + if self.n_pings > self.read_end_ping: + # we have a end ping and this data comes after it + # so we are actually done reading - set the finished + # field in our return dict and return + result['finished'] = True + return result + + # Update the end_time property. + if self.end_time is not None: + # We can't assume data will be read in time order. + if self.end_time < new_datagram['timestamp']: + self.end_time = new_datagram['timestamp'] + else: + self.end_time = new_datagram['timestamp'] + + # Process and store the datagrams by type. + + # FIL datagrams store parameters used to filter the received signal + # SEAT class stores the filters for the currently being read file. + # Filters are stored by channel ID and then by filter stage + if new_datagram['type'].startswith('FIL'): + + # Check if we're storing this channel + if new_datagram['channel_id'] in self.channel_ids: + + # add the filter parameters for this filter stage + self._filters[new_datagram['channel_id']][new_datagram['stage']] = \ + {'stage':new_datagram['stage'], + 'n_coefficients':new_datagram['n_coefficients'], + 'decimation_factor':new_datagram['decimation_factor'], + 'coefficients':new_datagram['coefficients']} + + # RAW datagrams store raw acoustic data for a channel. + elif new_datagram['type'].startswith('RAW'): + # Check if we're storing this channel + if new_datagram['channel_id'] in self.channel_ids: + + # check if we should set our start time property + if not self.start_time: + self.start_time = new_datagram['timestamp'] + # Set the first ping number we read. + if not self.start_ping: + self.start_ping = self.n_pings + # Update the last ping number. + self.end_ping = self.n_pings + + # loop through the raw_data objects to find the raw_data object + # to store this data. + this_raw_data = None + for raw_obj in self.raw_data[new_datagram['channel_id']]: + if raw_obj.data_type == '': + # This raw_data object has no type so is empty and can store anything + this_raw_data = raw_obj + break + ############################ EDIT + elif (new_datagram['mode'] == 1 and raw_obj.data_type == 'power' or + new_datagram['mode'] == 2 and raw_obj.data_type == 'angle' or + new_datagram['mode'] == 3 and raw_obj.data_type == 'power/angle'): + # This raw_data object contains a matching data type + this_raw_data = raw_obj + break + ############################ END EDIT + + # check if we need to create a new raw_data object + if this_raw_data is None: + # create the new raw_data object + this_raw_data = raw_data(new_datagram['channel_id'], + store_power=self.store_power, + store_angles=self.store_angles, + #store_complex=self.store_complex, + max_sample_number=self.read_max_sample_count) + # Set the transceiver type + this_raw_data.transceiver_type = "GPT" + # and add it to this channel's list of raw_data objects + self.raw_data[new_datagram['channel_id']].append(this_raw_data) + + # Call this channel's append_ping method. + this_raw_data.append_ping(new_datagram, + self._config[new_datagram['channel_id']],self._environment, + start_sample=self.read_start_sample, + end_sample=self.read_end_sample) + + # NME datagrams store ancillary data as NMEA-0183 style ASCII data. + elif new_datagram['type'].startswith('NME'): + # Add the datagram to our nmea_data object. + self.nmea_data.add_datagram(new_datagram['timestamp'], + new_datagram['nmea_string']) + + # TAG datagrams contain time-stamped annotations inserted via the + # recording software. They are not associated with a specific channel + elif new_datagram['type'].startswith('TAG'): + # Add this datagram to our annotation_data object + self.annotations.add_datagram(new_datagram['timestamp'], + new_datagram['text']) + + # MRU datagrams contain vessel motion data + elif new_datagram['type'].startswith('MRU'): + # append this motion datagram to the motion_data object + self.motion_data.add_datagram(new_datagram['timestamp'], + new_datagram['heave'], new_datagram['pitch'], + new_datagram['roll'], new_datagram['heading']) + + else: + # report an unknown datagram type + print("Unknown datagram type: " + str(new_datagram['type'])) + + # return our result dict which contains some basic info about + # the datagram we just read + return result + + + def get_channel_data(self, frequencies=None, channel_ids=None): + """returns a dict containing lists of raw_data objects for the specified channel IDs, + or frequencies. Channel number isn't part of the EK80 raw file specification. + + This method returns a dictionary of lists that contain references to one or + more raw_data objects containing the data that has been read. The dictionary keys + and the raw_data object(s) returned depend on the argument(s) passed. You can + specify one or more frequencies and/or channel IDs and the dictionary will be + keyed by frequency and/or channel ID and the key values will be a list containing + the raw_data objects that correspond to that frequency or channel ID. + + If you specify one or more frequencies, this method will return a dict, keyed + by frequency where the values are a lists containing one or more raw_data objects + associated with the specified frequency. The length of the list is equal to the + number of unique channel + datatype combinations sharing that transmit frequency. + + If you specify one or more channel IDs, this method will return a dict, keyed + by channel ID where the values are a lists containing one or more raw_data objects + associated with the specified channel ID. The length of the list is equal to the + number of unique channel + datatype combinations sharing that channel ID. + + If called without arguments, it returns a dictionary, keyed by channel ID, that + contains all channels read. The values are a lists containing one or more raw_data + objects associated with the specified channel ID just as if you called the method + specifying all channel IDs. + + You can specify multiple keyword arguments to return references to the data mapped + in multiple ways if your analysis requires that flexibility. This method only returns + references, it does not copy the data, so there is no real penalty for doing this. + Do remember that if there is a valid reference to an object in memory it will continue + to exist and consume RAM. If memory management is important for your application + you will want to manage these and not just grab a bunch of references willy-nilly. + + Args: + frequencies (float, list): Specify one or more frequencies in a list to return + data objects associated with that frequency. + channel_number (int, list): Specify one or more channel numbers in a list to + return data objects associated with that channel number. + Returns: + Dictionary keyed by frequency/channel_id/channel_number. Values are lists + of raw_data objects associated with the specified frequency/channel_id/ + channel_number. + """ + + # create the return dict + channel_data = {} + + if channel_ids is not None: + # Channel id is specified. + + # if we're passed a string, make it a list + if isinstance(channel_ids, str): + channel_ids = [channel_ids] + + # work thru the channel ids and add if they exist + for id in channel_ids: + channel_data[id] = self.raw_data.get(id, None) + + elif frequencies is not None: + # frequency is specified + + # if we're passed a number, make it a list + if not isinstance(frequencies, list): + frequencies = [frequencies] + + # and work through the freqs, adding if they exist + for freq in frequencies: + # There is not a 1:1 mapping of freqs to data so we have to + # create a list and then extend it for each matching frequency. + channel_data[freq] = [] + + data_objs = self.frequency_map.get(freq, [None]) + for id in data_objs: + channel_data[freq].extend(self.raw_data[id]) + + else: + # No keywords specified - return all in a dict keyed by channel ID + channel_data = self.raw_data + + return channel_data + + + def __str__(self): + """ + Reimplemented string method that provides some basic info about the + SEAT class instance's contents. + + Returns: Message to print when calling print method on SEAT class + instance object. + """ + + # Print the class and address. + msg = str(self.__class__) + " at " + str(hex(id(self))) + "\n" + + # Print some more info about the EK80 instance. + if self.channel_ids: + n_channels = len(self.channel_ids) + if n_channels > 1: + msg = msg + (" SEAT object contains data from " + str( + n_channels) + " channels:\n") + else: + msg = msg + (" SEAT object contains data from 1 channel:\n") + + for channel_id in self.channel_ids: + for raw in self.raw_data[channel_id]: + msg = msg + (" " + channel_id + " :: " + raw.data_type + " " + str(raw.shape) + "\n") + msg = msg + (" data start time: " + str(self.start_time) + "\n") + msg = msg + (" data end time: " + str(self.end_time) + "\n") + msg = msg + (" number of pings: " + str(self.end_ping - + self.start_ping + 1) + "\n") + + else: + msg = msg + (" SEAT object contains no data\n") + + return msg + +class raw_data(ping_data): + """ + the raw_data class contains a single channel's data extracted from a + Simrad SEAT raw file (same formate as ER60). + """ + + # Define some instrument specific constants. + + # Simrad recommends a TVG correction factor of 2 samples to compensate + # for receiver delay and TVG start time delay in EK60 and related + # hardware. Note that this correction factor is only applied when + # computing Sv/sv and not Sp/sp. + TVG_CORRECTION = 2 + + # Define constants used to specify the target resampling interval for the + # power and angle conversion functions. These values represent the + # standard sampling intervals for EK60 hardware when operated with the + # ER60 software as well as ES60/70 systems and the ME70(?). + RESAMPLE_SHORTEST = 0 + RESAMPLE_16 = 0.000016 + RESAMPLE_32 = 0.000032 + RESAMPLE_64 = 0.000064 + RESAMPLE_128 = 0.000128 + RESAMPLE_256 = 0.000256 + RESAMPLE_512 = 0.000512 + RESAMPLE_1024 = 0.001024 + RESAMPLE_2048 = 0.002048 + RESAMPLE_LONGEST = 1 + + # Create a constant to convert indexed power to power. + INDEX2POWER = (10.0 * np.log10(2.0) / 256.0) + + # Create a constant to convert from indexed angles to electrical angles. + INDEX2ELEC = 180.0 / 128.0 + + + def __init__(self, channel_id, n_pings=500, n_samples=-1, + rolling=False, store_power=True, + store_angles=True, max_sample_number=None): + """Creates a new, empty raw_data object. + + The raw_data class stores raw echosounder data from a single channel + recorded using the Simrad ER60 application. + + NOTE: power is *always* stored in log form. If you manipulate power + values directly, make sure they are stored in log form. + + The data arrays are not created upon instantiation. They will be created + when the first ping is appended using the append_ping method. + + If rolling is True, and both the n_pings and n_samples arguments are + provided, arrays of size (n_pings, n_samples) are created upon + instantiation. Otherwise, the data arrays are created + + + Args: + channel_id (str): The channel ID of channel whose data are stored + in this raw_data instance. + n_pings (int): Sets the fixed "width" of the arrays for rolling + arrays and if the object is not using rolling arrays it defines + the "width" of the allocation chunks when appending data. + n_samples (int): Sets the number of samples (rows) of the + arrays. Default value is -1 samples which means the arrays + will automatically resize to hold all of the sample data. + rolling (bool): True = arrays have fixed sizes of n_pings set when + the class is instantiated. When the n_pings + 1 ping is added + the array is rolled left, dropping oldest ping and adding newest. + store_power (bool): Boolean to control whether power data are + stored in this raw_data object. + store_angles (bool): Boolean to control whether angle data are + stored in this raw_data object. + max_sample_number (int): Integer specifying the maximum number of + samples that will be stored in this instance's data arrays. + """ + + # Initialize the superclass + super(raw_data, self).__init__() + + # Specify if data array size is fixed and the array data is rolled left + # if the array fills up (True) or if the arrays are expanded when + # necessary to hold additional data (False). + self.rolling_array = bool(rolling) + + # If rolling is set, ensure we have been passed n_samples + if self.rolling_array and n_samples < 1: + raise ValueError('The n_samples argument must be defined and ' + + 'greater than 0 when rolling == True.') + + # The channel ID is the unique identifier of the channel stored in + # the object. + self.channel_id = channel_id + + # Specify the horizontal size (columns) of the array allocation size. + self.chunk_width = n_pings + + # Keep note if we should store the power, angle, or complex data. + self.store_power = bool(store_power) + self.store_angles = bool(store_angles) + + # Max_sample_number can be set to an integer specifying the maximum + # number of samples that will be stored in the sample data arrays. + # Samples beyond this will be discarded. + self.max_sample_number = max_sample_number + + # data_type will be set to a string describing the type of sample data + # stored. This value is an empty string until the first raw datagram + # is added. At that point the data_type is set. A raw_data object can + # only store 1 type of data. The data types are: + # + # angle - contains angle data only + # power - contains power data only + # power/angle - contains power and angle data + self.data_type = '' + + # transceiver_type stores the hardware identifier of the transceiver + # used to collect the data. For EK60 on ER60 collection, this is always + # GPT. + self.transceiver_type = 'GPT' + + # motion_data stores a reference to the motion_data object created by + # the EK80 class when data are read. The motion_data object stores the + # heading, pitch, roll, and heave data recorded by the EK80 system if + # the motion sensor was installed. + self.motion_data = None + + # nmea_data stores a reference to the nmea_data object created by + # the EK80 class when data are read. The nmea_data object stores the + # asyncronous NMEA-0183 data input into the EK80 system such as GPS, + # Gyro, vessel distance log, etc. + self.nmea_data = None + + # Data_attributes is an internal list that contains the names of all + # of the class's "data" properties. The echolab2 package uses this + # attribute to generalize various functions that manipulate these + # data. Here we *extend* the list that is defined in the parent class. + self._data_attributes += ['configuration', + 'transducer_depth', + 'frequency', + 'transmit_power', + 'pulse_length', + 'bandwidth', + 'sample_interval', + 'sound_speed', + 'absorption_coefficient', + 'temperature', + 'transmit_mode', + 'sample_offset', + 'sample_count'] + + + # create a list that stores the scalar object attributes + self._obj_attributes = ['rolling_array', + 'chunk_width', + 'store_power', + 'store_angles', + 'max_sample_number', + 'data_type', + 'transceiver_type', + 'motion_data', + 'nmea_data'] + + + def empty_like(self, n_pings, empty_times=True, no_data=False): + """Returns raw_data object with data arrays filled with NaNs. + + The raw_data object has the same general characteristics of "this" + object, but with data arrays filled with NaNs. + + Args: + n_pings (int): Set n_pings to an integer specifying the number of + pings in the new object. The vertical axis (both number of + samples and depth/range values) will be the same as this object. + empty_times (bool) Set to True to return an object with ping times + set to NaT. Set to False to copy the ping times from this object. + """ + + # Create an instance of echolab2.EK60.raw_data and set the same basic + # properties as this object. Return the empty raw_data object. + empty_obj = raw_data(self.channel_id, n_pings=n_pings, + n_samples=self.n_samples, rolling=self.rolling_array, + store_power=self.store_power, store_angles=self.store_angles, + max_sample_number=self.max_sample_number) + + empty_obj.data_type = self.data_type + + return self._like(empty_obj, n_pings, np.nan, empty_times=empty_times, + no_data=no_data) + + + def copy(self): + """creates a deep copy of this object.""" + + # Create an empty raw_data object with the same basic props as this object. + rd_copy = raw_data(self.channel_id) + + # Call the parent _copy helper method and return the result. + return self._copy(rd_copy) + + + + def append_ping(self, sample_datagram, config_params,environment_datagram, start_sample=None, + end_sample=None): + """Adds a "pings" worth of data to the object. + + This method extracts data from the provided sample_datagram dict and + inserts it into the data arrays. Managing the data array sizes is the + bulk of what this method does: + + If the raw_data.rolling_array attribute is false, columns are added to + the data arrays as needed. To reduce overhead, the arrays are extended in + chunks, not on a ping by ping basis. If the recording range or the pulse + length changes requiring additional rows to be added, the data arrays will be + resized to accomodate the maximum number of samples being stored. Existing + samples are padded with NaNs as required. This vertical resize does not + occur in chunks and the data are copied each time samples are added. + This can have significant performance impacts in very specific cases. + + If raw_data.rolling_array is true, the data arrays are not resized but + the data within the arrays is "rolled" or shifted left and the column at + index 0 is moved to index n_pings - 1. Additional samples are discarded + and pings with fewer samples are padded with NaNs as required. + + This method is typically only called by the EK60 class when reading a raw file. + + Args: + sample_datagram (dict): The dictionary containing the parsed sample datagram. + config_datagram (dict): The dictionary containing the parsed XML configuration + datagram that goes with this sample data. + start_sample (int): The first sample to extract. Samples less than start_sample + will be discarded. Default (None) starts with the first + sample. + end_sample (int): The last sample to extract. Samples greater than end_sample + will be discarded. Default (None) ends with the last + sample. + """ + + # Set some defaults + power_samps = -1 + angle_samps = -1 + max_data_samples = [] + + # The contents of the first ping appended to a raw_data object defines + # the data_type and determines how the data arrays will be created. Also, + # since a raw_data object can only store one data_type, we disable saving + # of the other types. + if self.n_pings == -1: + + # Set defaults + create_power = False + create_angles = False + + # This must be a power, angle, or power/angle datagram + # Check if either or both are provided + if sample_datagram['power'] is not None: + create_power = True + power_samps = sample_datagram['power'].shape[0] + self.data_type = 'power' + + if sample_datagram['angle'] is not None: + create_angles = True + angle_samps = sample_datagram['angle'].shape[0] + if create_power: + self.data_type = 'power/angle' + else: + self.store_power = False + self.data_type = 'angle' + + # At this point if we're power, we're not storing angle data + if self.data_type == 'power': + self.store_angles = False + + # determine the initial number of samples in our arrays + if self.max_sample_number: + n_samples = self.max_sample_number + else: + n_samples = max([angle_samps, power_samps]) + + # set the initial sample offset + self.sample_offset = sample_datagram['offset'] + + # Initialize the data arrays + self._create_arrays(self.chunk_width, n_samples, initialize=False, + create_power=create_power, create_angles=create_angles) + + # Initialize the ping counter to indicate that our data arrays + # have been allocated. + self.n_pings = 0 + + # determine the number of samples in this datagram as well + # as the number of samples in our data array(s). + if self.store_angles: + angle_samps = sample_datagram['angle'].shape[0] + max_data_samples.append(self.angles_alongship_e.shape[1]) + max_data_samples.append(self.angles_athwartship_e.shape[1]) + if self.store_power: + power_samps = sample_datagram['power'].shape[0] + max_data_samples.append(self.power.shape[1]) + + max_data_samples = max(max_data_samples) + max_new_samples = max([power_samps, angle_samps]) + + # Check if we need to truncate the sample data. + if self.max_sample_number and (max_new_samples > self.max_sample_number): + max_new_samples = self.max_sample_number + if self.store_angles: + sample_datagram['angle'] = \ + sample_datagram['angle'][0:self.max_sample_number] + if self.store_power: + sample_datagram['power'] = \ + sample_datagram['power'][0:self.max_sample_number] + + # Create 2 variables to store our current array size. + ping_dims = self.ping_time.size + sample_dims = max_data_samples + + # Check if we need to re-size or roll our data arrays. + if self.rolling_array == False: + # Check if we need to resize our data arrays. + ping_resize = False + sample_resize = False + + # Check the ping dimension. + if self.n_pings == ping_dims: + # Need to resize the ping dimension. + ping_resize = True + # Calculate the new ping dimension. + ping_dims = ping_dims + self.chunk_width + + # Check the samples dimension. + if max_new_samples > max_data_samples: + # Need to resize the samples dimension. + sample_resize = True + # Calculate the new samples dimension. + sample_dims = max_new_samples + + # Determine if we resize. + if ping_resize or sample_resize: + # We call the parent method's resize here to avoid updating + # our n_pings attribute while reading data. + super(raw_data, self).resize(ping_dims, sample_dims) + #elf.resize(ping_dims, sample_dims) + + # Get an index into the data arrays for this ping and increment + # our ping counter. + this_ping = self.n_pings + self.n_pings += 1 + + else: + # Check if we need to roll. + if self.n_pings == ping_dims - 1: + # When a rolling array is "filled" we stop incrementing the + # ping counter and repeatedly append pings to the last ping + # index in the array. + this_ping = self.n_pings + + # Roll our array 1 ping. + self._roll_arrays(1) + + # Update the channel configuration dict's end_* values + config_params['end_ping'] = self.n_pings + config_params['end_time'] = sample_datagram['timestamp'] + + # Insert the config, environment, and filter object references for this ping. + self.configuration[this_ping] = config_params + self.environment[this_ping] = environment_datagram + + # Now insert the data into our numpy arrays. + self.ping_time[this_ping] = sample_datagram['timestamp'] + self.transducer_depth[this_ping] = sample_datagram['transducer_depth'] + self.frequency[this_ping] = sample_datagram['frequency'] + self.transmit_power[this_ping] = sample_datagram['transmit_power'] + self.pulse_length[this_ping] = sample_datagram['pulse_length'] + self.bandwidth[this_ping] = sample_datagram['bandwidth'] + self.sample_interval[this_ping] = sample_datagram['sample_interval'] + self.sound_speed[this_ping] = sample_datagram['sound_velocity'] + self.absorption_coefficient[this_ping] = sample_datagram['absorption_coefficient'] + self.temperature[this_ping] = sample_datagram['temperature'] + self.transmit_mode[this_ping] = sample_datagram['transmit_mode'] + + # Update sample count and sample offset values + if start_sample: + self.sample_offset[this_ping] = start_sample + sample_datagram['offset'] + + if end_sample: + self.sample_count[this_ping] = end_sample - start_sample + 1 + else: + self.sample_count[this_ping] = sample_datagram['count'] - \ + start_sample + else: + self.sample_offset[this_ping] = sample_datagram['offset'] + start_sample = 0 + if end_sample: + self.sample_count[this_ping] = end_sample + 1 + else: + self.sample_count[this_ping] = sample_datagram['count'] + end_sample = sample_datagram['count'] + + # Now store the "sample" data. + + # Check if we need to store power data. + if self.store_power: + if sample_datagram['power'].size > 0: + # Get the subset of samples we're storing. + power = sample_datagram['power'][start_sample:self.sample_count[this_ping]] + + # Convert the indexed power data to power dB. + power = power.astype(self.sample_dtype) * self.INDEX2POWER + + # Check if we need to pad or trim our sample data. + sample_pad = sample_dims - power.shape[0] + if sample_pad > 0: + # The data array has more samples than this datagram - we + # need to pad the datagram. + self.power[this_ping,:] = np.pad(power,(0,sample_pad), + 'constant', constant_values=np.nan) + elif sample_pad < 0: + # The data array has fewer samples than this datagram - we + # need to trim the datagram. + self.power[this_ping,:] = power[0:sample_pad] + else: + # The array has the same number of samples. + self.power[this_ping,:] = power + else: + # This is an empty ping + self.power[this_ping,:] = np.nan + + # Check if we need to store angle data. + if self.store_angles: + if sample_datagram['angle'].size > 0: + # Convert from indexed to electrical angles. + alongship_e = sample_datagram['angle'][start_sample:self.sample_count[this_ping], 1].astype(self.sample_dtype) + alongship_e *= self.INDEX2ELEC + athwartship_e = sample_datagram['angle'][start_sample:self.sample_count[this_ping], 0].astype(self.sample_dtype) + athwartship_e *= self.INDEX2ELEC + + # Check if we need to pad or trim our sample data. + sample_pad = sample_dims - athwartship_e.shape[0] + if sample_pad > 0: + # The data array has more samples than this datagram - we + # need to pad the datagram + self.angles_alongship_e[this_ping,:] = np.pad(alongship_e, + (0,sample_pad), 'constant', constant_values=np.nan) + self.angles_athwartship_e[this_ping,:] = np.pad( + athwartship_e,(0,sample_pad), 'constant', constant_values=np.nan) + elif sample_pad < 0: + # The data array has fewer samples than this datagram - we + # need to trim the datagram. + self.angles_alongship_e[this_ping,:] = alongship_e[0:sample_pad] + self.angles_athwartship_e[this_ping,:] = athwartship_e[0:sample_pad] + else: + # The array has the same number of samples. + self.angles_alongship_e[this_ping,:] = alongship_e + self.angles_athwartship_e[this_ping,:] = athwartship_e + + else: + # This is an empty ping + self.angles_alongship_e[this_ping,:] = np.nan + self.angles_athwartship_e[this_ping,:] = np.nan + + + def get_calibration(self, **kwargs): + """Returns a calibration object populated from the data contained in this + raw_data object. + + Calibration objects are passed to methods that transform the raw data + to more useful forms and provide those methods the parameters they require + such as sound speed, salinity, sampling interval, etc. The + + Args: + + absorption_method (str): Set to 'F&G' to use the method published by + Francois and Garrison to compute absorption. This is the default + in Echoview. You can also specify ‘A&M’ to use the method + published by Ainslie and McColm. + Default: 'F&G' + + Returns: + A ek60_calibration object containing values from the raw data. + """ + + # Create an empty ek60 calibration object + cal_obj = ek60_calibration(**kwargs) + + # Populate it using the data from "this" object + cal_obj.from_raw_data(self) + + return cal_obj + + + def get_power(self, calibration=None, **kwargs): + """Returns a processed data object that contains the power data. + + This method performs all of the required transformations to place the + raw power data into a rectangular array where all samples share the same + thickness and are correctly arranged relative to each other. + + This process happens in 3 steps: + + Data are resampled so all samples have the same thickness. + Data are shifted vertically to account for the sample offsets. + Data are then regridded to a fixed time, range grid. + + Each step is performed only when required. Calls to this method will + return much faster if the raw data share the same sample thickness, + offset and sound speed. + + If calibration is set to an instance of EK60.ek60_ calibration the + values in that object (if set) will be used when performing the + transformations required to return the results. If the required + parameters are not set in the calibration object or if no object is + provided, this method will extract these parameters from the raw file + data. + + Args: + + resample_interval (float): Set this to a float specifying the + sampling interval (in seconds) used when generating the + vertical axis for the return data. If the raw data sampling + interval is different than the specified interval, the raw + data will be resampled at the specified rate. 0 and 1 have + special meaning. If set to 0 or 1, the data will only be resampled + if the sampling interval changes. If it does change, when + set to 0, the data will be resampled to the shortest sampling + interval present in the data. If set to 1, it will be resampled + to the longest interval present in the data. + + The following constants are defined in the class: + + RESAMPLE_SHORTEST = 0 + RESAMPLE_16 = 0.000016 + RESAMPLE_32 = 0.000032 + RESAMPLE_64 = 0.000064 + RESAMPLE_128 = 0.000128 + RESAMPLE_256 = 0.000256 + RESAMPLE_512 = 0.000512 + RESAMPLE_1024 = 0.001024 + RESAMPLE_2048 = 0.002048 + RESAMPLE_LONGEST = 1 + + Default: RESAMPLE_SHORTEST + + return_indices (np.array uint32): Set this to a numpy array that contains + the index values to return in the processed data object. This can be + used for more advanced anipulations where start/end ping/time are + inadequate. + + calibration (EK60.ek60_calibration): Set to an instance of + EK60.ek60_calibration containing the calibration parameters + you want to use when transforming to Sv/sv. If no calibration + object is provided, the values will be extracted from the raw + data. + + start_time (datetime64): Set to a numpy datetime64 oject specifying + the start time of the data to convert. All data between the start + and end time will be returned. If set to None, the start time is + the first ping. + Default: None + + end_time (datetime64): Set to a numpy datetime64 oject specifying + the end time of the data to convert. All data between the start + and end time will be returned. If set to None, the end time is + the last ping. + Default: None + + start_ping (int): Set to an integer specifying the first ping number + to return. All pings between the start and end ping will be + returned. If set to None, the first ping is set as the start ping. + Default: None + + end_ping (int): Set to an integer specifying the end ping number + to return. All pings between the start and end ping will be + returned. If set to None, the last ping is set as the end ping. + Default: None + + Note that you can set a start/end time OR a start/end ping. If you + set both, one will be ignored. + + time_order (bool): Set to True to return data in time order. If + False, data will be returned in the order it was read. + Default: True + + Returns: + A processed_data object containing power data. + + """ + + # Check if user provided a cal object + if calibration is None: + # No - get one populated from raw data + calibration = self.get_calibration() + + # Call the _get_sample_data method requesting the appropriate sample attribute. + if hasattr(self, 'power'): + p_data, return_indices = self._get_sample_data('power', calibration=calibration, + **kwargs) + else: + raise AttributeError('raw_data object does not contain power ' + + 'data required to return, er, power.') + + # Set the data type. + p_data.data_type = 'power' + + # Set the is_log attribute and return it. + p_data.is_log = True + + return p_data + + + def _get_power(self, calibration=None, **kwargs): + """Returns a processed data object that contains the power data and + an index array. + + This method is identical to get_power except that it also returns an + index array that maps the pings in the processed_data object to the + same pings in the "this" object. This is used internally. + """ + + # Check if user provided a cal object + if calibration is None: + # No - get one populated from raw data + calibration = self.get_calibration() + + # Call the _get_sample_data method requesting the appropriate sample attribute. + if hasattr(self, 'power'): + p_data, return_indices = self._get_sample_data('power', calibration=calibration, + **kwargs) + else: + raise AttributeError('Raw data object does not contain power ' + + 'data required to return, er, power.') + + # Set the data type. + p_data.data_type = 'power' + + # Set the is_log attribute and return it. + p_data.is_log = True + + return p_data, return_indices + + + def get_sv(self, calibration=None, **kwargs): + """Gets sv data. + + This is a convenience method which simply calls get_Sv and forces + the linear keyword to True. + + Args: + See getSv for arguments. + + Returns: + A processed_data object containing sv. + + """ + + # Check if user provided a cal object + if calibration is None: + # No - get one populated from raw data + calibration = self.get_calibration() + + # Remove the linear keyword. + kwargs.pop('linear', None) + + # Call get_Sp forcing linear to True. + return self.get_Sv(linear=True, calibration=calibration, **kwargs) + + + def get_Sv(self, calibration=None, linear=False, tvg_correction=True, + return_depth=False, **kwargs): + """Gets Sv data + + This method returns a processed_data object containing Sv or sv data. + + This method performs all of the required transformations to place the + raw power data into a rectangular array where all samples share the same + thickness and are correctly arranged relative to each other. It then + computes Sv/sv as follows: + + Sv = power + 20 log10(Range) + (2 * alpha * Range) - (10 * ... + log10((TransmitPower * (10^(Gain/10))^2 * lambda^2 * ... + c * tau * 10^(psi/10)) / (32 * pi^2)) - (2 * SaCorrection) + + Args: + resample_interval (float): Set this to a float specifying the + sampling interval (in seconds) used when generating the + vertical axis for the return data. If the raw data sampling + interval is different than the specified interval, the raw + data will be resampled at the specified rate. 0 and 1 have + special meaning. If set to 0 or 1, the data will only be resampled + if the sampling interval changes. If it does change, when + set to 0, the data will be resampled to the shortest sampling + interval present in the data. If set to 1, it will be resampled + to the longest interval present in the data. + + The following constants are defined in the class: + + RESAMPLE_SHORTEST = 0 + RESAMPLE_16 = 0.000016 + RESAMPLE_32 = 0.000032 + RESAMPLE_64 = 0.000064 + RESAMPLE_128 = 0.000128 + RESAMPLE_256 = 0.000256 + RESAMPLE_512 = 0.000512 + RESAMPLE_1024 = 0.001024 + RESAMPLE_2048 = 0.002048 + RESAMPLE_LONGEST = 1 + + Default: RESAMPLE_SHORTEST + + return_indices (np.array uint32): Set this to a numpy array that contains + the index values to return in the processed data object. This can be + used for more advanced anipulations where start/end ping/time are + inadequate. + + calibration (EK60.ek60_calibration): Set to an instance of + EK60.ek60_calibration containing the calibration parameters + you want to use when transforming to Sv/sv. If no calibration + object is provided, the values will be extracted from the raw + data. + + linear (bool): Set to True if getting "sv" data + Default: False + + tvg_correction (bool): Set to True to apply TVG range correction. + Typically you want to leave this at True. + Default: True + + return_depth (bool): Set to True to return a processed_data object + with a depth axis. When False, the processed_data object has + a range axis. + Default: False + + start_time (datetime64): Set to a numpy datetime64 oject specifying + the start time of the data to convert. All data between the start + and end time will be returned. If set to None, the start time is + the first ping. + Default: None + + end_time (datetime64): Set to a numpy datetime64 oject specifying + the end time of the data to convert. All data between the start + and end time will be returned. If set to None, the end time is + the last ping. + Default: None + + start_ping (int): Set to an integer specifying the first ping number + to return. All pings between the start and end ping will be + returned. If set to None, the first ping is set as the start ping. + Default: None + + end_ping (int): Set to an integer specifying the end ping number + to return. All pings between the start and end ping will be + returned. If set to None, the last ping is set as the end ping. + Default: None + + Note that you can set a start/end time OR a start/end ping. If you + set both, one will be ignored. + + time_order (bool): Set to True to return data in time order. If + False, data will be returned in the order it was read. + Default: True + + Returns: + A processed_data object containing Sv (or sv if linear is True). + + """ + # If we're not given a cal object, create an empty one + if calibration is None: + calibration = self.get_calibration() #ek60_calibration() + + # Get the power data - this step also resamples and arranges the raw data. + p_data, return_indices = self._get_power(calibration=calibration, **kwargs) + + # Set the data type and is_log attribute. + if linear: + attribute_name = 'sv' + p_data.is_log = False + + else: + attribute_name = 'Sv' + p_data.is_log = True + p_data.data_type = attribute_name + + # Convert power to Sv/sv. + sv_data = self._convert_power(p_data, calibration, attribute_name, + linear, return_indices, tvg_correction) + + # Set the data attribute in the processed_data object. + p_data.data = sv_data + + # Check if we need to convert to depth + if return_depth: + p_data.to_depth() + + return p_data + + + def get_sp(self, **kwargs): + """Gets sp data. + + This method returns a processed_data object containing sp data. This is + a convenience method which simply calls get_Sp and forces the linear + keyword to True. + + See get_Sp for more detail. + + Args: + See get_Sp for argument descriptions. + + Returns: + returns a processed_data object containing sp + """ + + # Remove the linear keyword. + kwargs.pop('linear', None) + + # Call get_Sp, forcing linear to True. + return self.get_Sp(linear=True, **kwargs) + + + def get_Sp(self, calibration=None, linear=False, tvg_correction=False, + return_depth=False, **kwargs): + """Gets Sp data. + + This method returns a processed_data object containing Sp or sp data. + + This method performs all of the required transformations to place the + raw power data into a rectangular array where all samples share the same + thickness and are correctly arranged relative to each other. It then + computes Sp as follows: + + Sp = power + 40 * log10(Range) + (2 * alpha * Range) - (10 + * ... log10((TransmitPower * (10^(gain/10))^2 * lambda^2) / (16 * + pi^2))) + + Args: + resample_interval (float): Set this to a float specifying the + sampling interval (in seconds) used when generating the + vertical axis for the return data. If the raw data sampling + interval is different than the specified interval, the raw + data will be resampled at the specified rate. 0 and 1 have + special meaning. If set to 0 or 1, the data will only be resampled + if the sampling interval changes. If it does change, when + set to 0, the data will be resampled to the shortest sampling + interval present in the data. If set to 1, it will be resampled + to the longest interval present in the data. + + The following constants are defined in the class: + + RESAMPLE_SHORTEST = 0 + RESAMPLE_16 = 0.000016 + RESAMPLE_32 = 0.000032 + RESAMPLE_64 = 0.000064 + RESAMPLE_128 = 0.000128 + RESAMPLE_256 = 0.000256 + RESAMPLE_512 = 0.000512 + RESAMPLE_1024 = 0.001024 + RESAMPLE_2048 = 0.002048 + RESAMPLE_LONGEST = 1 + + Default: RESAMPLE_SHORTEST + + return_indices (np.array uint32): Set this to a numpy array that contains + the index values to return in the processed data object. This can be + used for more advanced anipulations where start/end ping/time are + inadequate. + + calibration (EK60.ek60_calibration): Set to an instance of + EK60.ek60_calibration containing the calibration parameters + you want to use when transforming to Sv/sv. If no calibration + object is provided, the values will be extracted from the raw + data. + + linear (bool): Set to True if getting "sv" data + Default: False + + tvg_correction (bool): Set to True to apply TVG range correction. + By default, TVG range correction is not applied to the data. This + results in output that is consistent with the Simrad "P" telegram + and TS data exported from Echoview version 4.3 and later (prior + versions applied the correction by default). + + If you intend to perform single target detections you must apply + the TVG range correction at some point in your process. This can + be done by either setting the tvgCorrection keyword of this function + or it can be done as part of your single target detection routine. + Default: False + + return_depth (bool): Set to True to return a processed_data object + with a depth axis. When False, the processed_data object has + a range axis. + Default: False + + start_time (datetime64): Set to a numpy datetime64 oject specifying + the start time of the data to convert. All data between the start + and end time will be returned. If set to None, the start time is + the first ping. + Default: None + + end_time (datetime64): Set to a numpy datetime64 oject specifying + the end time of the data to convert. All data between the start + and end time will be returned. If set to None, the end time is + the last ping. + Default: None + + start_ping (int): Set to an integer specifying the first ping number + to return. All pings between the start and end ping will be + returned. If set to None, the first ping is set as the start ping. + Default: None + + end_ping (int): Set to an integer specifying the end ping number + to return. All pings between the start and end ping will be + returned. If set to None, the last ping is set as the end ping. + Default: None + + Note that you can set a start/end time OR a start/end ping. If you + set both, one will be ignored. + + time_order (bool): Set to True to return data in time order. If + False, data will be returned in the order it was read. + Default: True + + Returns: + A processed_data object containing Sp (or sp if linear is True). + + """ + # Check if user provided a cal object + if calibration is None: + # No - get one populated from raw data + calibration = self.get_calibration() + + # Get the power data - this step also resamples and arranges the raw data. + p_data, return_indices = self._get_power(calibration=calibration, **kwargs) + + # Set the data type. + if linear: + attribute_name = 'sp' + p_data.is_log = False + else: + attribute_name = 'Sp' + p_data.is_log = True + p_data.data_type = attribute_name + + # Convert + sp_data = self._convert_power(p_data, calibration, attribute_name, + linear, return_indices, tvg_correction) + + # Set the data attribute in the processed_data object. + p_data.data = sp_data + + # Check if we need to convert to depth or heave correct. + if return_depth: + p_data.to_depth() + + return p_data + + + def get_bottom(self, calibration=None, return_indices=None, **kwargs): + """Gets a echolab2 line object containing the sounder detected bottom + depths. + + The sounder detected bottom depths are computed using the sound speed + setting at the time of recording. If you are applying a different sound + speed setting via the calibration argument when getting the converted + sample data, you must also pass the same calibration object to this method + to ensure that the sounder detected bottom depths align with your sample + data. + + Also, if you request sample data on a depth grid, you should set the + return_depth keyword here too. + + Args: + return_indices (np.array uint32): Set this to a numpy array that contains + the index values to return in the processed data object. This can be + used for more advanced anipulations where start/end ping/time are + inadequate. + + calibration (EK60.ek60_calibration): Set to an instance of + EK60.ek60_calibration containing the calibration parameters + you want to use when transforming to Sv/sv. If no calibration + object is provided, the values will be extracted from the raw + data. + + start_time (datetime64): Set to a numpy datetime64 oject specifying + the start time of the data to convert. All data between the start + and end time will be returned. If set to None, the start time is + the first ping. + Default: None + + end_time (datetime64): Set to a numpy datetime64 oject specifying + the end time of the data to convert. All data between the start + and end time will be returned. If set to None, the end time is + the last ping. + Default: None + + start_ping (int): Set to an integer specifying the first ping number + to return. All pings between the start and end ping will be + returned. If set to None, the first ping is set as the start ping. + Default: None + + end_ping (int): Set to an integer specifying the end ping number + to return. All pings between the start and end ping will be + returned. If set to None, the last ping is set as the end ping. + Default: None + + Note that you can set a start/end time OR a start/end ping. If you + set both, one will be ignored. + + time_order (bool): Set to True to return data in time order. If + False, data will be returned in the order it was read. + Default: True + + Raises: + ValueError: The return indices exceed the number of pings in + the raw data object + + Returns: + A line object containing the sounder detected bottom depths. + + """ + + # Check if the user supplied an explicit list of indices to return. + if isinstance(return_indices, np.ndarray): + if max(return_indices) > self.ping_time.shape[0]: + raise ValueError("One or more of the return indices provided " + + "exceeds the number of pings in the raw_data object") + else: + # Get an array of index values to return. + return_indices = self.get_indices(**kwargs) + + # Check if user provided a cal object + if calibration is None: + # No - create an empty one - all cal values will come from the raw data + calibration = ek60_calibration() + + # Extract the recorded sound velocity and transducer offset + sv_recorded = self.sound_speed[return_indices] + offset_recorded = self.transducer_depth[return_indices] + + # Get the calibration params required for detected depth conversion. + cal_parms = {'sound_speed':None, + 'transducer_depth':None} + + # Next, iterate through the dict, calling the method to extract the + # values for each parameter. + for key in cal_parms: + cal_parms[key] = calibration.get_parameter(self, key, + return_indices) + + # Check if we have to adjust the depth due to a change in sound speed. + if np.all(np.isclose(sv_recorded, cal_parms['sound_speed'])): + converted_depths = self.detected_bottom[return_indices] + else: + cf = cal_parms['sound_speed'].astype('float') / sv_recorded + converted_depths = cf * self.detected_bottom[return_indices] + + # Check if we have to adjust the depth due to a change in transducer draft + if not np.all(np.isclose(offset_recorded, cal_parms['transducer_depth'])): + converted_depths = converted_depths + (cal_parms['transducer_depth']- + offset_recorded) + + # Create a line object to return with our adjusted data. + bottom_line = line.line(ping_time=self.ping_time[return_indices], + data=converted_depths, **kwargs) + + # Get the transducer offset attribute. + xdcr_draft = cal_parms['transducer_depth'][return_indices] + + # Get the heave attribute if available and add to the xdcr draft + _, heave_data = self.motion_data.interpolate(bottom_line, 'heave') + if heave_data['heave'] is not None: + xdcr_draft += heave_data['heave'] + + # and then add the transducer_draft attribute to our line + bottom_line.add_data_attribute('transducer_draft', xdcr_draft) + + return bottom_line + + + def get_physical_angles(self, calibration=None, **kwargs): + """Gets the alongship and athwartship angle data. + + This method returns an tuple of processed data objects (alongship, + athwartship) containing the physical angle data. + + This method performs all of the required transformations to place the + raw electrical angle data into rectangular arrays where all samples + share the same thickness and are correctly arranged relative to each other. + It then transform the electrical angles into physical angles. + + Args: + return_indices (np.array uint32): Set this to a numpy array that contains + the index values to return in the processed data object. This can be + used for more advanced anipulations where start/end ping/time are + inadequate. + + calibration (EK60.ek60_calibration): Set to an instance of + EK60.ek60_calibration containing the calibration parameters + you want to use when transforming to Sv/sv. If no calibration + object is provided, the values will be extracted from the raw + data. + + return_depth (bool): Set to True to return a line object with a + with a depth axis. When False, the line object has a range axis. + Default: False + + start_time (datetime64): Set to a numpy datetime64 oject specifying + the start time of the data to convert. All data between the start + and end time will be returned. If set to None, the start time is + the first ping. + Default: None + + end_time (datetime64): Set to a numpy datetime64 oject specifying + the end time of the data to convert. All data between the start + and end time will be returned. If set to None, the end time is + the last ping. + Default: None + + start_ping (int): Set to an integer specifying the first ping number + to return. All pings between the start and end ping will be + returned. If set to None, the first ping is set as the start ping. + Default: None + + end_ping (int): Set to an integer specifying the end ping number + to return. All pings between the start and end ping will be + returned. If set to None, the last ping is set as the end ping. + Default: None + + Note that you can set a start/end time OR a start/end ping. If you + set both, one will be ignored. If you provide an index array, both + ping and time bounds will be ignored. + + time_order (bool): Set to True to return data in time order. If + False, data will be returned in the order it was read. + Default: True + + Returns: + Two rocessed_data objects with alongship and athwartship angle data. + """ + # Check if user provided a cal object + if calibration is None: + # No - get one populated from raw data + calibration = self.get_calibration() + + # Get the electrical angles. + alongship, athwartship, return_indices = \ + self._get_electrical_angles(calibration=calibration, **kwargs) + + # Get the calibration params required for angle conversion. + cal_parms = {'angle_sensitivity_alongship':None, + 'angle_sensitivity_athwartship':None, + 'angle_offset_alongship':None, + 'angle_offset_athwartship':None} + + # Next, iterate through the dict, calling the method to extract the + # values for each parameter. + for key in cal_parms: + cal_parms[key] = calibration.get_parameter(self, key, + return_indices) + + # Compute the physical angles. + alongship.data /= cal_parms['angle_sensitivity_alongship'][:, np.newaxis] + alongship.data -= cal_parms['angle_offset_alongship'][:, np.newaxis] + athwartship.data /= cal_parms['angle_sensitivity_athwartship'][:, np.newaxis] + athwartship.data -= cal_parms['angle_offset_athwartship'][:, np.newaxis] + + # Set the data types. + alongship.data_type = 'angles_alongship' + athwartship.data_type = 'angles_athwartship' + + # We do not need to convert to depth here since the electrical_angle + # data will already have been converted to depth if requested. + + return alongship, athwartship + + + def get_electrical_angles(self, return_depth=False, calibration=None, **kwargs): + """Gets unconverted angle data. + + This method returns a tuple of processed_data objects containing angle data + as electrical angles. + + See get_physical_angles for more detail. + + Args: + See get_physical_angles for argument descriptions. + + Returns: + Two processed data objects containing the unconverted + angles_alongship_e and angles_athwartship_e data. + """ + # Check if user provided a cal object + if calibration is None: + # No - get one populated from raw data + calibration = self.get_calibration() + + # Call the _get_sample_data method requesting the 'angles_alongship_e' + # sample attribute. The method will return a reference to a newly created + # processed_data object. + alongship, return_indices = self._get_sample_data('angles_alongship_e', + calibration=calibration, **kwargs) + + # Repeat for the athwartship data. + athwartship, return_indices = self._get_sample_data('angles_athwartship_e', + calibration=calibration, **kwargs) + + # Set the data type. + alongship.data_type = 'angles_alongship_e' + athwartship.data_type = 'angles_athwartship_e' + + # Covert to depth + if return_depth: + alongship.to_depth() + athwartship.to_depth() + + return alongship, athwartship + + + def resize(self, new_ping_dim, new_sample_dim): + """Resizes raw_data object. + + This method re-implements ping_data.resize, updating n_pings + attribute after resizing. + + Args: + new_ping_dim (int): Used to resize the sample data array + (horizontal axis). + new_sample_dim (int): Used to resize the sample data array + (vertical axis). + """ + # Call the parent method to resize the arrays (n_samples is updated here). + super(raw_data, self).resize(new_ping_dim, new_sample_dim) + + # Update n_pings. + self.n_pings = self.ping_time.shape[0] + + + def match_pings(self, other_data, match_to='cs'): + """Matches the ping times in this object to the ping times in the EK60.raw_data + object provided. It does this by matching times, inserting and/or deleting + pings as needed. It does not interpolate. Ping times in the other object + that aren't in this object are inserted. Ping times in this object that + aren't in the other object are deleted. If the time axes do not intersect + at all, all of the data in this object will be deleted and replaced with + empty pings for the ping times in the other object. + + + Args: + other_data (ping_data): A ping_data type object that this object + will be matched to. + + match_to (str): Set to a string defining the precision of the match. + + cs : Match to a 100th of a second + ds : Match to a 10th of a second + s : Match to the second + + Returns: + A dictionary with the keys 'inserted' and 'removed' containing the + indices of the pings inserted and removed. + """ + return super(raw_data, self).match_pings(other_data, match_to='cs') + + + def _get_electrical_angles(self, return_depth=False, calibration=None, + **kwargs): + """Retrieves unconverted angles_alongship_e and angles_athwartship_e + data and creates an index array mapping pings. + + _get_electrical_angles is identical to get_electrical_angles except + that it also returns an index array that maps the pings in the + processed_data object to the same pings in the "this" object. This is + used internally. + + Args: + return_depth (bool): If true, return the vertical axis of the + data as depth. Otherwise, return as range. + calibration (calibration object): The data calibration object where + calibration data will be retrieved. + **kwargs + + Returns: + Two processed data objects containing the unconverted + angles_alongship_e and angles_athwartship_e data and an index array + mapping pings to this object. + """ + # Check if user provided a cal object + if calibration is None: + # No - get one populated from raw data + calibration = self.get_calibration() + + # Call the _get_sample_data method requesting the 'angles_alongship_e' + # sample attribute. The method will return a reference to a newly created + # processed_data object. + if hasattr(self, 'angles_alongship_e') or hasattr(self, 'angles_athwartship_e'): + if hasattr(self, 'angles_alongship_e'): + alongship, return_indices = self._get_sample_data('angles_alongship_e', + calibration=calibration, **kwargs) + else: + raise AttributeError('Raw data object does not contain the ' + + 'angles_alongship_e attribute required to return angle data.') + + # use the already computed return_indices from the first + # call to _get_sample_data to get the athwartship data + if hasattr(self, 'angles_athwartship_e'): + kwargs.pop('return_indices', None) + athwartship, ri = self._get_sample_data('angles_athwartship_e', + return_indices=return_indices, calibration=calibration, **kwargs) + else: + raise AttributeError('Raw data object does not contain the ' + + 'angles_athwartship_e attribute required to return angle data.') + + else: + # We don't have complex nor electrical angle data required so + # we can't do anything. + raise AttributeError('Raw data object does not contain the ' + + 'data required to return angle data.') + + # Set the data type. + alongship.data_type = 'angles_alongship_e' + athwartship.data_type = 'angles_athwartship_e' + + # Apply depth and/or heave correction + if return_depth: + alongship.to_depth() + athwartship.to_depth() + + return alongship, athwartship, return_indices + + + def is_fm(self): + '''Convenience method that returns True if the raw data contains FM pings. + This method exists for API parity with the EK80 class. + ''' + + # EK60 hardware is CW only + return False + + + def is_cw(self): + '''Convenience method that returns True if the raw data contains CW pings. + This method exists for API parity with the EK80 class. + ''' + # EK60 hardware is CW only + return True + + + def get_frequency(self, unique=False): + '''Convenience method that returns the frequency of the transmit signals + of the data stored in the raw_data object. For the EK60 this simply + returns the frequency attribute. + + This method has the same interface as the EK80.get_frequency() method. + + ''' + + if unique: + return np.unique(self.frequency) + else: + return self.frequency + + + def _get_sample_data(self, property_name, calibration=None, + resample_interval=RESAMPLE_SHORTEST, return_indices=None, + **kwargs): + """Retrieves sample data. + + This method returns a processed data object that contains the + sample data from the property name provided. It performs all of the + required transformations to place the data into a rectangular + array where all samples in all pings share the same thickness and are + correctly arranged relative to each other. + + This process happens in 3 steps: + + Data are resampled so all samples have the same sampling interval. + Data are shifted vertically to account for the sample offsets. + Data are then interpolated to a common sound velocity + + Each step is performed only when required. Calls to this method will + return much faster if the raw data share the same sample thickness, + offset and sound speed. + + If calibration is set to an instance of an ek60_calibration object + the values in that object (if set) will be used when performing the + transformations required to return the results. If the required + parameters are not set in the calibration object or if no object is + provided, this method will extract these parameters from the raw data. + + Args: + property_name (str): The attribute name of the sample data + to be returned. For the EK60 the sample data attributes + are 'complex', 'power', 'angles_alongship_e', and + 'angles_athwartship_e' and the available attributes will + depend on how the data were collected and stored. + + calibration (ek60_calibration object): The calibration object where + calibration data will be retrieved. If this is set to None, + calibration data will be directly extracted from the raw data. + + resample_interval (float): The echosounder sampling interval used to + define the vertical position of the samples. If data was collected + with a different sampling interval it will be resampled to + the specified interval. The default behavior is to resample + to the shortest sampling interval in the data. This value has + no effect when all data share the same sampling interval. + + return_indices (array): A numpy array of indices to return. + + Raises: + ValueError: Return indices exceeds the number of pings. + AttributeError: The attribute name doesn't exist. + + Returns: + The processed data object containing the sample data. + """ + + def get_range_vector(num_samples, sample_interval, sound_speed, + sample_offset): + """ + get_range_vector returns a NON-CORRECTED range vector. + """ + # Calculate the thickness of samples with this sound speed. + thickness = sample_interval * sound_speed / 2.0 + # Calculate the range vector. + #range = (np.arange(0, num_samples) + sample_offset) * thickness + range = (np.arange(-1, num_samples-1) + sample_offset) * thickness + + return range + + # Check if the user supplied an explicit list of indices to return. + if isinstance(return_indices, np.ndarray): + if max(return_indices) > self.ping_time.shape[0]: + raise ValueError("One or more of the return indices provided " + "exceeds the " + "number of pings in the " + "raw_data object") + else: + # Get an array of index values to return. + return_indices = self.get_indices(**kwargs) + + # Create the processed_data object we will return. + p_data = processed_data(self.channel_id, + self.get_frequency(unique=True)[0], None) + + # Populate it with time and ping number. + p_data.ping_time = self.ping_time[return_indices].copy() + + # Get a reference to the data we're operating on. + if hasattr(self, property_name): + data = getattr(self, property_name) + else: + raise AttributeError("The attribute name " + property_name + + " does not exist.") + + # Populate the calibration parameters required for this method. + # First, create a dict with key names that match the attributes names + # of the calibration parameters we require for this method. + cal_parms = {'sample_interval':None, + 'sound_speed':None, + 'sample_offset':None, + 'pos_z':None, + 'transducer_depth':None} + + # Next, iterate through the dictionary calling the method to extract + # the values for each parameter. + for key in cal_parms: + cal_parms[key] = calibration.get_parameter(self, key, + return_indices) + + # Check if we have multiple sample offset values and get the minimum. + unique_sample_offsets = np.unique( + cal_parms['sample_offset'][~np.isnan(cal_parms['sample_offset'])]) + min_sample_offset = min(unique_sample_offsets) + + # Check if we need to resample our sample data. + unique_sample_interval = np.unique( + cal_parms['sample_interval'][~np.isnan(cal_parms['sample_interval'])]) + if unique_sample_interval.shape[0] > 1: + # There are at least 2 different sample intervals in the data. We + # must resample the data. We'll deal with adjusting sample offsets + # here too. + (output, sample_interval) = self._vertical_resample(data[return_indices], + cal_parms['sample_interval'], unique_sample_interval, resample_interval, + cal_parms['sample_offset'], min_sample_offset, + is_power=property_name == 'power') + else: + # We don't have to resample, but check if we need to shift any + # samples based on their sample offsets. + if unique_sample_offsets.shape[0] > 1: + # We have multiple sample offsets so we need to shift some of + # the samples. + output = self._vertical_shift(data[return_indices], + cal_parms['sample_offset'], unique_sample_offsets, + min_sample_offset) + else: + # The data all have the same sample intervals and sample + # offsets. Simply copy the data as is. + output = data[return_indices].copy() + + # Get the sample interval value to use for range conversion below. + sample_interval = unique_sample_interval[0] + + # Check if we have a fixed sound speed. + unique_sound_velocity = np.unique(cal_parms['sound_speed'][~np.isnan(cal_parms['sound_speed'])]) + if unique_sound_velocity.shape[0] > 1: + # There are at least 2 different sound speeds in the data or + # provided calibration data. Interpolate all data to the most + # common range (which is the most common sound speed). + sound_velocity = None + n = 0 + for speed in unique_sound_velocity: + # Determine the sound speed with the most pings. + if np.count_nonzero(cal_parms['sound_speed'] == speed) > n: + sound_velocity = speed + + # Calculate the target range. + range = get_range_vector(output.shape[1], sample_interval, + sound_velocity, min_sample_offset) + + # Now interpolate the samples for pings with different sound speeds + for speed in unique_sound_velocity: + # Only interpolate the "other" speeds + if speed != sound_velocity: + + # Get an array of indexes in the output array to interpolate. + pings_to_interp = np.where(cal_parms['sound_speed'] == speed)[0] + + # Compute this data's range + resample_range = get_range_vector(output.shape[1], sample_interval, + cal_parms['sound_speed'][pings_to_interp[0]], min_sample_offset) + + # Interpolate samples to target range + interp_f = interp1d(resample_range, output[pings_to_interp,:], axis=1, + bounds_error=False, fill_value=np.nan, assume_sorted=True) + output[pings_to_interp,:] = interp_f(resample_range) + + else: + # We have a fixed sound speed and only need to calculate a single + # range vector. + sound_velocity = unique_sound_velocity[0] + range = get_range_vector(output.shape[1], sample_interval, + sound_velocity, min_sample_offset) + + # Assign the results to the "data" processed_data object. + p_data.add_data_attribute('data', output) + + # Calculate the sample thickness. + sample_thickness = sample_interval * sound_velocity / 2.0 + + # Now assign range, sound_velocity, sample thickness and offset to + # the processed_data object. + p_data.add_data_attribute('range', range) + p_data.sound_speed = sound_velocity + p_data.sample_thickness = sample_thickness + p_data.sample_offset = min_sample_offset + p_data.sample_interval = sample_interval + + # Add the transducer offset attribute + try: + xdcr_offset = cal_parms['pos_z'] + cal_parms['transducer_depth'] + p_data.add_data_attribute('transducer_offset', xdcr_offset) + except: + pass + + # Add the heave attribute is available. + #_, heave_data = self.motion_data.interpolate(p_data, 'heave') + #if heave_data['heave'] is not None: + # p_data.add_data_attribute('heave', heave_data['heave']) + + + # Return the processed_data object containing the requested data. + return p_data, return_indices + + + def _convert_power( + self, power_data, calibration, convert_to, linear, + return_indices, tvg_correction): + """Converts power to Sv/sv/Sp/sp + + Args: + power_data (processed_data): A processed_data object with the raw power + data read from the file. + calibration (calibration object): The data calibration object where + calibration data will be retrieved. + convert_to (str): A string that specifies what to convert the + power to. Possible values are: Sv, sv, Sp, or sp. + linear (bool): Set to True to return linear values. + return_indices (array): A numpy array of indices to return. + tvg_correction (bool): Set to True to apply a correction to the + range of 2 * sample thickness. + + Returns: + An array with the converted data. + """ + + # Populate the calibration parameters required for this method. + # First, create a dictionary with key names that match the attribute + # names of the calibration parameters we require for this method. + cal_parms = {'gain':None, + 'transmit_power':None, + 'equivalent_beam_angle':None, + 'pulse_length':None, + 'absorption_coefficient':None, + 'sa_correction':None} + + # Next, iterate through the dictionary, calling the method to extract + # the values for each parameter. + for key in cal_parms: + cal_parms[key] = calibration.get_parameter(self, key, + return_indices) + + # Get sound_speed from the power data since get_power might have + # manipulated this value. Remember that we're operating on a + # processed_data object so all pings share the same sound speed. + cal_parms['sound_speed'] = np.empty((return_indices.shape[0]), + dtype=self.sample_dtype) + cal_parms['sound_speed'].fill(power_data.sound_speed) + + # For EK60 hardware use pulse duration when computing gains + effective_pulse_duration = cal_parms['pulse_length'] + + # Calculate the system gains. + wavelength = cal_parms['sound_speed'] / power_data.frequency + if convert_to in ['sv','Sv']: + gains = 10 * np.log10((cal_parms['transmit_power'] * (10**( cal_parms['gain'] / 10.0))**2 * + wavelength**2 * cal_parms['sound_speed'] * effective_pulse_duration * + 10**(cal_parms['equivalent_beam_angle']/10.0)) / (32 * np.pi**2)) + else: + gains = 10 * np.log10((cal_parms['transmit_power'] * (10**( + cal_parms['gain']/10.0))**2 * wavelength**2) / (16 * np.pi**2)) + + # Get the range for TVG calculation. + c_range = np.zeros(power_data.shape, dtype=power_data.sample_dtype) + c_range += power_data.range + c_range += power_data.sample_thickness + + if tvg_correction: + # For the Ex60 hardware, the corrected range is computed as: + # c_range = range - (2 * sample_thickness) + c_range -= (2.0 * power_data.sample_thickness) + + # zero out negative ranges + c_range[c_range < 0] = 0 + + # Calculate time varied gain. + tvg = c_range.copy() + tvg[tvg < 1] = 1 + if convert_to in ['sv','Sv']: + tvg = 20.0 * np.log10(tvg) + else: + tvg = 40.0 * np.log10(tvg) + + # Calculate absorption - our starting point. + data = (2.0 * cal_parms['absorption_coefficient'])[:,np.newaxis] * c_range + + # Add in power and TVG. + data += power_data.data + data += tvg + + # Subtract the applied gains. + data -= gains[:, np.newaxis] + + # Apply sa correction for Sv/sv. + if convert_to in ['sv','Sv']: + data -= (2.0 * cal_parms['sa_correction'])[:, np.newaxis] + + # Check if we're returning linear or log values. + if linear: + # Convert to linear units. + data[:] = 10 ** (data / 10.0) + + # Return the result. + return data + + + def _create_arrays(self, n_pings, n_samples, initialize=False, create_power=False, + create_angles=False): + """Initializes raw_data data arrays. + + This is an internal method. Note that all arrays must be numpy arrays. + + Args: + n_pings (int): Number of pings. + n_samples (int): Number of samples. + initialize (bool): Set to True to initialize arrays. + create_power (bool): Set to True to create the power attribute. + create_angles (bool): Set to True to create the angles_alongship_e + angles_athwartship_e attributes. + """ + + # First, create uninitialized arrays. + self.ping_time = np.empty((n_pings), dtype='datetime64[ms]') + + # create the arrays that contain references to the async data objects + self.configuration = np.empty((n_pings), dtype='object') + self.environment = np.empty((n_pings), dtype='object') + # the rest of the arrays store syncronous ping data + self.transducer_depth = np.empty((n_pings), np.float32) + self.frequency = np.empty((n_pings), np.float32) + self.transmit_power = np.empty((n_pings), np.float32) + self.pulse_length = np.empty((n_pings), np.float32) + self.bandwidth = np.empty((n_pings), np.float32) + self.sample_interval = np.empty((n_pings), np.float32) + self.sound_speed = np.empty((n_pings), np.float32) + self.absorption_coefficient = np.empty((n_pings), np.float32) + self.temperature = np.empty((n_pings), np.float32) + self.transmit_mode = np.empty((n_pings), np.uint8) + self.sample_offset = np.empty((n_pings), np.uint32) + self.sample_count = np.empty((n_pings), np.uint32) + + + # and the 2d sample data arrays + if create_power and self.store_power: + self.power = np.empty((n_pings, n_samples), + dtype=self.sample_dtype, order='C') + self.n_samples = n_samples + self._data_attributes.append('power') + if create_angles and self.store_angles: + self.angles_alongship_e = np.empty((n_pings, n_samples), + dtype=self.sample_dtype, order='C') + self._data_attributes.append('angles_alongship_e') + self.angles_athwartship_e = np.empty((n_pings, n_samples), + dtype=self.sample_dtype, order='C') + self._data_attributes.append('angles_athwartship_e') + self.n_samples = n_samples + + # update our shape attribute + self._shape() + + # Check if we should initialize them. + if initialize: + self.ping_time.fill(np.datetime64('NaT')) + self.transducer_depth.fill(np.nan) + self.frequency.fill(np.nan) + self.transmit_power.fill(np.nan) + self.pulse_length.fill(np.nan) + self.bandwidth.fill(np.nan) + self.sample_interval.fill(np.nan) + self.sound_speed.fill(np.nan) + self.absorption_coefficient.fill(np.nan) + self.temperature.fill(np.nan) + self.transmit_mode.fill(0) + self.sample_offset.fill(0) + self.sample_count.fill(0) + + if create_angles and self.store_power: + self.power.fill(np.nan) + if create_angles and self.store_angles: + self.angles_alongship_e.fill(np.nan) + self.angles_athwartship_e.fill(np.nan) + + + def __str__(self): + """ + Reimplemented string method that provides some basic info about the + raw_data object. + """ + + # Print the class and address. + msg = str(self.__class__) + " at " + str(hex(id(self))) + "\n" + + # Print some more info about the EK60.raw_data instance. + n_pings = len(self.ping_time) + if n_pings > 0: + msg = msg + " channel(s): " + self.channel_id + "\n" + msg = msg + " frequency (first ping): " + str( + self.frequency[0]) + " Hz\n" + msg = msg + " pulse length (first ping): " + str( + self.pulse_length[0] * 1000) + " ms\n" + msg = msg + " data start time: " + str( + self.ping_time[0]) + "\n" + msg = msg + " data end time: " + str( + self.ping_time[n_pings-1]) + "\n" + msg = msg + " number of pings: " + str(n_pings) + "\n" + if hasattr(self, 'power'): + n_pings,n_samples = self.power.shape + msg = msg + (" power array dimensions: (" + str(n_pings) + + "," + str(n_samples) + ")\n") + if hasattr(self, 'angles_alongship_e'): + n_pings,n_samples = self.angles_alongship_e.shape + msg = msg + (" angle array dimensions: (" + str(n_pings) + + "," + str(n_samples) + ")\n") + else: + msg = msg + " raw_data object contains no data\n" + + return msg + + +class ek60_calibration(calibration): + """ + The calibration class contains parameters required for transforming power + and electrical angle data to Sv/sv, Sp/sp and physical angles. + + When converting raw data to power, Sv/sv, Sp/sp, or to physical angles + you have the option of passing a calibration object containing the data + you want used during these conversions. To use this object you create an + instance and populate the attributes with your calibration data. + + You can provide the data in 2 forms: + As a scalar - the single value will be used for all pings. + As a vector - a vector of values as long as the number of pings + in the raw_data object where the first value will be used + with the first ping, the second with the second, and so on. + + If you set any attribute to None, that attribute's values will be obtained + from the raw_data object which contains the value at the time of recording. + If you do not pass a calibration object to the conversion methods *all* + of the cal parameter values will be extracted from the raw_data object. + """ + + def __init__(self): + '''Create an instance of an ek60_calibration object. + + ''' + + # call the parent init + super(ek60_calibration, self).__init__() + + # these attributes are properties of the raw_data class + self._raw_attributes = ['transducer_depth', 'frequency', 'transmit_power', 'pulse_length', + 'bandwidth' ,'sample_interval' ,'sound_speed' ,'absorption_coefficient', 'temperature', + 'transmit_mode','sample_offset','sample_count'] + self._init_attributes(self._raw_attributes) + + # these attributes are found in the configuration datagram + self._config_attributes = ['transect_name', 'transducer_serial_number' ,'survey_name', 'sounder_name','pulse_duration', + 'channel_id', 'transducer_beam_type', 'frequency', 'gain', 'sa_correction','equivalent_beam_angle', + 'beam_width_alongship', 'beam_width_athwartship', 'angle_sensitivity_alongship', + 'angle_sensitivity_athwartship', 'angle_offset_alongship', 'angle_offset_athwartship', + 'transducer_offset_x', 'transducer_offset_y', 'transducer_offset_z', 'dir_x', 'dir_y','dir_z', + 'pulse_length_table','gain_table','sa_correction_table'] + self._init_attributes(self._config_attributes) + + # These attributes are found in the environment datagrams + #self._environment_attributes = ['depth', 'acidity', 'salinity', 'sound_speed', 'temperature', + # 'latitude', 'transducer_sound_speed', 'sound_velocity_profile', 'drop_keel_offset', + # 'water_level_draft'] + #self._init_attributes(self._environment_attributes) + + + def from_raw_data(self, raw_data, return_indices=None): + """Populates the calibration object. + + This method uses the values extracted from a raw_data object to + populate the calibration object. + + Args: + raw_data (raw_data): The object where parameters will be extracted + from and used to populate the calibration object. + return_indices (array): A numpy array of indices to return. + """ + + # call the parent method + super(ek60_calibration, self).from_raw_data(raw_data, + return_indices=return_indices) + + # Handle our special attributes + # None for ek60_calibration + + + def get_attribute_from_raw(self, raw_data, param_name, return_indices=None): + """get_attribute_from_raw gets an individual attribute using the data + within the provided raw_data object. + """ + + if param_name == "gain_table": + param_data, return_indices = super(ek60_calibration, self).get_attribute_from_raw( + raw_data, "gain", return_indices=return_indices) + elif param_name == "pulse_length_table": + param_data, return_indices = super(ek60_calibration, self).get_attribute_from_raw( + raw_data, "pulse_duration", return_indices=return_indices) + elif param_name == "sa_correction_table": + param_data, return_indices = super(ek60_calibration, self).get_attribute_from_raw( + raw_data, "sa_correction", return_indices=return_indices) + elif param_name in ["gain","sa_correction","pulse_duration"]: + param_data, return_indices = super(ek60_calibration, self).get_attribute_from_raw( + raw_data, param_name, return_indices=return_indices) + new_data = np.empty((return_indices.shape[0]), dtype=np.float32) + for idx, config_obj in enumerate(raw_data.configuration[return_indices]): + new_data[idx] = param_data[config_obj['pulse_duration'] == raw_data.pulse_length[idx]][0] + ## ASSUMES that gain, Sa and pulse duraiton do not change + if param_name == "sa_correction": + param_data = new_data + else: + param_data = new_data[0] + else: + # first call the parent method to get the "standard" attributes + param_data, return_indices = super(ek60_calibration, self).get_attribute_from_raw( + raw_data, param_name, return_indices=return_indices) + + + return param_data + + + def __str__(self): + """Re-implements string method that provides some basic info about + the ek60_calibration object + + Returns: + A string with information about the ek60_calibration instance. + """ + + # print the class and address + msg = str(self.__class__) + " at " + str(hex(id(self))) + "\n" + + # Create a list of attributes to print out - I'm just adding them + # all right now. Can set specific attributes if this is too much. + attr_to_display = [] + attr_to_display.extend(self._raw_attributes) + attr_to_display.extend(self._config_attributes) + attr_to_display.extend(self._environment_attributes) + + # And assemble the string + for param_name in attr_to_display: + n_spaces = 32 - len(param_name) + msg += n_spaces * ' ' + param_name + # Extract data from raw_data attribues + if hasattr(self, param_name): + attr = getattr(self, param_name) + if isinstance(attr, np.ndarray): + msg += ' :: array ' + str(attr.size) + ' :: First value: ' + str(attr[0]) + '\n' + else: + if attr: + msg += ' :: scalar :: Value: ' + str(attr) + '\n' + else: + msg += ' :: No value set\n' + + return msg diff --git a/echolab2/instruments/util/simrad_parsers.py b/echolab2/instruments/util/simrad_parsers.py index a218433..619944c 100644 --- a/echolab2/instruments/util/simrad_parsers.py +++ b/echolab2/instruments/util/simrad_parsers.py @@ -978,8 +978,13 @@ def dict_to_dict(xml_dict, data_dict, parse_opts): data['environment']['transducer_sound_speed'] = [] for h in root_node.iter('Transducer'): transducer_xml = h.attrib - data['environment']['transducer_name'].append(transducer_xml['TransducerName']) - data['environment']['transducer_sound_speed'].append(float(transducer_xml['SoundSpeed'])) + ## Some files do not have this info and hence an error is raised without this addition + try: + data['environment']['transducer_name'].append(transducer_xml['TransducerName']) + data['environment']['transducer_sound_speed'].append(float(transducer_xml['SoundSpeed'])) + except: + pass + return data @@ -1233,7 +1238,7 @@ class SimradConfigParser(_SimradDatagramParser): sound_velocity_transducer [float] [m/s] beam_config [str] Raw XML string containing beam config. info - transceiver specific keys (ER60/ES60 sounders): + transceiver specific keys (ER60/ES60/ES70 sounders): channel_id [str] channel ident string beam_type [long] Type of channel (0 = Single, 1 = Split) frequency [float] channel frequency @@ -1369,6 +1374,32 @@ def __init__(self): ('gpt_software_version', '16s'), ('spare4', '28s') ], + 'ES70':[('channel_id', '128s'), + ('beam_type', 'l'), + ('frequency', 'f'), + ('gain', 'f'), + ('equivalent_beam_angle', 'f'), + ('beamwidth_alongship', 'f'), + ('beamwidth_athwartship', 'f'), + ('angle_sensitivity_alongship', 'f'), + ('angle_sensitivity_athwartship', 'f'), + ('angle_offset_alongship', 'f'), + ('angle_offset_athwartship', 'f'), + ('pos_x', 'f'), + ('pos_y', 'f'), + ('pos_z', 'f'), + ('dir_x', 'f'), + ('dir_y', 'f'), + ('dir_z', 'f'), + ('pulse_length_table', '5f'), + ('spare1', '8s'), + ('gain_table', '5f'), + ('spare2', '8s'), + ('sa_correction_table', '5f'), + ('spare3', '8s'), + ('gpt_software_version', '16s'), + ('spare4', '28s') + ], 'MBES':[('channel_id', '128s'), ('beam_type', 'l'), ('frequency', 'f'), @@ -1467,13 +1498,13 @@ def _unpack_contents(self, raw_string, bytes_read, version): txcvr_header_values = list(txcvr_header_values_encoded) for tx_idx, tx_val in enumerate(txcvr_header_values_encoded): if isinstance(tx_val, bytes): - txcvr_header_values[tx_idx] = tx_val.decode() + txcvr_header_values[tx_idx] = tx_val.decode(errors = "ignore") # RP added so that ES70 files parse channel_id = txcvr_header_values[0].strip('\x00') txcvr = data['configuration'].setdefault(channel_id, {}) txcvr.update(common_params) - if _sounder_name_used in ['ER60', 'ES60']: + if _sounder_name_used in ['ER60', 'ES60', 'ES70']: for txcvr_field_indx, field in enumerate(txcvr_header_fields[:17]): txcvr[field] = txcvr_header_values[txcvr_field_indx] @@ -1554,7 +1585,7 @@ def _pack_contents(self, data, version): for txcvr_indx, txcvr in list(data['configuration'].items()): txcvr_contents = [] - if _sounder_name_used in ['ER60', 'ES60']: + if _sounder_name_used in ['ER60', 'ES60', 'ES70']: for field in txcvr_header_fields[:17]: # Python 3 convert str to bytes if isinstance(txcvr[field], str):