Merge pull request #65 from danmergens/13713

13713

RELEASE_NOTES.md

@@ -1,3 +1,11 @@
+# Version 0.7.0
+
+* Issue #13713 - prevent cg_dcl_eng parser from hanging on ingest
+   * Encoding errors will no longer result in parser exiting prematurely (all parsers)
+   * Added range checking option for particle value encoding (all parsers)
+   * Corrected regex resulting in catastrophic backtracking (cg_dcl_eng)
+   * Optimized file parsing logic (cg_dcl_eng)
+
 # Version 0.6.9
 
 * Issue #13722 - Platform node deployment update

mi/core/instrument/dataset_data_particle.py

@@ -330,17 +330,58 @@ def _check_preferred_timestamps(self):
 
         return True
 
-    def _encode_value(self, name, value, encoding_function):
+    def _encode_value(self, name, value, encoding_function, value_range=None):
         """
         Encode a value using the encoding function, if it fails store the error in a queue
+
+        :param value_range  tuple containing min/max numerical values or min/max lengths
         """
         encoded_val = None
 
+        # noinspection PyBroadException
+        # - custom encoding_function exceptions are not known a priori
         try:
             encoded_val = encoding_function(value)
+
+        except ValueError as e:
+            log.error('Unable to convert %s to %s.', encoded_val, encoding_function)
+            self._encoding_errors.append({name: value})
         except Exception as e:
-            log.error("Data particle error encoding. Name:%s Value:%s", name, value)
+            log.error('Data particle error encoding. Name: %s Value: %s, Encoding: %s', name, value, encoding_function)
             self._encoding_errors.append({name: value})
+
+        # optional range checking
+        if value_range:
+            try:
+                vmin, vmax = value_range
+
+            except ValueError as e:  # this only occurs as a programming error and should cause the parser to exit
+                log.exception('_encode_value must have exactly two values for tuple argument value_range')
+                raise ValueError(e)
+
+            if encoding_function in [int, float]:
+                if vmin and encoded_val < vmin:
+                    log.error('Particle value (%s) below minimum threshold (%s < %s)', name, value, vmin)
+                    self._encoding_errors.append({name: value})
+                elif vmax and encoded_val > vmax:
+                    log.error('Particle value (%s) exceeds maximum threshold (%s > %s)', name, value, vmax)
+                    self._encoding_errors.append({name: value})
+            elif hasattr(encoded_val, '__len__'):
+                try:
+                    if vmin and len(encoded_val) < vmin:
+                        log.error('Particle value (%s) length below minimum threshold (%s < %s)',
+                                  name, value, vmin)
+                        self._encoding_errors.append({name: value})
+                    elif vmax and len(encoded_val) > vmax:
+                        log.error('Particle value (%s) length exceeds maximum threshold (%s > %s)',
+                                  name, value, vmax)
+                        self._encoding_errors.append({name: value})
+                # in the unlikely event that a range was specified and the encoding object created a bogus len()
+                # we'll just ignore the range check
+                except TypeError:
+                    log.warning('_encode_value received an encoding function (%s) that claimed to implement len() but '
+                                'does not. Unable to apply range test to %s', encoding_function, name)
+
         return {DataParticleKey.VALUE_ID: name,
                 DataParticleKey.VALUE: encoded_val}
 

mi/core/instrument/instrument_protocol.py

@@ -1275,6 +1275,7 @@ def _wakeup_until(self, timeout, desired_prompt, delay=1, no_tries=5):
                 if count >= no_tries:
                     raise InstrumentProtocolException('Incorrect prompt.')
 
+
 class MenuInstrumentProtocol(CommandResponseInstrumentProtocol):
     """
     Base class for menu-based instrument interfaces that can use a cmd/response approach to

mi/dataset/dataset_parser.py

@@ -123,8 +123,8 @@ def _extract_sample(self, particle_class, regex, raw_data, port_timestamp=None,
 
         try:
             if regex is None or regex.match(raw_data):
-                particle = particle_class(raw_data, port_timestamp=port_timestamp, internal_timestamp=internal_timestamp,
-                                          preferred_timestamp=preferred_ts)
+                particle = particle_class(raw_data, port_timestamp=port_timestamp,
+                                          internal_timestamp=internal_timestamp, preferred_timestamp=preferred_ts)
 
                 # need to actually parse the particle fields to find out of there are errors
                 particle.generate_dict()

mi/dataset/driver/cg_dcl_eng/dcl/cg_dcl_eng_dcl_recovered_driver.py

@@ -13,7 +13,7 @@
 
 from mi.dataset.dataset_parser import DataSetDriverConfigKeys
 from mi.dataset.dataset_driver import SimpleDatasetDriver
-from mi.dataset.parser.cg_dcl_eng_dcl import CgDclEngDclParser, CgDclEngDclParticleClassTypes, \
+from mi.dataset.parser.cg_dcl_eng_dcl import CgDclEngDclParser, ParticleClassTypes, \
     CgDclEngDclMsgCountsRecoveredDataParticle, \
     CgDclEngDclCpuUptimeRecoveredDataParticle, \
     CgDclEngDclErrorRecoveredDataParticle, \
@@ -57,25 +57,25 @@ def _build_parser(self, stream_handle):
             DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.cg_dcl_eng_dcl',
             DataSetDriverConfigKeys.PARTICLE_CLASS: None,
             DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
-                CgDclEngDclParticleClassTypes.MSG_COUNTS_PARTICLE_CLASS:
+                ParticleClassTypes.MSG_COUNTS_PARTICLE_CLASS:
                     CgDclEngDclMsgCountsRecoveredDataParticle,
-                CgDclEngDclParticleClassTypes.CPU_UPTIME_PARTICLE_CLASS:
+                ParticleClassTypes.CPU_UPTIME_PARTICLE_CLASS:
                     CgDclEngDclCpuUptimeRecoveredDataParticle,
-                CgDclEngDclParticleClassTypes.ERROR_PARTICLE_CLASS:
+                ParticleClassTypes.ERROR_PARTICLE_CLASS:
                     CgDclEngDclErrorRecoveredDataParticle,
-                CgDclEngDclParticleClassTypes.GPS_PARTICLE_CLASS:
+                ParticleClassTypes.GPS_PARTICLE_CLASS:
                     CgDclEngDclGpsRecoveredDataParticle,
-                CgDclEngDclParticleClassTypes.PPS_PARTICLE_CLASS:
+                ParticleClassTypes.PPS_PARTICLE_CLASS:
                     CgDclEngDclPpsRecoveredDataParticle,
-                CgDclEngDclParticleClassTypes.SUPERV_PARTICLE_CLASS:
+                ParticleClassTypes.SUPERV_PARTICLE_CLASS:
                     CgDclEngDclSupervRecoveredDataParticle,
-                CgDclEngDclParticleClassTypes.DLOG_MGR_PARTICLE_CLASS:
+                ParticleClassTypes.DLOG_MGR_PARTICLE_CLASS:
                     CgDclEngDclDlogMgrRecoveredDataParticle,
-                CgDclEngDclParticleClassTypes.DLOG_STATUS_PARTICLE_CLASS:
+                ParticleClassTypes.DLOG_STATUS_PARTICLE_CLASS:
                     CgDclEngDclDlogStatusRecoveredDataParticle,
-                CgDclEngDclParticleClassTypes.STATUS_PARTICLE_CLASS:
+                ParticleClassTypes.STATUS_PARTICLE_CLASS:
                     CgDclEngDclStatusRecoveredDataParticle,
-                CgDclEngDclParticleClassTypes.DLOG_AARM_PARTICLE_CLASS:
+                ParticleClassTypes.DLOG_AARM_PARTICLE_CLASS:
                     CgDclEngDclDlogAarmRecoveredDataParticle,
             }
 

mi/dataset/driver/cg_dcl_eng/dcl/cg_dcl_eng_dcl_telemetered_driver.py

@@ -13,7 +13,7 @@
 
 from mi.dataset.dataset_parser import DataSetDriverConfigKeys
 from mi.dataset.dataset_driver import SimpleDatasetDriver
-from mi.dataset.parser.cg_dcl_eng_dcl import CgDclEngDclParser, CgDclEngDclParticleClassTypes, \
+from mi.dataset.parser.cg_dcl_eng_dcl import CgDclEngDclParser, ParticleClassTypes, \
     CgDclEngDclMsgCountsTelemeteredDataParticle, \
     CgDclEngDclCpuUptimeTelemeteredDataParticle, \
     CgDclEngDclErrorTelemeteredDataParticle, \
@@ -57,25 +57,25 @@ def _build_parser(self, stream_handle):
             DataSetDriverConfigKeys.PARTICLE_MODULE: 'mi.dataset.parser.cg_dcl_eng_dcl',
             DataSetDriverConfigKeys.PARTICLE_CLASS: None,
             DataSetDriverConfigKeys.PARTICLE_CLASSES_DICT: {
-                CgDclEngDclParticleClassTypes.MSG_COUNTS_PARTICLE_CLASS:
+                ParticleClassTypes.MSG_COUNTS_PARTICLE_CLASS:
                     CgDclEngDclMsgCountsTelemeteredDataParticle,
-                CgDclEngDclParticleClassTypes.CPU_UPTIME_PARTICLE_CLASS:
+                ParticleClassTypes.CPU_UPTIME_PARTICLE_CLASS:
                     CgDclEngDclCpuUptimeTelemeteredDataParticle,
-                CgDclEngDclParticleClassTypes.ERROR_PARTICLE_CLASS:
+                ParticleClassTypes.ERROR_PARTICLE_CLASS:
                     CgDclEngDclErrorTelemeteredDataParticle,
-                CgDclEngDclParticleClassTypes.GPS_PARTICLE_CLASS:
+                ParticleClassTypes.GPS_PARTICLE_CLASS:
                     CgDclEngDclGpsTelemeteredDataParticle,
-                CgDclEngDclParticleClassTypes.PPS_PARTICLE_CLASS:
+                ParticleClassTypes.PPS_PARTICLE_CLASS:
                     CgDclEngDclPpsTelemeteredDataParticle,
-                CgDclEngDclParticleClassTypes.SUPERV_PARTICLE_CLASS:
+                ParticleClassTypes.SUPERV_PARTICLE_CLASS:
                     CgDclEngDclSupervTelemeteredDataParticle,
-                CgDclEngDclParticleClassTypes.DLOG_MGR_PARTICLE_CLASS:
+                ParticleClassTypes.DLOG_MGR_PARTICLE_CLASS:
                     CgDclEngDclDlogMgrTelemeteredDataParticle,
-                CgDclEngDclParticleClassTypes.DLOG_STATUS_PARTICLE_CLASS:
+                ParticleClassTypes.DLOG_STATUS_PARTICLE_CLASS:
                     CgDclEngDclDlogStatusTelemeteredDataParticle,
-                CgDclEngDclParticleClassTypes.STATUS_PARTICLE_CLASS:
+                ParticleClassTypes.STATUS_PARTICLE_CLASS:
                     CgDclEngDclStatusTelemeteredDataParticle,
-                CgDclEngDclParticleClassTypes.DLOG_AARM_PARTICLE_CLASS:
+                ParticleClassTypes.DLOG_AARM_PARTICLE_CLASS:
                     CgDclEngDclDlogAarmTelemeteredDataParticle,
             }
 
@@ -86,5 +86,3 @@ def _build_parser(self, stream_handle):
                                    self._exception_callback)
 
         return parser
-
-