Skip to content

Commit 29786ba

Browse files
authored
Add check to ensure file index is used correctly (#333)
# Fixes - Add check for when `MixedLogReader` is rewound in order to prevent a numerical overflow error - Set default next index element when file is being read from beginning
2 parents 702e065 + 57558ee commit 29786ba

File tree

1 file changed

+5
-1
lines changed

1 file changed

+5
-1
lines changed

python/fusion_engine_client/parsers/mixed_log_reader.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,9 @@ def _print_progress(self, file_size=None):
416416
if file_size is None:
417417
file_size = min(self.file_size_bytes, self.max_bytes)
418418

419-
if self.total_bytes_read - self.last_print_bytes > 10e6 or self.total_bytes_read == file_size:
419+
if self.total_bytes_read < self.last_print_bytes or \
420+
self.total_bytes_read - self.last_print_bytes > 10e6 or \
421+
self.total_bytes_read == file_size:
420422
elapsed_sec = (datetime.now() - self.start_time).total_seconds()
421423
self.logger.log(logging.INFO if show_progress else logging.DEBUG,
422424
'Processed %d/%d bytes (%.1f%%). [elapsed=%.1f sec, rate=%.1f MB/s]' %
@@ -589,6 +591,8 @@ def filter_in_place(self, key, clear_existing: Union[bool, str] = False,
589591
if self.index is not None:
590592
if len(self.index) == 0:
591593
self.next_index_elem = 0
594+
elif prev_offset_bytes < 0:
595+
self.next_index_elem = 0
592596
else:
593597
idx = np.argmax(self.index.offset > prev_offset_bytes)
594598
if idx == 0 and self.index.offset[0] <= prev_offset_bytes:

0 commit comments

Comments
 (0)