From 1e4f8925e372e0ad5461991342fbf2fda9b92d0a Mon Sep 17 00:00:00 2001 From: Bea Steers Date: Mon, 30 Oct 2023 21:44:42 -0400 Subject: [PATCH] logging --- redis_record/record/streams.py | 7 ++++++- redis_record/storage/recorder/base.py | 6 ++++++ redis_record/storage/recorder/json.py | 4 ++++ redis_record/storage/recorder/mcap.py | 13 +++++++++---- redis_record/storage/recorder/zip.py | 8 ++++++++ redis_record/storage/replay/mcap.py | 10 ++++++++++ redis_record/storage/replay/zip.py | 4 +++- 7 files changed, 46 insertions(+), 6 deletions(-) diff --git a/redis_record/record/streams.py b/redis_record/record/streams.py index b889a89..90cf673 100644 --- a/redis_record/record/streams.py +++ b/redis_record/record/streams.py @@ -165,9 +165,14 @@ def record( cmd.stop_recording(r) + def cli(): + import logging + from tqdm.contrib.logging import logging_redirect_tqdm import fire - fire.Fire(record) + logging.basicConfig(level=logging.DEBUG) + with logging_redirect_tqdm(): + fire.Fire(record) if __name__ == '__main__': from pyinstrument import Profiler diff --git a/redis_record/storage/recorder/base.py b/redis_record/storage/recorder/base.py index faa453a..7f120bc 100644 --- a/redis_record/storage/recorder/base.py +++ b/redis_record/storage/recorder/base.py @@ -1,4 +1,10 @@ +import logging + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) + + class BaseRecorder: def __init__(self, out_dir='.', schema=None): diff --git a/redis_record/storage/recorder/json.py b/redis_record/storage/recorder/json.py index aff5fa6..65ddd8f 100644 --- a/redis_record/storage/recorder/json.py +++ b/redis_record/storage/recorder/json.py @@ -3,6 +3,10 @@ from .base import BaseRecorder from ...util import move_with_suffix +import logging + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) class JsonRecorder(BaseRecorder): diff --git a/redis_record/storage/recorder/mcap.py b/redis_record/storage/recorder/mcap.py index 124ba77..99765d2 100644 --- a/redis_record/storage/recorder/mcap.py +++ b/redis_record/storage/recorder/mcap.py @@ -3,13 +3,18 @@ import os import time import json -import tqdm import base64 from mcap.writer import Writer from .base import BaseRecorder from ...config import DEFAULT_CHANNEL from ...util import move_with_suffix +import logging + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) + + class MCAPRecorder(BaseRecorder): def __init__(self, schema=None, out_dir='.'): self.out_dir = out_dir @@ -35,7 +40,7 @@ def ensure_writer(self, record_name, force=False): self.fhandle = fhandle = open(fname, "wb") self.writer = writer = Writer(fhandle) writer.start() - tqdm.tqdm.write(f"Opening writer {fname}") + log.info("Created Recorder: %s", fname) # https://mcap.dev/docs/python/mcap-apidoc/mcap.well_known#mcap.well_known.MessageEncoding kw = {"properties": self.schema} if self.schema else {} @@ -48,7 +53,7 @@ def ensure_writer(self, record_name, force=False): def ensure_channel(self, channel=DEFAULT_CHANNEL): if channel not in self.channel_ids: - tqdm.tqdm.write(f"opening channel {channel}") + log.debug("Opening Recorder Channel: %s %s", self.fname, channel) self.channel_ids[channel] = self.writer.register_channel( schema_id=self.schema_id, topic=channel, @@ -67,11 +72,11 @@ def write(self, stream_id, timestamp, data): def close(self): if self.writer is not None: - tqdm.tqdm.write(f"closing {self.fname}") self.writer.finish() self.fhandle.close() self.writer = self.fhandle = None self.channel_ids.clear() + log.info("Closed Recorder: %s", self.fname) diff --git a/redis_record/storage/recorder/zip.py b/redis_record/storage/recorder/zip.py index c2759c2..58faa6f 100644 --- a/redis_record/storage/recorder/zip.py +++ b/redis_record/storage/recorder/zip.py @@ -3,6 +3,12 @@ from .base import BaseRecorder from ...util import format_epoch_time, move_with_suffix +import logging + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) + + MB = 1024 * 1024 class ZipRecorder(BaseRecorder): @@ -24,6 +30,7 @@ def close(self): if self.writer: for _, w in self.writer.items(): w.close() + log.info("Closed Recorder: %s", self.recording_dir) self.writer = None def ensure_writer(self, record_name, force=False): @@ -34,6 +41,7 @@ def ensure_writer(self, record_name, force=False): self.recording_dir = os.path.join(self.out_dir, record_name) move_with_suffix(self.recording_dir) os.makedirs(self.recording_dir, exist_ok=True) + log.info("Created Recorder: %s", self.recording_dir) return self def ensure_channel(self, channel): diff --git a/redis_record/storage/replay/mcap.py b/redis_record/storage/replay/mcap.py index a1eac23..459e4ef 100644 --- a/redis_record/storage/replay/mcap.py +++ b/redis_record/storage/replay/mcap.py @@ -4,11 +4,17 @@ from mcap.reader import make_reader from redis_record.config import RECORDING_DIR +import logging + +log = logging.getLogger(__name__) +log.setLevel(logging.INFO) + class MCAPPlayer: def __init__(self, name, recording_dir=RECORDING_DIR, subset=None) -> None: self.path = name if os.path.isfile(name) else os.path.join(recording_dir, f'{name}.mcap') self.subset = subset + log.info("Created Player: %s - streams: %s", self.path, self.subset or 'all') def __enter__(self): self.fh = open(self.path, "rb") @@ -17,7 +23,11 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_value, trace): + self.close() + + def close(self): self.fh.close() + log.info("Closed Player: %s - streams: %s", self.path, self.subset or 'all') def __iter__(self): yield from self.iter_messages() diff --git a/redis_record/storage/replay/zip.py b/redis_record/storage/replay/zip.py index 199c62e..1878ac8 100644 --- a/redis_record/storage/replay/zip.py +++ b/redis_record/storage/replay/zip.py @@ -10,7 +10,7 @@ import logging log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) +log.setLevel(logging.INFO) class ZipPlayer: @@ -24,6 +24,7 @@ def __init__(self, path, recording_dir=RECORDING_DIR, subset=None, raw_timestamp self.file_end_timestamps = {} self.queue = queue.PriorityQueue() self.raw_timestamp = raw_timestamp + log.info("Created Player: %s - streams: %s", self.recording_dir, self.subset or 'all') self._load_file_index() for stream_id in self.file_index: @@ -82,6 +83,7 @@ def close(self): for zf in self.zipfh.values(): zf.close() self.zipfh.clear() + log.info("Closed Player: %s - streams: %s", self.recording_dir, self.subset or 'all') def _get_time_range_from_file(self, fname): t0, t1 = fname.split(os.sep)[-1].removesuffix('.zip').split('_')