Skip to content

Commit

Permalink
log status changes
Browse files Browse the repository at this point in the history
  • Loading branch information
denniswittich committed Dec 5, 2024
1 parent 6a04d2a commit b95eb2f
Show file tree
Hide file tree
Showing 5 changed files with 18 additions and 3 deletions.
3 changes: 2 additions & 1 deletion learning_loop_node/annotation/annotator_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ async def send_status(self):
capabilities=['segmentation']
)

self.log.debug('Sending status %s', status)
self.log_status_on_change(status.state.value if status.state else 'None', status)

try:
result = await self.sio_client.call('update_annotation_node', jsonable_encoder(asdict(status)), timeout=10)
except Exception:
Expand Down
4 changes: 3 additions & 1 deletion learning_loop_node/detector/detector_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ def __init__(self, name: str, detector: DetectorLogic, uuid: Optional[str] = Non
self.log.info('Using %s/%s', self.organization, self.project)
self.operation_mode: OperationMode = OperationMode.Startup
self.connected_clients: List[str] = []
self.previous_state: Optional[str] = None

self.detection_lock = asyncio.Lock()

Expand Down Expand Up @@ -292,7 +293,8 @@ async def sync_status_with_learning_loop(self) -> None:
model_format=self.detector_logic.model_format,
)

self.log.debug('sending status %s', status)
self.log_status_on_change(status.state or 'None', status)

response = await self.sio_client.call('update_detector', (self.organization, self.project, jsonable_encoder(asdict(status))))
if not response:
self.socket_connection_broken = True
Expand Down
9 changes: 9 additions & 0 deletions learning_loop_node/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,15 @@ def __init__(self, name: str, uuid: Optional[str] = None, node_type: str = 'node

self.repeat_loop_lock = asyncio.Lock()

self.previous_state: Optional[str] = None

def log_status_on_change(self, current_state_str: str, full_status: Any):
if self.previous_state != current_state_str:
self.previous_state = current_state_str
self.log.info('Status changed to %s', full_status)
else:
self.log.debug('sending status %s', full_status)

def init_loop_communicator(self):
self.loop_communicator = LoopCommunicator()
self.websocket_url = self.loop_communicator.websocket_url()
Expand Down
1 change: 1 addition & 0 deletions learning_loop_node/trainer/trainer_logic_generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,7 @@ async def _download_model(self) -> None:

logger.info('loading model from Learning Loop')
logger.info('downloading model %s as %s', base_model_uuid, self.model_format)
assert base_model_uuid is not None
await self.node.data_exchanger.download_model(self.training.training_folder, self.training.context, base_model_uuid, self.model_format)
shutil.move(f'{self.training.training_folder}/model.json',
f'{self.training.training_folder}/base_model.json')
Expand Down
4 changes: 3 additions & 1 deletion learning_loop_node/trainer/trainer_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ def __init__(self, name: str, trainer_logic: TrainerLogicGeneric, uuid: Optional
self.trainer_logic = trainer_logic
self.last_training_io = LastTrainingIO(self.uuid)
self.trainer_logic._last_training_io = self.last_training_io
self.previous_state: Optional[str] = None

self._first_idle_time: float | None = None
if os.environ.get('TRAINER_IDLE_TIMEOUT_SEC', 0.0):
Expand Down Expand Up @@ -81,7 +82,8 @@ async def send_status(self):
return

status = self.trainer_logic.generate_status_for_loop(self.uuid, self.name)
self.log.debug('sending status: %s', status.short_str())
self.log_status_on_change(status.state or 'None', status.short_str())

result = await self.sio_client.call('update_trainer', jsonable_encoder(asdict(status)), timeout=30)
if isinstance(result, Dict) and not result['success']:
self.socket_connection_broken = True
Expand Down

0 comments on commit b95eb2f

Please sign in to comment.