Skip to content

Commit

Permalink
add time units to throughput metrics and also secs_per_step metric.
Browse files Browse the repository at this point in the history
  • Loading branch information
ekelsen committed Oct 31, 2024
1 parent eb51f08 commit ce41bf1
Showing 1 changed file with 9 additions and 0 deletions.
9 changes: 9 additions & 0 deletions composer/callbacks/speed_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,7 @@ def __init__(
self.history_flops: deque[float] = deque(maxlen=window_size + 1)

self.gpu_flops_available = gpu_flops_available
self.time_unit = time_unit

self.divider = 1
if time_unit == 'seconds':
Expand Down Expand Up @@ -360,10 +361,18 @@ def batch_end(self, state: State, logger: Logger):
# Log the time
# `state.timestamp` excludes any time spent in evaluation
train_wct = state.timestamp.total_wct.total_seconds()
secs_per_step = 0
if len(self.history_wct) > 1:
secs_per_step = self.history_wct[-1] - self.history_wct[-2]

logger.log_metrics({
'time/train': train_wct / self.divider,
'time/val': self.total_eval_wct / self.divider,
'time/total': (train_wct + self.total_eval_wct) / self.divider,
f'time_{self.time_unit}/train': train_wct / self.divider,
f'time_{self.time_unit}/val': self.total_eval_wct / self.divider,
f'time_{self.time_unit}/total': (train_wct + self.total_eval_wct) / self.divider,
'time/secs_per_step': secs_per_step,
})

def eval_end(self, state: State, logger: Logger):
Expand Down

0 comments on commit ce41bf1

Please sign in to comment.