Skip to content

Commit

Permalink
Merge pull request #3427 from rebeccacremona/scoop-stats
Browse files Browse the repository at this point in the history 
Add capture errors panel to admin stats page.
  • Loading branch information
@rebeccacremona
rebeccacremona authored Nov 2, 2023
2 parents 2452363 + 2899930 commit 7007e80
Show file tree
Hide file tree
Showing 5 changed files with 273 additions and 1 deletion.
88 changes: 88 additions & 0 deletions perma_web/perma/templates/user_management/stats.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
<li class="active"><a href="#celery_data" data-toggle="tab">Celery Queues</a></li>
<li><a href="#rate-limits-pane" data-toggle="tab">Internet Archive</a></li>
<li><a href="#capture-job-pane" data-toggle="tab">Capture Jobs</a></li>
<li><a href="#capture-error-pane" data-toggle="tab">Capture Errors</a></li>
<li><a href="#days-pane" data-toggle="tab">This Month's Links</a></li>
<li><a href="#random-pane" data-toggle="tab">Random</a></li>
<li><a href="#emails-pane" data-toggle="tab">Users by Domain</a></li>
Expand Down Expand Up @@ -42,6 +43,13 @@ <h3 class="body-ah">Capture jobs</h3>
<div id="job_queue">Loading...</div>
</div>

<div class="tab-pane" id="capture-error-pane">
<h3 class="body-ah">Capture errors</h3>
<button type="button" id="refresh-capture-errors">Refresh</button>&nbsp;<button type="button" id="auto-refresh-capture-errors">Start Auto-Refresh (every 15s)</button><span id="capture-errors-status"></span>
<br><br>
<div id="capture_errors">Loading...</div>
</div>


<div class="tab-pane" id="days-pane">
<h3 class="body-ah">Links from the past month</h3>
Expand Down Expand Up @@ -339,5 +347,85 @@ <h5>Finished task count:</h5>
{{/each}}
</div>
</script>

<script id="capture_errors-template" type="text/x-handlebars-template">
<div class="row">

<div class="col-sm-3">
<h4>Last hour</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.last_hour.completed }} ({{ this.last_hour.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_hour.failed }} ({{ this.last_hour.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_hour.celery_timeout }} ({{ this.last_hour.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.last_hour.mystery_error }} ({{ this.last_hour.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_hour.timeout }} ({{ this.last_hour.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_hour.didnt_load }} ({{ this.last_hour.didnt_load_percent}}%)</dd>
</dl>
</div>

<div class="col-sm-3">
<h4>Last 3 hours</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.last_3_hrs.completed }} ({{ this.last_3_hrs.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_3_hrs.failed }} ({{ this.last_3_hrs.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_3_hrs.celery_timeout }} ({{ this.last_3_hrs.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.last_3_hrs.mystery_error }} ({{ this.last_3_hrs.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_3_hrs.timeout }} ({{ this.last_3_hrs.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_3_hrs.didnt_load }} ({{ this.last_3_hrs.didnt_load_percent}}%)</dd>
</dl>
</div>

<div class="col-sm-3">
<h4>Last 24 hrs</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.last_24_hrs.completed }} ({{ this.last_24_hrs.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_24_hrs.failed }} ({{ this.last_24_hrs.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_24_hrs.celery_timeout }} ({{ this.last_24_hrs.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.last_24_hrs.mystery_error }} ({{ this.last_24_hrs.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_24_hrs.timeout }} ({{ this.last_24_hrs.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_24_hrs.didnt_load }} ({{ this.last_24_hrs.didnt_load_percent}}%)</dd>
</dl>
</div>
</div>

<div class="row">

<div class="col-sm-3">
<h4>Last hour, previous day</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.last_hour_on_previous_day.completed }} ({{ this.last_hour_on_previous_day.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_hour_on_previous_day.failed }} ({{ this.last_hour_on_previous_day.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_hour_on_previous_day.celery_timeout }} ({{ this.last_hour_on_previous_day.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.last_hour_on_previous_day.mystery_error }} ({{ this.last_hour_on_previous_day.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_hour_on_previous_day.timeout }} ({{ this.last_hour_on_previous_day.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_hour_on_previous_day.didnt_load }} ({{ this.last_hour_on_previous_day.didnt_load_percent}}%)</dd>
</dl>
</div>

<div class="col-sm-3">
<h4>Last 3 hours, previous day</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.last_3_hrs_on_previous_day.completed }} ({{ this.last_3_hrs_on_previous_day.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.last_3_hrs_on_previous_day.failed }} ({{ this.last_3_hrs_on_previous_day.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.last_3_hrs_on_previous_day.celery_timeout }} ({{ this.last_3_hrs_on_previous_day.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.last_3_hrs_on_previous_day.mystery_error }} ({{ this.last_3_hrs_on_previous_day.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.last_3_hrs_on_previous_day.timeout }} ({{ this.last_3_hrs_on_previous_day.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.last_3_hrs_on_previous_day.didnt_load }} ({{ this.last_3_hrs_on_previous_day.didnt_load_percent}}%)</dd>
</dl>
</div>


<div class="col-sm-3">
<h4>Previous 24 hrs</h4>
<dl class="dl-horizontal">
<dt>Completed:</dt><dd>{{ this.previous_24_hrs.completed }} ({{ this.previous_24_hrs.completed_percent}}%)</dd>
<dt>Failed:</dt><dd>{{ this.previous_24_hrs.failed }} ({{ this.previous_24_hrs.failed_percent}}%)</dd>
<dt>Killed by Celery:</dt><dd>{{ this.previous_24_hrs.celery_timeout }} ({{ this.previous_24_hrs.celery_timeout_percent}}%)</dd>
<dt>Mystery error:</dt><dd>{{ this.previous_24_hrs.mystery_error }} ({{ this.previous_24_hrs.mystery_error_percent}}%)</dd>
<dt>Scoop timeout:</dt><dd>{{ this.previous_24_hrs.timeout }} ({{ this.previous_24_hrs.timeout_percent}}%)</dd>
<dt>URL didn't load:</dt><dd>{{ this.previous_24_hrs.didnt_load }} ({{ this.previous_24_hrs.didnt_load_percent}}%)</dd>
</dl>
</div>
</script>
{% endverbatim %}
{% endblock %}
116 changes: 116 additions & 0 deletions perma_web/perma/views/user_management.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,122 @@ def stats(request, stat_type=None):
out['total_ia_queue'] = r.llen('ia')
out['total_ia_readonly_queue'] = r.llen('ia-readonly')

elif stat_type == 'capture_errors':

# Set up time ranges
now = timezone.now()
last_24_hrs = (
now - timedelta(days=1),
now
)
previous_24_hrs = (
now - timedelta(days=2),
now - timedelta(days=1)
)
last_hour = (
now - timedelta(hours=1),
now
)
last_hour_on_previous_day = (
now - timedelta(days=1) - timedelta(hours=1),
now - timedelta(days=1)
)
last_3_hrs = (
now - timedelta(hours=3),
now
)
last_3_hrs_on_previous_day = (
now - timedelta(days=1) - timedelta(hours=3),
now - timedelta(hours=3)
)
ranges = {
"last_24_hrs": last_24_hrs,
"previous_24_hrs": previous_24_hrs,
"last_hour": last_hour,
"last_hour_on_previous_day": last_hour_on_previous_day,
"last_3_hrs": last_3_hrs,
"last_3_hrs_on_previous_day": last_3_hrs_on_previous_day
}

out = {}

for range_name, range_tuple in ranges.items():

#
# Get Totals
#

completed = CaptureJob.objects.filter(
engine='scoop-api',
status='completed',
capture_start_time__range=range_tuple
).count()

failed = CaptureJob.objects.filter(
engine='scoop-api',
status='failed',
capture_start_time__range=range_tuple
).count()

denominator = completed + failed

#
# Get By Error Type
#
if denominator:

celery_timeout = Link.objects.all_with_deleted().filter(
tags__name__in=['timeout-failure'],
creation_timestamp__range=range_tuple
).count()

mystery_error = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-stopped-failure'],
creation_timestamp__range=range_tuple
).count()

timeout = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-silent-failure'],
creation_timestamp__range=range_tuple
).count()

didnt_load = Link.objects.all_with_deleted().filter(
tags__name__in=['scoop-load-failure'],
creation_timestamp__range=range_tuple
).count()

out[range_name] = {
"completed": completed,
"completed_percent": completed/denominator * 100,
"failed": failed,
"failed_percent": failed/denominator * 100,
"celery_timeout": celery_timeout,
"celery_timeout_percent": celery_timeout/denominator * 100,
"mystery_error": mystery_error,
"mystery_error_percent": mystery_error/denominator * 100,
"timeout": timeout,
"timeout_percent": timeout/denominator * 100,
"didnt_load": didnt_load,
"didnt_load_percent": didnt_load/denominator * 100,
}

else:

out[range_name] = {
"completed": 0,
"completed_percent": 0,
"failed": 0,
"failed_percent": 0,
"celery_timeout": 0,
"celery_timeout_percent": 0,
"mystery_error": 0,
"mystery_error_percent": 0,
"timeout": 0,
"timeout_percent": 0,
"didnt_load": 0,
"didnt_load_percent": 0,
}

if out:
return JsonResponse(out)

Expand Down
36 changes: 36 additions & 0 deletions perma_web/static/bundles/admin-stats.js
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion perma_web/static/bundles/admin-stats.js.map
View file

Large diffs are not rendered by default.

32 changes: 32 additions & 0 deletions perma_web/static/js/admin-stats.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ function fillSection(name, callback){
});
}

fillSection("capture_errors");
fillSection("celery_queues");
fillSection("celery");
fillSection("rate_limits");
Expand Down Expand Up @@ -89,6 +90,37 @@ document.getElementById('toggle-capture-jobs-auto-refresh').addEventListener('cl
})


// Refresh the capture errors once, on button press
// or, start auto-refreshing every 15s, when the other button is pressed
function refresh_capture_errors(){
let status = document.getElementById('capture-errors-status')
status.innerText = 'Refreshing...';
fillSection("capture_errors", () => {
status.innerText = 'Refreshed!';
setTimeout(()=> status.innerText = '', 2000);
});
}
function auto_refresh_capture_errors(){
return setInterval(function(){ refresh_capture_errors()}, 15000);
}
document.getElementById('refresh-capture-errors').addEventListener('click', (e) => {
refresh_capture_errors()
})
let capture_errors_refresh = null;
document.getElementById('auto-refresh-capture-errors').addEventListener('click', (e) => {
if (capture_errors_refresh){
clearInterval(capture_errors_refresh);
capture_errors_refresh = null;
e.target.innerText = 'Start Auto-Refresh (every 15s)';
} else {
e.target.innerText = 'Stop Auto-Refresh';
refresh_capture_errors()
capture_errors_refresh = auto_refresh_capture_errors();
}
})



// Select the tab specified in the hash, if present on page load
if (window.location.hash) {
let tabNav = document.querySelector(`a[href="${window.location.hash}"]`);
Expand Down

0 comments on commit 7007e80

Please sign in to comment.