Migrate to using django-rq for scheduled calls (#63)

- CRON is no longer needed
- Improvements to Request list page

pdf/generate.py

@@ -2,22 +2,25 @@
 import locale
 import logging
 import os
+import re
 import tempfile
+from datetime import datetime
 from math import ceil
+from time import time
 
 import weasyprint
+from rq import Retry
 from weasyprint.logger import PROGRESS_LOGGER
 from django.conf import settings
 from django.core.files import File
 from django.template.loader import render_to_string
 from django.urls import reverse
 from django.utils import translation
-from django_rq import job
+from django_rq import job, get_queue
 from django_weasyprint.utils import django_url_fetcher
 
 from pdf.locales import changed_locale, lang_to_locale
 from pdf.models.request import PDFRequest, Status
-from pdf.utils import Timer, ProgressFilter
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -88,3 +91,47 @@ def generate_pdf(request: PDFRequest):
 def generate_pdf_job(request: PDFRequest):
     """Generates PDF from request in the background"""
     generate_pdf(request)
+
+
+def schedule_generation(request: PDFRequest, schedule_time: datetime):
+    """Schedules generation of a request at a specific time"""
+    queue = get_queue('default')
+    created_job = queue.enqueue_at(
+        schedule_time,
+        generate_pdf,
+        request,
+        retry = Retry(max=5, interval=120)
+    )
+    logger.info("Schedule PDF generation of request %s at %s", request.id, schedule_time)
+    return created_job
+
+
+class ProgressFilter(logging.Filter):
+    """Filters Weasyprint progress messages, highly dependent on implementation!"""
+    STEP_NUMBER = re.compile(r"(?<=Step\s)\d")
+
+    def __init__(self, request):
+        super().__init__()
+        self.request = request
+
+    def filter(self, record):
+        step = self.STEP_NUMBER.search(record.getMessage()).group(0)
+        if self.request.progress != step:
+            self.request.progress = step
+            self.request.save()
+        return True
+
+
+class Timer:
+    """Context manager for measuring time"""
+    def __init__(self):
+        self.duration = 0
+        self.start = 0
+        self.end = 0
+
+    def __enter__(self):
+        self.start = time()
+
+    def __exit__(self, exit_type, value, traceback):
+        self.end = time()
+        self.duration = self.end - self.start

pdf/management/commands/generate_pdf.py

@@ -42,7 +42,7 @@ def handle(self, *args, **options):
         if all_requests:
             objects = [generate_new_pdf_request(category) for category in Category.objects.filter(generate_pdf=True)]
         else:
-            objects = PDFRequest.objects.filter(status=Status.QUEUED)
+            objects = PDFRequest.objects.filter(status__in={Status.QUEUED, Status.SCHEDULED})
             if requests:
                 objects = objects[:requests]
 

pdf/migrations/0020_remove_pdfrequest_created_date_and_more.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.1.10 on 2023-09-09 18:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('pdf', '0019_pdfrequest_public'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='pdfrequest',
+            name='created_date',
+        ),
+        migrations.AddField(
+            model_name='pdfrequest',
+            name='scheduled_at',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='pdfrequest',
+            name='status',
+            field=models.CharField(choices=[('QU', 'Queued'), ('SC', 'Scheduled'), ('PR', 'In progress'), ('DO', 'Done'), ('FA', 'Failed')], default='QU', max_length=2),
+        ),
+    ]

pdf/models/request.py

@@ -3,7 +3,7 @@
 
 from django.core.validators import MinValueValidator
 from django.db.models import Model, DateField, DateTimeField, IntegerField, FileField, CharField, TextChoices, \
-    ManyToManyField, ForeignKey, CASCADE, SET_NULL, CheckConstraint, Q, PositiveIntegerField
+    ManyToManyField, ForeignKey, CASCADE, SET_NULL, CheckConstraint, Q, PositiveIntegerField, TextField
 from django.utils.translation import gettext_lazy as _
 
 from backend.models import Song
@@ -23,14 +23,15 @@ class RequestType(TextChoices):
 class Status(TextChoices):
     """Status of PDF Request"""
     QUEUED = "QU", _('Queued')
+    SCHEDULED = "SC", _('Scheduled')
     IN_PROGRESS = "PR", _('In progress')
     DONE = "DO", _("Done")
     FAILED = "FA", _("Failed")
 
 
 class PDFRequest(PDFOptions):
     """Request for PDF generation"""
-    created_date = DateField(auto_now_add=True, editable=False)
+#    created_date = DateField(auto_now_add=True, editable=False)
     update_date = DateTimeField(auto_now=True)
     type = CharField(
         max_length=2,
@@ -47,6 +48,7 @@ class PDFRequest(PDFOptions):
     file = FileField(null=True, storage=fs)
     songs = ManyToManyField(Song, through="PDFSong")
     category = ForeignKey(Category, null=True, on_delete=SET_NULL)
+    scheduled_at = DateTimeField(null=True)
 
     def get_songs(self) -> List[Song]:
         """Returns all songs for request"""

pdf/templates/pdf/requests/list.html

@@ -6,12 +6,22 @@
 {% block header %} {% trans "PDF Requests" %}  {% endblock %}
 
 {% block extra_head %}
-<script defer type="text/javascript" src="https://cdn.datatables.net/v/bs4/dt-1.12.1/datatables.min.js"></script>
-<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/bs4/dt-1.12.1/datatables.min.css"/>
+<script defer type="text/javascript" src="https://cdn.datatables.net/v/bs4/dt-1.13.6/datatables.min.js"></script>
+<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/bs4/dt-1.13.6/datatables.min.css"/>
 {% endblock %}
 
 {% block framed_body %}
-     <div class="input-group">
+    <div class="btn-group btn-group-toggle mb-2 float-left" data-toggle="buttons">
+      <label class="btn btn-outline-info">
+        <input type="radio" name="options" value="{% trans "Manual" %}" checked> {% trans "Manual" %}
+      </label>
+
+      <label class="btn btn-outline-info active">
+        <input type="radio" name="options" value="{% trans "Automated" %}"> {% trans "Automated" %}
+      </label>
+    </div>
+
+    <div class="input-group">
       <div class="input-group-prepend">
         <div class="input-group-text">{% trans "Search" %}</div>
       </div>
@@ -22,8 +32,6 @@
           <thead class="thead-light">
             <tr>
               <th scope="col">{% trans "Title" %}</th>
-    {#          <th scope="col">{% trans "Filename" %}</th>#}
-              <th scope="col">{% trans "Date Created" %}</th>
               <th scope="col">{% trans "Last updated" %}</th>
               <th scope="col">{% trans "Type" %}</th>
               <th scope="col">{% trans "Status" %}</th>
@@ -38,17 +46,9 @@
             {% for request in requests %}
             <tr>
               <td>{{ request.title }}</td>
-    {#          <td>#}
-    {#              {% if request.filename %}#}
-    {#                {{ request.filename }}#}
-    {#              {% else %}#}
-    {#                {% trans "Automatic" %}#}
-    {#              {% endif %}#}
-    {#          </td>#}
-              <td data-order="{{ request.created_date|date:"c" }}">{{ request.created_date}}</td>
               <td data-order="{{ request.update_date|date:"c" }}">{{ request.update_date }}</td>
               <td>{{ request.get_type_display }}</td>
-              <td style="background-color: {{ request.status|get_status_color }}">{{ request.get_status_display }} {% if request.status == "PR" %}{{ request.progress }}/7 {% endif %}</td>
+              <td style="background-color: {{ request.status|get_status_color }}">{{ request.get_status_display }} {% if request.status == "PR" %}{{ request.progress }}/7 {% endif %} {% if request.status == "SC" %} {{ request.scheduled_at }} {% endif %}</td>
               <td>
                   {% if request.file %}
                     <a href="{{ request.file.url }}">{{ request.file|filename }}</a>
@@ -84,18 +84,27 @@
         </table>
     </div>
     <script type="module">
+    const initial_value = $("input[type=radio][name=options]").val()
     const table = $('#datatable').DataTable({
         processing: true,
-        order: [[ 2, "desc" ]],
+        order: [[ 1, "desc" ]],
         dom: 'rtip',
+        searchCols: [
+          null,
+          null,
+          { "search": initial_value},
+        ],
         {% if request.LANGUAGE_CODE == "cs"  %}
             language: {
-                url: "//cdn.datatables.net/plug-ins/1.12.1/i18n/Czech.json"
+                url: "//cdn.datatables.net/plug-ins/1.13.6/i18n/cs.json"
             },
         {% endif %}
     });
     $("#searchInput").on("input", function (event) {
         table.search($(this).val()).draw();
     });
+    $("input[type=radio][name=options]").change(function() {
+        table.column(2).search(this.value).draw()
+    });
     </script>
 {% endblock %}

pdf/templatetags/types.py

@@ -11,7 +11,7 @@
 @register.filter(is_safe=True)
 def get_status_color(status):
     """Converts status to color"""
-    if status == str(Status.QUEUED):
+    if status == str(Status.QUEUED) or status == str(Status.SCHEDULED):
         return "yellow"
     if status == str(Status.IN_PROGRESS):
         return "blue"

pdf/utils.py

@@ -1,43 +1,26 @@
 """Utility functions"""
-import logging
-import re
-from time import time
+from datetime import timedelta
+from django.utils import timezone
 
-from django.conf import settings
 from django.db import transaction
 from django.utils import translation
 from django.utils.translation import gettext
 
+from pdf.generate import schedule_generation
 from pdf.models.request import PDFRequest, RequestType, Status, PDFSong
 
 
-class ProgressFilter(logging.Filter):
-    """Filters Weasyprint progress messages, highly dependent on implementation!"""
-    STEP_NUMBER = re.compile(r"(?<=Step\s)\d")
-
-    def __init__(self, request):
-        super().__init__()
-        self.request = request
-
-    def filter(self, record):
-        step = self.STEP_NUMBER.search(record.getMessage()).group(0)
-        if self.request.progress != step:
-            self.request.progress = step
-            self.request.save()
-        return True
-
-
 def request_pdf_regeneration(category, update: bool = False):
     """Requests automatic PDF regeneration if none is pending"""
-    objects = PDFRequest.objects.filter(type=RequestType.EVENT, status=Status.QUEUED, category=category)
+    objects = PDFRequest.objects.filter(type=RequestType.EVENT, status=Status.SCHEDULED, category=category)
     if not objects.exists():
         generate_new_pdf_request(category)
     elif not update:
         regenerate_pdf_request(objects.first(), category)
 
 
 def regenerate_pdf_request(request, category):
-    """Regenerates the PDF request with newest info"""
+    """Regenerates the PDF request with the newest info"""
     with transaction.atomic():
         PDFSong.objects.filter(request=request).delete()
         PDFSong.objects.bulk_create([
@@ -50,10 +33,12 @@ def regenerate_pdf_request(request, category):
 
 
 def generate_new_pdf_request(category):
-    """Returns PDFRequest for basic"""
+    """Returns PDFRequest for a category"""
     with transaction.atomic():
+        scheduled_times = PDFRequest.objects.filter(status=Status.SCHEDULED, type=RequestType.EVENT).values_list(
+            'scheduled_at', flat=True)
         request = PDFRequest(type=RequestType.EVENT,
-                             status=Status.QUEUED,
+                             status=Status.SCHEDULED,
                              category=category)
         request.copy_options(category)
         request.filename = request.filename or get_filename(category)
@@ -64,32 +49,34 @@ def generate_new_pdf_request(category):
                     song_number=song_number + 1)
             for song_number, song in enumerate(category.song_set.filter(archived=False).all())
         ])
+
+        time = generate_unique_time(scheduled_times, timedelta(minutes=30))
+
+        schedule_generation(request, time)
+        request.scheduled_at = time
+        request.save()
+
         return request
 
+def generate_unique_time(times, delta: timedelta):
+    """Generate unique time, so jobs will not clash on scheduler"""
+    time = timezone.now() + delta
+    for _ in range(len(times) + 1):
+
+        valid = True
+        for existing_time in times:
+            if not existing_time:
+                continue
+            if time - existing_time < delta:
+                valid = False
+
+        if valid:
+            return time
+        time = time + delta
+    raise AttributeError("Unable to assign unique generation time")
 
 def get_filename(category):
     """Returns filename for category based on its locale"""
     with translation.override(category.locale):
         text = gettext('songbook')
         return f"{text}-{category.name}"
-
-
-def get_name(category):
-    """Returns filename for category based on its locale"""
-    with translation.override(category.locale):
-        return gettext(settings.SITE_NAME)
-
-
-class Timer:
-    """Context manager for measuring time"""
-    def __init__(self):
-        self.duration = 0
-        self.start = 0
-        self.end = 0
-
-    def __enter__(self):
-        self.start = time()
-
-    def __exit__(self, exit_type, value, traceback):
-        self.end = time()
-        self.duration = self.end - self.start

pdf/views.py

@@ -42,8 +42,9 @@ def get(self, request, pk):
             return redirect("pdf:list")
         obj.status = Status.QUEUED
         obj.save()
+        generate_pdf_job.delay(obj)
 
-        messages.success(request, _("Request %(id)s was marked for regeneration") % {"id": obj.id})
+        messages.success(request, _("Request %(id)s was scheduled for regeneration") % {"id": obj.id})
         return redirect("pdf:list")