Clean up TextOfEmail message texts

esp/esp/dbmail/admin.py

@@ -60,9 +60,19 @@ class MessageRequestAdmin(admin.ModelAdmin):
     date_hierarchy = 'processed_by'
 admin_site.register(MessageRequest, MessageRequestAdmin)
 
+def fill_msgtext(modeladmin, request, queryset):
+    for toe in queryset:
+        toe.fill_msgtext()
+fill_msgtext.short_description = "Fill in message text"
+
+def clear_msgtext(modeladmin, request, queryset):
+    queryset.update(msgtext="")
+clear_msgtext.short_description = "Clear message text"
+
 class TextOfEmailAdmin(admin.ModelAdmin):
     list_display = ('id', 'send_from', 'send_to', 'subject', 'sent', 'user')
     search_fields = ('=id', 'send_from', 'send_to', 'subject', 'user')
     date_hierarchy = 'sent'
     list_filter = ('send_from',)
+    actions = [fill_msgtext, clear_msgtext]
 admin_site.register(TextOfEmail, TextOfEmailAdmin)

esp/esp/dbmail/cronmail.py

@@ -123,6 +123,7 @@ def send_email_requests():
     # So we do our own batching on top of that.
     batch_size = 1000
     for i in xrange(int(math.ceil(float(mailtxts.count()) / batch_size))):
+        # .iterator() re-evaulates the QuerySet each time, moving to the remaining unsent texts
         for mailtxt in mailtxts[:batch_size].iterator():
             exception = mailtxt.send()
             if exception is not None:

esp/esp/dbmail/migrations/0007_textofemail_messagerequest.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2024-05-09 20:45
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import math
+
+def fill_requests(apps, schema_editor):
+    TextOfEmail = apps.get_model('dbmail', 'TextOfEmail')
+    MessageRequest = apps.get_model('dbmail', 'MessageRequest')
+    requests = MessageRequest.objects.all()
+    for req in requests:
+        toes = TextOfEmail.objects.filter(created_at=req.created_at,
+                                          subject = req.subject,
+                                          send_from = req.sender)
+        toes.update(messagerequest=req, msgtext="")
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dbmail', '0006_textofemail_user'),
+    ]
+
+    operations = [
+        # need to add the field with null=True
+        migrations.AddField(
+            model_name='textofemail',
+            name='messagerequest',
+            field=models.ForeignKey(null=True, blank=True, on_delete=django.db.models.deletion.CASCADE, to='dbmail.MessageRequest'),
+        ),
+        # fill in the message request for all texts
+        migrations.RunPython(fill_requests, migrations.RunPython.noop),
+        # now we can make null=False (requiring the field)
+        migrations.AlterField(
+            model_name='textofemail',
+            name='messagerequest',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dbmail.MessageRequest'),
+        ),
+    ]

esp/esp/dbmail/models.py

@@ -372,6 +372,7 @@ def process(self):
                 newemailrequest = {'target': user, 'msgreq': self}
                 send_to = ESPUser.email_sendto_address(*address_pair)
                 newtxt = {
+                    'messagerequest': self,
                     'user': user,
                     'send_to': send_to,
                     'send_from': send_from,
@@ -406,6 +407,7 @@ def process(self):
 
 class TextOfEmail(models.Model):
     """ Contains the processed form of an EmailRequest, ready to be sent.  SmartText becomes plain text. """
+    messagerequest = models.ForeignKey(MessageRequest)
     user = AjaxForeignKey(ESPUser, blank=True, null=True) # blank=True because there isn't an easy way to backfill this
     send_to = models.CharField(max_length=1024)  # Valid email address, "Name" <[email protected]>
     send_from = models.CharField(max_length=1024) # Valid email address
@@ -462,8 +464,18 @@ def send(self):
             return e
         else:
             self.sent = now
+            # clear the msgtext to save DB space
+            # we can always repopulate it using self.fill_msgtext()
+            self.msgtext = ""
             self.save()
 
+    def fill_msgtext(self):
+        """ Repopulate the msgtext based on the messagerequest """
+        msg_req = self.messagerequest
+        msgtext = msg_req.parseSmartText(msg_req.msgtext, self.user)
+        self.msgtext = msgtext
+        self.save()
+
     @classmethod
     def expireUnsentEmails(cls, min_tries=0, orm_class=None):
         """