add bulk-email capability to Maillog

scipost_django/SciPost_v1/settings/base.py

@@ -423,6 +423,7 @@ WEBPACK_LOADER = {
 }
 
 # Email
+BULK_EMAIL_THROTTLE = 5
 EMAIL_BACKEND = "mails.backends.filebased.EmailBackend"
 EMAIL_FILE_PATH = "local_files/email/"
 EMAIL_SUBJECT_PREFIX = "[SciPost Server] "

scipost_django/mails/admin.py

@@ -19,8 +19,8 @@ class MailLogRelationInline(admin.TabularInline):
 
 @admin.register(MailLog)
 class MailLogAdmin(admin.ModelAdmin):
-    list_display = ["__str__", "to_recipients", "created", "status"]
-    list_filter = ["status"]
+    list_display = ["__str__", "to_recipients", "created", "status", "type"]
+    list_filter = ["status", "type"]
     readonly_fields = ["created", "latest_activity"]
     search_fields = [
         "to_recipients",

scipost_django/mails/management/commands/send_mails.py

+from typing import Iterable
+
+from django.core.mail import get_connection
 from django.core.management.base import BaseCommand
 from django.conf import settings
 
 from ...core import MailEngine
 from ...models import MailLog
 
+BULK_EMAIL_THROTTLE = getattr(settings, "BULK_EMAIL_THROTTLE", 5)
+
+if hasattr(settings, "EMAIL_BACKEND_ORIGINAL"):
+    backend = settings.EMAIL_BACKEND_ORIGINAL
+else:
+    # Fallback to Django's default
+    backend = "django.core.mail.backends.smtp.EmailBackend"
+
+if backend == "mails.backends.filebased.ModelEmailBackend":
+    raise AssertionError("The `EMAIL_BACKEND_ORIGINAL` cannot be the ModelEmailBackend")
+
+connection = get_connection(backend=backend, fail_silently=False)
+
 
 class Command(BaseCommand):
     """
@@ -18,7 +34,7 @@ class Command(BaseCommand):
             help="The id in the `MailLog` table for a specific mail, Leave blank to send all",
         )
 
-    def _process_mail(self, mail):
+    def _render_mail(self, mail):
         """
         Render the templates for the mail if not done yet.
         """
@@ -29,51 +45,90 @@ class Command(BaseCommand):
             body=message, body_html=html_message, status="rendered"
         )
 
-    def send_mails(self, mails):
-        from django.core.mail import get_connection, EmailMultiAlternatives
-
-        if hasattr(settings, "EMAIL_BACKEND_ORIGINAL"):
-            backend = settings.EMAIL_BACKEND_ORIGINAL
-        else:
-            # Fallback to Django's default
-            backend = "django.core.mail.backends.smtp.EmailBackend"
+    def _send_mail(self, connection, mail_log: "MailLog", recipients, **kwargs):
+        """
+        Build and send the mail, returning the response.
+        """
+        from django.core.mail import EmailMultiAlternatives
 
-        if backend == "mails.backends.filebased.ModelEmailBackend":
-            raise AssertionError(
-                "The `EMAIL_BACKEND_ORIGINAL` cannot be the ModelEmailBackend"
-            )
+        mail = EmailMultiAlternatives(
+            mail_log.subject,
+            mail_log.body,
+            mail_log.from_email,
+            recipients,
+            connection=connection,
+            cc=mail_log.cc_recipients,
+            bcc=mail_log.bcc_recipients,
+            reply_to=(mail_log.from_email,),
+            **kwargs,
+        )
+        mail.attach_alternative(mail_log.body_html, "text/html")
+        response = mail.send()
+        return response
 
-        connection = get_connection(backend=backend, fail_silently=False)
+    def process_mail_logs(self, mail_logs: "Iterable[MailLog]"):
+        """
+        Process the MailLogs according to their type and send the mails.
+        """
         count = 0
-        for db_mail in mails:
-            if db_mail.status == "not_rendered":
-                self._process_mail(db_mail)
-                db_mail.refresh_from_db()
-
-            mail = EmailMultiAlternatives(
-                db_mail.subject,
-                db_mail.body,
-                db_mail.from_email,
-                db_mail.to_recipients,
-                cc=db_mail.cc_recipients,
-                bcc=db_mail.bcc_recipients,
-                reply_to=(db_mail.from_email,),
-                connection=connection,
-            )
-            if db_mail.body_html:
-                mail.attach_alternative(db_mail.body_html, "text/html")
-            response = mail.send()
-            if response:
-                count += 1
-                db_mail.processed = True
-                db_mail.status = "sent"
-                db_mail.save()
+        for mail_log in mail_logs:
+            if mail_log.status == "not_rendered":
+                self._render_mail(mail_log)
+                mail_log.refresh_from_db()
+
+            # If single entry per mail, send regularly
+            if mail_log.type == mail_log.TYPE_SINGLE:
+                response = self._send_mail(
+                    connection,
+                    mail_log,
+                    mail_log.to_recipients,
+                )
+
+                if response:
+                    count += 1
+                    mail_log.status = "sent"
+                    mail_log.processed = True
+
+            # If bulk, build separate instances of the mail
+            # for each recipient up to the throttle limit per run
+            elif mail_log.type == mail_log.TYPE_BULK:
+                if mail_log.sent_to is None:
+                    mail_log.sent_to = []
+                if mail_log.to_recipients is None:
+                    self.stdout.write(
+                        "MailLog {} has no recipients. Skipping.".format(mail_log.id)
+                    )
+                    continue
+
+                remaining_recipients = [
+                    recipient
+                    for recipient in mail_log.to_recipients
+                    if recipient not in mail_log.sent_to
+                ]
+
+                # Guard against empty recipients, updating the status
+                if not remaining_recipients:
+                    mail_log.status = "sent"
+                    mail_log.processed = True
+                    mail_log.save()
+                    continue
+
+                # For each recipient not yet sent to and up to the throttle limit
+                for recipient in remaining_recipients[:BULK_EMAIL_THROTTLE]:
+                    response = self._send_mail(connection, mail_log, [recipient])
+
+                    if response:
+                        count += 1
+                        mail_log.sent_to.append(recipient)
+
+            mail_log.save()
+
         return count
 
     def handle(self, *args, **options):
         if options.get("id"):
-            mails = MailLog.objects.filter(id=options["id"])
+            mail_logs = MailLog.objects.filter(id=options["id"])
         else:
-            mails = MailLog.objects.not_sent().order_by("created")[:10]
-        nr_mails = self.send_mails(mails)
+            mail_logs = MailLog.objects.not_sent().order_by("created")[:10]
+        nr_mails = self.process_mail_logs(mail_logs)
         self.stdout.write("Sent {} mails.".format(nr_mails))

scipost_django/mails/migrations/0009_maillog_bulk_recipients_maillog_sent_to_maillog_type.py

+# Generated by Django 4.2.10 on 2024-05-21 12:39
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("mails", "0008_maillog_cc_recipients"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="maillog",
+            name="sent_to",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.EmailField(max_length=254),
+                blank=True,
+                null=True,
+                size=None,
+            ),
+        ),
+        migrations.AddField(
+            model_name="maillog",
+            name="type",
+            field=models.CharField(
+                choices=[("single", "Single"), ("bulk", "Bulk")],
+                default="single",
+                max_length=64,
+            ),
+        ),
+    ]

scipost_django/mails/models.py

@@ -24,9 +24,22 @@ class MailLog(models.Model):
     Mails are not directly sent, but added to this table first.
     Using a cronjob, the unsent messages are eventually sent using
     the chosen MailBackend.
+
+    A mail can be of two types:
+    - Single: This log entry represents a single mail (optionally with multiple recipients)
+    - Bulk: This log entry represents an array of (identical) mails,
+            each with a single recipient, e.g. announcements or newsletters.
     """
 
+    TYPE_SINGLE = "single"
+    TYPE_BULK = "bulk"
+    TYPE_CHOICES = (
+        (TYPE_SINGLE, "Single"),
+        (TYPE_BULK, "Bulk"),
+    )
+
     processed = models.BooleanField(default=False)
+    type = models.CharField(max_length=64, choices=TYPE_CHOICES, default=TYPE_SINGLE)
     status = models.CharField(
         max_length=16, choices=MAIL_STATUSES, default=MAIL_RENDERED
     )
@@ -40,6 +53,8 @@ class MailLog(models.Model):
     cc_recipients = ArrayField(models.EmailField(), blank=True, null=True)
     bcc_recipients = ArrayField(models.EmailField(), blank=True, null=True)
 
+    sent_to = ArrayField(models.EmailField(), blank=True, null=True)
+
     from_email = models.CharField(max_length=254, blank=True)
     subject = models.CharField(max_length=254, blank=True)
 
@@ -49,16 +64,31 @@ class MailLog(models.Model):
     objects = MailLogQuerySet.as_manager()
 
     def __str__(self):
-        return "{id}. {subject} ({count} recipients)".format(
+        nr_recipients = self.get_number_of_recipients()
+        if self.type == self.TYPE_SINGLE:
+            recipients_str = f"{nr_recipients} recipients"
+        elif self.type == self.TYPE_BULK:
+            nr_sent = len(self.sent_to) if self.sent_to else 0
+            recipients_str = f"{nr_sent}/{nr_recipients} recipients"
+
+        return "{id}. {subject} ({recipients_str})".format(
             id=self.id,
             subject=self.subject[:30],
-            count=(
-                len(self.to_recipients)
-                + len(self.bcc_recipients)
-                + (len(self.cc_recipients) if self.cc_recipients else 0)
-            ),
+            recipients_str=recipients_str,
         )
 
+    def get_number_of_recipients(self):
+        def sum_optional(*args):
+            return sum([len(arg) for arg in args if arg])
+
+        if self.type == self.TYPE_SINGLE:
+            return sum_optional(
+                self.to_recipients, self.cc_recipients, self.bcc_recipients
+            )
+        elif self.type == self.TYPE_BULK:
+            return sum_optional(self.to_recipients)
+        return 0
+
     def get_full_context(self):
         """Get the full template context needed to render the template."""
         if hasattr(self, "_context"):