Add class StoredMessageAttachment and related facilities

SciPost_v1/settings/base.py

@@ -336,6 +336,7 @@ EMAIL_SUBJECT_PREFIX = '[SciPost Server] '
 MAILCHIMP_DATABASE_CODE = 'us6'
 MAILCHIMP_API_USER = 'test_API-user'
 MAILCHIMP_API_KEY = 'test_API-key'
+MAX_EMAIL_ATTACHMENT_FILE_SIZE = 20000000
 
 
 # iThenticate

apimail/admin.py

@@ -4,10 +4,22 @@ __license__ = "AGPL v3"
 
 from django.contrib import admin
 
-from .models import Event
+from .models import Event, StoredMessage, StoredMessageAttachment
 
 
 class EventAdmin(admin.ModelAdmin):
     pass
 
 admin.site.register(Event, EventAdmin)
+
+
+class StoredMessageAttachmentInline(admin.StackedInline):
+    model = StoredMessageAttachment
+    extra = 0
+    min_num = 0
+
+
+class StoredMessageAdmin(admin.ModelAdmin):
+    inlines = [StoredMessageAttachmentInline,]
+
+admin.site.register(StoredMessage, StoredMessageAdmin)

apimail/management/commands/mailgun_get_stored_messages.py

@@ -2,13 +2,16 @@ __copyright__ = "Copyright © Stichting SciPost (SciPost Foundation)"
 __license__ = "AGPL v3"
 
 
+from tempfile import TemporaryFile
+
 import requests
 
 from django.conf import settings
+from django.core.files import File
 from django.core.management import BaseCommand
 
 from ...exceptions import APIMailError
-from ...models import Event, StoredMessage
+from ...models import Event, StoredMessage, StoredMessageAttachment
 
 
 class Command(BaseCommand):
@@ -34,3 +37,13 @@ class Command(BaseCommand):
                 sm = StoredMessage.objects.create(data=response)
                 orphan.stored_message = sm
                 orphan.save()
+                # Now deal with attachments
+                for att_item in response['attachments']:
+                    with TemporaryFile() as tf:
+                        r = requests.get(att_item['url'], stream=True)
+                        for chunk in r.iter_content(chunk_size=8192):
+                            tf.write(chunk)
+                        tf.seek(0)
+                        sma = StoredMessageAttachment.objects.create(
+                            message=sm, data=att_item)
+                        sma._file.save(att_item['name'], File(tf))

apimail/migrations/0003_auto_20191113_2226.py

+# Generated by Django 2.1.8 on 2019-11-13 21:26
+
+import apimail.validators
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import scipost.storage
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('apimail', '0002_auto_20191113_1547'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='StoredMessageAttachment',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('data', django.contrib.postgres.fields.jsonb.JSONField(default=dict)),
+                ('_file', models.FileField(storage=scipost.storage.SecureFileStorage(), upload_to='uploads/mail/stored_messages/attachments/%Y/%m/%d/', validators=[apimail.validators.validate_max_email_attachment_file_size])),
+            ],
+        ),
+        migrations.AlterModelOptions(
+            name='storedmessage',
+            options={'ordering': ['-data__Date']},
+        ),
+        migrations.AddField(
+            model_name='storedmessageattachment',
+            name='message',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='attachments', to='apimail.StoredMessage'),
+        ),
+    ]

apimail/models/__init__.py

@@ -4,4 +4,4 @@ __license__ = "AGPL v3"
 
 from .event import Event
 
-from .stored_message import StoredMessage
+from .stored_message import StoredMessage, StoredMessageAttachment

apimail/models/stored_message.py

@@ -8,6 +8,10 @@ from django.contrib.postgres.fields import JSONField
 from django.db import models
 from django.urls import reverse
 
+from scipost.storage import SecureFileStorage
+
+from ..validators import validate_max_email_attachment_file_size
+
 
 class StoredMessage(models.Model):
     """
@@ -19,5 +23,21 @@ class StoredMessage(models.Model):
         editable=False)
     data = JSONField(default=dict)
 
+    class Meta:
+        ordering = ['-data__Date',]
+
     def get_absolute_url(self):
         return reverse('apimail:stored_message_detail', kwargs={'uuid': self.uuid})
+
+
+class StoredMessageAttachment(models.Model):
+    message = models.ForeignKey(
+        'apimail.StoredMessage',
+        on_delete=models.CASCADE,
+        related_name='attachments' # doesn't collide with StoredMessage.data.attachments
+    )
+    data = JSONField(default=dict)
+    _file = models.FileField(
+        upload_to='uploads/mail/stored_messages/attachments/%Y/%m/%d/',
+        validators=[validate_max_email_attachment_file_size,],
+        storage=SecureFileStorage())

apimail/validators.py

+__copyright__ = "Copyright © Stichting SciPost (SciPost Foundation)"
+__license__ = "AGPL v3"
+
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.template.defaultfilters import filesizeformat
+
+
+def validate_max_email_attachment_file_size(value):
+    if value.size > int(settings.MAX_EMAIL_ATTACHMENT_FILE_SIZE):
+        raise ValidationError(
+            'Please keep filesize under %s. Current filesize: %s' % (
+                filesizeformat(settings.MAX_EMAIL_ATTACHMENT_FILE_SIZE),
+                filesizeformat(value.size))
+        )