Switches to Celery for async metacore tasks backend. Still in experimental phase

SciPost_v1/__init__.py

+from __future__ import absolute_import, unicode_literals
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ('celery_app')

SciPost_v1/celery.py

+from __future__ import absolute_import, unicode_literals
+import os
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'SciPost_v1.settings')
+
+app = Celery('SciPost_v1')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+# app.conf.broker_url = 'redis://localhost:6379/0'
+# app.conf.result_backend = 'redis://localhost:6379/0'
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+# app.autodiscover_tasks(related_name='services')
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))
+

SciPost_v1/settings/base.py

@@ -78,36 +78,37 @@ INSTALLED_APPS = (
     'django_countries',
     'django_extensions',
     'django_mathjax',
+    'affiliations',
     'ajax_select',
+    'background_task',
     'captcha',
-    'guardian',
-    'haystack',
-    'rest_framework',
-    'sphinxdoc',
-    'affiliations',
     'colleges',
     'commentaries',
     'comments',
+    'django_celery_results',
     'finances',
+    'funders',
+    'guardian',
+    'haystack',
     'invitations',
     'journals',
-    'mails',
     'mailing_lists',
+    'mails',
+    'metacore',
     'news',
     'notifications',
+    'partners',
+    'petitions',
+    'proceedings',
+    'production',
+    'rest_framework',
     'scipost',
+    'sphinxdoc',
+    'stats',
     'submissions',
     'theses',
     'virtualmeetings',
-    'proceedings',
-    'production',
-    'partners',
-    'funders',
-    'stats',
-    'petitions',
     'webpack_loader',
-    'metacore',
-    'background_task',
 )
 
 
@@ -366,3 +367,7 @@ LOGGING = {
         },
     },
 }
+
+# Celery extra config
+CELERY_RESULT_BACKEND = 'django-db'
+CELERY_BROKER_URL = 'amqp://localhost'

metacore/admin.py

@@ -13,7 +13,9 @@ class JournalAdmin(admin.ModelAdmin):
         """ Starts background task to import all works by this journal """
 
         for journal in queryset:
-            t = import_journal_full(journal.ISSN_digital)
+            # Celery Async version
+            t = import_journal_full.delay(journal.ISSN_digital)
+
             messages.add_message(request, messages.INFO, 'Import task for journal {} added. Go to Background Tasks -> Tasks in admin to view'.format(journal.name))
 
         messages.add_message(request, messages.WARNING, 'Make sure that "./manage.py process_tasks" is running (otherwise start it).')

metacore/services.py

+from __future__ import absolute_import, unicode_literals
 import requests
 from .models import Citable, CitableWithDOI, Journal
 from background_task import background
@@ -5,10 +6,12 @@ from rest_framework import serializers
 from mongoengine.python_support import pymongo
 from django.utils import timezone
 import logging
+from celery import shared_task
 
 logger = logging.getLogger(__name__)
 
-@background()
+# @background()
+@shared_task
 def import_journal_full(issn, cursor='*'):
     """
     Task to query CrossRef for all works of a journal with given ISSN
@@ -16,7 +19,8 @@ def import_journal_full(issn, cursor='*'):
     """
     import_journal(issn=issn, cursor=cursor, from_index_date=None)
 
-@background()
+# @background()
+@shared_task
 def import_journal_incremental(issn, from_index_date, cursor='*'):
     """
     Task to query CrossRef for all works of a journal with given ISSN

metacore/tasks.py

+from __future__ import absolute_import, unicode_literals
+from celery import shared_task
+
+@shared_task
+def test_celery_task():
+    return 100

requirements.txt

@@ -52,6 +52,9 @@ html2text
 # Mongo (Metacore)
 mongoengine==0.15.0
 django-background-tasks==1.1.13
+celery==4.1.1
+django-celery-results==1.0.1
+flower==0.9.2
 
 # Possibly dead (most probably not used anymore and possibly not up-to-date packages)  -- JdW (August 15th, 2017)
 imagesize==0.7.1