From 04b21ce1fbde1e3d109d4fd3de46d2f700f36dd9 Mon Sep 17 00:00:00 2001
From: Jorran de Wit <jorrandewit@outlook.com>
Date: Mon, 19 Nov 2018 10:44:08 +0100
Subject: [PATCH] Add police
---
scipost/management/commands/check_celery.py | 40 +++++++++++++++++++++
1 file changed, 40 insertions(+)
create mode 100644 scipost/management/commands/check_celery.py
diff --git a/scipost/management/commands/check_celery.py b/scipost/management/commands/check_celery.py
new file mode 100644
index 000000000..ac932aa2c
--- /dev/null
+++ b/scipost/management/commands/check_celery.py
@@ -0,0 +1,40 @@
+__copyright__ = "Copyright 2016-2018, Stichting SciPost (SciPost Foundation)"
+__license__ = "AGPL v3"
+
+
+import datetime
+
+from django.core import mail
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from django_celery_results.models import TaskResult
+
+
+class Command(BaseCommand):
+ help = 'Check if Celery is still running, or at least not failing.'
+
+ def handle(self, *args, **kwargs):
+ # check failed.
+ compare_dt = timezone.now() - datetime.timedelta(hours=1)
+ results_failed = TaskResult.objects.filter(
+ status='FAILURE', date_done__gt=compare_dt).values_list('id', flat=True)
+ if results_failed:
+ # Mail failed
+ body = 'Celery has failed task results. Failed IDs: {}'.format(
+ ', '.join(results_failed))
+ mail.mail_admins('Celery failed', body)
+ self.stdout.write(
+ self.style.SUCCESS('Celery failed, IDs: {}.'.format(', '.join(results_failed))))
+ else:
+ last_result = TaskResult.objects.filter(
+ date_done__gt=compare_dt).order_by('date_done').last()
+ if last_result and last_result.date_done < compare_dt:
+ # Mail inactive
+ body = 'No results for Celery found. Celery seems to be inactive.'
+ body += ' Last result ID: {}'.format(last_result.id)
+ mail.mail_admins('Celery inactive', body)
+ self.stdout.write(
+ self.style.SUCCESS('Celery inactive, last ID: {}.'.format(last_result.id)))
+ if not results_failed and not last_result:
+ self.stdout.write(self.style.SUCCESS('Celery alive!'))
--
GitLab