Disables pagination in metacore for empty query: seems to be much faster

metacore/services.py

@@ -30,31 +30,25 @@ def get_crossref_test(cursor='*'):
 
         params = {'cursor': cursor, 'rows': rows, 'mailto': 'b.g.t.ponsioen@uva.nl'}
         last_cursor = cursor
-        for j in range(0,2):
-            r = requests.get(url, params=params)
-            r_json = r.json()
-
-            citables_json = r_json['message']['items']
-            cursor = r_json['message']['next-cursor']
-            number_of_results = len(r_json['message']['items'])
-
-            citables = [parse_crossref_citable(it) for it in citables_json]
-            # Parser returns None if there's an error
-            errors = any([not i for i in citables if i == False])
-            orig_citables = citables
-            citables = [citable for citable in citables if citable]
-
-            # Mass insert in database (will fail on encountering existing documents
-            # with same DOI
-            if citables:
-                Citable.objects.insert(citables)
-                break
-            elif errors:
-                print("Trying again")
-            else:
-                break
+        r = requests.get(url, params=params)
+        r_json = r.json()
+
+        citables_json = r_json['message']['items']
+        cursor = r_json['message']['next-cursor']
+        number_of_results = len(r_json['message']['items'])
+
+        citables = [parse_crossref_citable(it) for it in citables_json]
+        # Parser returns None if there's an error
+        errors = any([not i for i in citables if i == False])
+        orig_citables = citables
+        citables = [citable for citable in citables if citable]
+
+        # Mass insert in database (will fail on encountering existing documents
+        # with same DOI
+        if citables:
+            Citable.objects.insert(citables)
 
-            citable = []
+        citable = []
 
         if number_of_results < rows:
             print(number_of_results)

metacore/views.py

 from django.shortcuts import render
 from django.views.generic.list import ListView
 from django.utils import timezone
+from django.core.paginator import Paginator
 
 from .models import Citable
 from .forms import CitableSearchForm
@@ -19,7 +20,7 @@ class CitableListView(ListView):
             queryset = self.form.search_results()
         else:
             # queryset = Citable.objects.simple().limit(100)
-            queryset = Citable.objects.simple().order_by('-metadata.is-referenced-by-count')
+            queryset = Citable.objects.simple().order_by('-metadata.is-referenced-by-count').limit(10)
 
         return queryset
 
@@ -36,3 +37,18 @@ class CitableListView(ListView):
             context['browse'] = True
 
         return context
+
+    def get_paginate_by(self, queryset):
+        """
+        Dynamically compute pagination setting.
+
+        Can be used to disable pagination on 'empty' search -> manually doing .limit(N) seems
+        to be much faster with mongoengine than Django's pagination
+
+        Also you can add an extra parameter to specify pagination size, like so:
+            return self.request.GET.get('paginate_by', self.paginate_by)
+        """
+        if self.request.GET:
+            return self.paginate_by
+        else:
+            return None