From 95dfb5c78b790752c6e9243709073dd5112667a9 Mon Sep 17 00:00:00 2001
From: Boris Ponsioen <b.g.t.ponsioen@uva.nl>
Date: Mon, 16 Apr 2018 17:53:31 +0200
Subject: [PATCH] Disables pagination in metacore for empty query: seems to be
much faster
---
metacore/services.py | 42 ++++++++++++++++++------------------------
metacore/views.py | 18 +++++++++++++++++-
2 files changed, 35 insertions(+), 25 deletions(-)
diff --git a/metacore/services.py b/metacore/services.py
index 9f3b858d9..8ad65b969 100644
--- a/metacore/services.py
+++ b/metacore/services.py
@@ -30,31 +30,25 @@ def get_crossref_test(cursor='*'):
params = {'cursor': cursor, 'rows': rows, 'mailto': 'b.g.t.ponsioen@uva.nl'}
last_cursor = cursor
- for j in range(0,2):
- r = requests.get(url, params=params)
- r_json = r.json()
-
- citables_json = r_json['message']['items']
- cursor = r_json['message']['next-cursor']
- number_of_results = len(r_json['message']['items'])
-
- citables = [parse_crossref_citable(it) for it in citables_json]
- # Parser returns None if there's an error
- errors = any([not i for i in citables if i == False])
- orig_citables = citables
- citables = [citable for citable in citables if citable]
-
- # Mass insert in database (will fail on encountering existing documents
- # with same DOI
- if citables:
- Citable.objects.insert(citables)
- break
- elif errors:
- print("Trying again")
- else:
- break
+ r = requests.get(url, params=params)
+ r_json = r.json()
+
+ citables_json = r_json['message']['items']
+ cursor = r_json['message']['next-cursor']
+ number_of_results = len(r_json['message']['items'])
+
+ citables = [parse_crossref_citable(it) for it in citables_json]
+ # Parser returns None if there's an error
+ errors = any([not i for i in citables if i == False])
+ orig_citables = citables
+ citables = [citable for citable in citables if citable]
+
+ # Mass insert in database (will fail on encountering existing documents
+ # with same DOI
+ if citables:
+ Citable.objects.insert(citables)
- citable = []
+ citable = []
if number_of_results < rows:
print(number_of_results)
diff --git a/metacore/views.py b/metacore/views.py
index b7fe95759..712de954d 100644
--- a/metacore/views.py
+++ b/metacore/views.py
@@ -1,6 +1,7 @@
from django.shortcuts import render
from django.views.generic.list import ListView
from django.utils import timezone
+from django.core.paginator import Paginator
from .models import Citable
from .forms import CitableSearchForm
@@ -19,7 +20,7 @@ class CitableListView(ListView):
queryset = self.form.search_results()
else:
# queryset = Citable.objects.simple().limit(100)
- queryset = Citable.objects.simple().order_by('-metadata.is-referenced-by-count')
+ queryset = Citable.objects.simple().order_by('-metadata.is-referenced-by-count').limit(10)
return queryset
@@ -36,3 +37,18 @@ class CitableListView(ListView):
context['browse'] = True
return context
+
+ def get_paginate_by(self, queryset):
+ """
+ Dynamically compute pagination setting.
+
+ Can be used to disable pagination on 'empty' search -> manually doing .limit(N) seems
+ to be much faster with mongoengine than Django's pagination
+
+ Also you can add an extra parameter to specify pagination size, like so:
+ return self.request.GET.get('paginate_by', self.paginate_by)
+ """
+ if self.request.GET:
+ return self.paginate_by
+ else:
+ return None
--
GitLab