From 728b537daea155a69e85e9e92cda8397e819a83e Mon Sep 17 00:00:00 2001
From: Boris Ponsioen <b.g.t.ponsioen@uva.nl>
Date: Mon, 16 Apr 2018 15:29:14 +0200
Subject: [PATCH] Readds error handling in get_crossref_test temporarily
---
metacore/services.py | 39 +++++++++++++++++++++------------------
1 file changed, 21 insertions(+), 18 deletions(-)
diff --git a/metacore/services.py b/metacore/services.py
index 0bb08a22d..bd12be201 100644
--- a/metacore/services.py
+++ b/metacore/services.py
@@ -28,31 +28,34 @@ def get_crossref_test(cursor='*'):
print("Last cursor: ", last_cursor)
print("Current cursor: ", cursor)
- params = {'cursor': cursor, 'rows': rows, 'mailto': 'b.g.t.ponsioen@uva.nl'}
- r = requests.get(url, params=params)
- r_json = r.json()
-
- citables_json = r_json['message']['items']
- last_cursor = cursor
- cursor = r_json['message']['next-cursor']
- number_of_results = len(r_json['message']['items'])
-
- citables = [parse_crossref_citable(it) for it in citables_json]
- citables = [citable for citable in citables if citable is not None]
+ for j in range(0,2):
+ params = {'cursor': cursor, 'rows': rows, 'mailto': 'b.g.t.ponsioen@uva.nl'}
+ r = requests.get(url, params=params)
+ r_json = r.json()
+
+ citables_json = r_json['message']['items']
+ last_cursor = cursor
+ cursor = r_json['message']['next-cursor']
+ number_of_results = len(r_json['message']['items'])
+
+ citables = [parse_crossref_citable(it) for it in citables_json]
+ citables = [citable for citable in citables if citable is not None]
+
+ # Mass insert in database (will fail on encountering existing documents
+ # with same DOI
+ if citables:
+ Citable.objects.insert(citables)
+ break
+ else:
+ print("Trying again")
- # Mass insert in database (will fail on encountering existing documents
- # with same DOI
- if citables:
- Citable.objects.insert(citables)
+ citable = []
if number_of_results < rows:
print(number_of_results)
print('End reached.')
break
- citable = []
-
-
def convert_doi_to_lower_case():
# If you accidentally import 100.000+ records that have random uppercase characters
# in their reference DOI list
--
GitLab