optimize profile list and duplicates view

scipost_django/profiles/managers.py

@@ -46,36 +46,29 @@ class ProfileQuerySet(QuerySet):
         last names, and separately by (case-insensitive) email).
         """
         # Start by treating name duplicates, excluding marked Profile non-duplicates
-        from .models import ProfileNonDuplicates
+        from .models import ProfileEmail
+
+        profiles = self.with_full_names().filter(profilenonduplicates__isnull=True)
 
-        profiles = self.with_full_names().exclude(
-            id__in=ProfileNonDuplicates.objects.values_list("profiles", flat=True)
-        )
         duplicates_by_full_name = (
             profiles.values("full_name_annot")
             .annotate(nr_count=Count("full_name_annot"))
             .filter(nr_count__gt=1)
-            .values_list("full_name_annot", flat=True)
+            .values("full_name_annot")
         )
-        from .models import ProfileEmail
 
-        # Now for email duplicates. Because of case-insensitivity, we need some gymnastics
-        pel = ProfileEmail.objects.annotate(email_lower=Lower("email"))
-        # Build list of all duplicate lowercased emails
-        duplicate_emails = [
-            pe["email_lower"]
-            for pe in pel.values("email_lower")
-            .annotate(nel=Count("email_lower"))
-            .filter(nel__gt=1)
-        ]
-        # Then determine all ids of related Profiles with an email in this list
-        ids_of_duplicates_by_email = []
-        for email in duplicate_emails:
-            prof_ids = pel.filter(email_lower=email).values_list("profile", flat=True)
-            prof_ids = set(prof_ids)
-            if len(prof_ids) > 1:
-                ids_of_duplicates_by_email.extend(prof_ids)
-        # Now return list of potential duplicates
+        # Find Profiles whose email is used by another Profile
+        ids_of_duplicates_by_email = (
+            ProfileEmail.objects.annotate(
+                used_by_another=Exists(
+                    ProfileEmail.objects.filter(
+                        ~Q(profile=OuterRef("profile")),
+                        email__iexact=OuterRef("email"),
+                    )
+                )
+            ).filter(used_by_another=True)
+        ).values("profile")
+
         return profiles.filter(
             Q(full_name_annot__in=duplicates_by_full_name)
             | Q(id__in=ids_of_duplicates_by_email)

scipost_django/profiles/templates/profiles/profile_list.html

@@ -146,9 +146,10 @@
                 </thead>
 	
                 <tbody>
- 
-                    {% for branch in branches %}
- 
+                      {% regroup academic_fields by branch as branches %}
+
+                      {% for branch, academic_fields_in_branch in branches %}
+
                         <tr>
  
                             <td class="align-middle">
@@ -160,12 +161,12 @@
 	
                                 <ul class="list-inline m-0">
  
-                                    {% for acad_field in branch.academic_fields.all %}
- 
+                                {% for acad_field in academic_fields_in_branch %}
+
                                         <li class="list-inline-item">
  
-                                            {% if acad_field.profiles.all|length > 0 %}
-	
+                                    {% if acad_field.nr_profiles > 0 %}
+
                                                 <div class="dropdown">
  
                                                     <button class="btn btn-sm btn-primary dropdown-toggle"

scipost_django/profiles/views.py

@@ -9,7 +9,7 @@ from django.core.exceptions import BadRequest
 from django.template.response import TemplateResponse
 from django.urls import reverse, reverse_lazy
 from django.db import transaction
-from django.db.models import Q
+from django.db.models import Q, Count
 from django.http import Http404, HttpResponse
 from django.shortcuts import get_object_or_404, render, redirect
 from django.views.generic.detail import DetailView
@@ -20,6 +20,7 @@ from dal import autocomplete
 from guardian.decorators import permission_required
 from mails.views import MailView
 from common.views import HXDynselAutocomplete
+from ontology.models.academic_field import AcademicField
 from profiles import constants
 
 from scipost.mixins import PermissionsMixin, PaginationMixin
@@ -264,7 +265,11 @@ class ProfileListView(PermissionsMixin, PaginationMixin, ListView):
         """
         Return a queryset of Profiles using optional GET data.
         """
-        queryset = Profile.objects.all()
+        queryset = (
+            Profile.objects.all()
+            .prefetch_related("specialties")
+            .select_related("contributor", "contributor__user")
+        )
         if self.request.GET.get("field"):
             queryset = queryset.filter(acad_field__slug=self.request.GET["field"])
             if self.request.GET.get("specialty"):
@@ -291,11 +296,19 @@ class ProfileListView(PermissionsMixin, PaginationMixin, ListView):
         refinv_wo_profile = RefereeInvitation.objects.filter(profile__isnull=True)
         reginv_wo_profile = RegistrationInvitation.objects.filter(profile__isnull=True)
 
+        academic_fields = (
+            AcademicField.objects.annotate(nr_profiles=Count("profiles"))
+            .prefetch_related("specialties")
+            .prefetch_related("branch")
+            .order_by("branch")
+        )
+
         context.update(
             {
                 "searchform": SearchTextForm(
                     initial={"text": self.request.GET.get("text")}
                 ),
+                "academic_fields": academic_fields,
                 "nr_contributors_w_duplicate_emails": contributors_w_duplicate_email.count(),
                 "nr_contributors_w_duplicate_names": contributors_w_duplicate_names.count(),
                 "nr_contributors_wo_profile": contributors_wo_profile.count(),