Optimize queries for participants_data view (#533)

* Small performance changes

* Add select_related()

* Add some management tools

* Remove timing utils

* removing unused code

* remove unused scripts

---------

Co-authored-by: Tomasz Waleń <[email protected]>

Author: 2 people

oioioi/participants/management/commands/dev_generate_random_oi_participants.py

@@ -0,0 +1,98 @@
+import os
+import urllib.request
+import random
+
+from datetime import date, timedelta
+
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand, CommandError
+from django.db import DatabaseError, transaction
+from django.utils.translation import gettext as _
+
+from oioioi.contests.models import Contest
+from oioioi.participants.models import Participant
+from oioioi.oi.models import OIRegistration, School, T_SHIRT_SIZES, CLASS_TYPES
+
+
+class Command(BaseCommand):
+    help = _(
+        "Imports users and adds them as participants to <contest_id>.\n"
+        "The users do not need to be in the database, they will be inserted dynamically.\n"
+        "There should exist some School objects in database. If not, you can generate them with import_schools.py\n"
+        "Each line must have: username first_name last_name (space or comma separated).\n"
+        "Lines starting with '#' are ignored."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument('contest_id', type=str, help='Contest to import to')
+        parser.add_argument('filename_or_url', type=str, help='Source file')
+
+    def handle(self, *args, **options):
+        try:
+            contest = Contest.objects.get(id=options['contest_id'])
+        except Contest.DoesNotExist:
+            raise CommandError(_("Contest %s does not exist") % options['contest_id'])
+
+        arg = options['filename_or_url']
+        if arg.startswith('http://') or arg.startswith('https://'):
+            self.stdout.write(_("Fetching %s...\n") % (arg,))
+            stream = urllib.request.urlopen(arg)
+            stream = (line.decode('utf-8') for line in stream)
+        else:
+            if not os.path.exists(arg):
+                raise CommandError(_("File not found: %s") % arg)
+            stream = open(arg, 'r', encoding='utf-8')
+
+        schools = list(School.objects.all())
+        if not schools:
+            raise CommandError("No schools found in the database.")
+
+        all_count = 0
+        with transaction.atomic():
+            ok = True
+            for line in stream:
+                line = line.strip()
+                if not line or line.startswith('#'):
+                    continue
+
+                parts = line.replace(',', ' ').split()
+                if len(parts) != 3:
+                    self.stdout.write(_("Invalid line format: %s\n") % line)
+                    ok = False
+                    continue
+
+                username, first_name, last_name = parts
+
+                try:
+                    user, created = User.objects.get_or_create(username=username)
+                    if created:
+                        user.first_name = first_name
+                        user.last_name = last_name
+                        user.set_unusable_password()
+                        user.save()
+
+                    Participant.objects.get_or_create(contest=contest, user=user)
+                    participant, _ = Participant.objects.get_or_create(contest=contest, user=user)
+
+                    OIRegistration.objects.create(
+                        participant=participant,
+                        address=f"ulica {random.randint(1, 100)}",
+                        postal_code=f"{random.randint(10, 99)}-{random.randint(100, 999)}",
+                        city=f"Miasto{random.randint(1, 50)}",
+                        phone=f"+48 123 456 {random.randint(100, 999)}",
+                        birthday=date.today() - timedelta(days=random.randint(5000, 8000)),
+                        birthplace=f"Miejsce{random.randint(1, 100)}",
+                        t_shirt_size=random.choice(T_SHIRT_SIZES)[0],
+                        class_type=random.choice(CLASS_TYPES)[0],
+                        school=random.choice(schools),
+                        terms_accepted=True,
+                    )
+                    all_count += 1
+                except DatabaseError as e:
+                    self.stdout.write(_("DB Error for user=%s: %s\n") % (username, str(e)))
+                    ok = False
+
+            if ok:
+                print(f"Successfully processed {all_count} entries.")
+            else:
+                raise CommandError(_("There were some errors. Database not changed."))

oioioi/participants/utils.py

@@ -2,6 +2,10 @@
 from django.contrib.auth.models import User
 from django.http import HttpResponse
 from django.utils.encoding import force_str
+from django.core.exceptions import ObjectDoesNotExist
+from django.db.models.fields.related import ForeignKey, OneToOneField
+
+from collections import deque
 
 from oioioi.base.permissions import make_request_condition
 from oioioi.base.utils import request_cached
@@ -90,7 +94,8 @@ def _fold_registration_models_tree(object):
     the object, gets models related to the model and lists
     all their fields."""
     result = []
-    objects_used = [object]
+    objects_used = set()
+    objects_used.add(object)
 
     # https://docs.djangoproject.com/en/1.9/ref/models/meta/#migrating-old-meta-api
     def get_all_related_objects(_meta):
@@ -100,16 +105,16 @@ def get_all_related_objects(_meta):
             if (f.one_to_many or f.one_to_one) and f.auto_created and not f.concrete
         ]
 
-    objs = [
-        getattr(object, rel.get_accessor_name())
-        for rel in get_all_related_objects(object._meta)
-        if hasattr(object, rel.get_accessor_name())
-    ]
+    objs = deque()
+    for rel in get_all_related_objects(object._meta):
+        if hasattr(object, rel.get_accessor_name()):
+            objs.append(getattr(object, rel.get_accessor_name()))
+
     while objs:
-        current = objs.pop(0)
+        current = objs.popleft()
         if current is None:
             continue
-        objects_used.append(current)
+        objects_used.add(current)
 
         for field in current._meta.fields:
             if (
@@ -123,17 +128,59 @@ def get_all_related_objects(_meta):
             if not field.auto_created:
                 if field.remote_field is None:
                     result += [(obj, field)]
+
     return result
 
 
-def serialize_participants_data(request, participants):
+def get_related_paths(model, prefix='', depth=5, visited=None):
+    if visited is None:
+        visited = set()
+    if model in visited or depth == 0:
+        return []
+
+    visited.add(model)
+    paths = []
+    try:
+        for field in model._meta.get_fields():
+            if isinstance(field, (ForeignKey, OneToOneField)) and not field.auto_created:
+                related_model = field.related_model
+                if related_model == Participant:
+                    continue  # skip backward pointer to Participant
+
+                full_path = f"{prefix}__{field.name}" if prefix else field.name
+                paths.append(full_path)
+
+                paths.extend(
+                    get_related_paths(related_model, prefix=full_path, depth=depth - 1, visited=visited)
+                )
+    finally:
+        visited.remove(model)
+
+    return paths
+
+
+def serialize_participants_data(request):
     """Serializes all personal data of participants to a table.
-    :param participants: A QuerySet from table participants.
     """
-
-    if not participants.exists():
+    participant = Participant.objects.filter(contest=request.contest).first()
+    if participant is None:
         return {'no_participants': True}
 
+    try: # Check if registration model exists
+        registration_model_instance = participant.registration_model
+        registration_model_class = registration_model_instance.__class__
+        registration_model_name = registration_model_instance._meta.get_field('participant').remote_field.related_name
+
+        related = get_related_paths(registration_model_class, prefix=registration_model_name, depth=10)
+        related.extend(['user', 'contest', registration_model_name])
+        participants = (
+            Participant.objects
+            .filter(contest=request.contest)
+            .select_related(*related)
+        )
+    except ObjectDoesNotExist: # It doesn't, so no need to select anything
+        participants = Participant.objects.filter(contest=request.contest)
+
     display_email = request.contest.controller.show_email_in_participants_data
 
     keys = ['username', 'user ID', 'first name', 'last name'] + (
@@ -144,9 +191,11 @@ def key_name(attr):
         (obj, field) = attr
         return str(obj.__class__.__name__) + ": " + field.verbose_name.title()
 
+    folded_participants = [(participant, _fold_registration_models_tree(participant)) for participant in participants]
+
     set_of_keys = set(keys)
-    for participant in participants:
-        for key in map(key_name, _fold_registration_models_tree(participant)):
+    for participant, folded in folded_participants:
+        for key in map(key_name, folded):
             if key not in set_of_keys:
                 set_of_keys.add(key)
                 keys.append(key)
@@ -156,8 +205,8 @@ def key_value(attr):
         return (key_name((obj, field)), field.value_to_string(obj))
 
     data = []
-    for participant in participants:
-        values = dict(list(map(key_value, _fold_registration_models_tree(participant))))
+    for participant, folded in folded_participants:
+        values = dict(list(map(key_value, folded)))
         values['username'] = participant.user.username
         values['user ID'] = participant.user.id
         values['first name'] = participant.user.first_name
@@ -169,8 +218,8 @@ def key_value(attr):
     return {'keys': keys, 'data': data}
 
 
-def render_participants_data_csv(request, participants, name):
-    data = serialize_participants_data(request, participants)
+def render_participants_data_csv(request, name):
+    data = serialize_participants_data(request)
     response = HttpResponse(content_type='text/csv')
     response['Content-Disposition'] = 'attachment; filename=%s-%s.csv' % (
         name,

oioioi/participants/views.py

@@ -84,9 +84,7 @@ def unregistration_view(request):
     not_anonymous & contest_exists & contest_has_participants & can_see_personal_data
 )
 def participants_data(request):
-    context = serialize_participants_data(
-        request, Participant.objects.filter(contest=request.contest)
-    )
+    context = serialize_participants_data(request)
     return TemplateResponse(request, 'participants/data.html', context)
 
 
@@ -95,5 +93,5 @@ def participants_data(request):
 )
 def participants_data_csv(request):
     return render_participants_data_csv(
-        request, Participant.objects.filter(contest=request.contest), request.contest.id
+        request, request.contest.id
     )