fix: build proceedings attendee list from MeetingRegistration table. … (#6567)

* fix: build proceedings attendee list from MeetingRegistration table. Fixes #6265

* fix: move participants_for_meeting to meeting.utils

* fix: move test_participants_for_meeting to meeting tests

ietf/meeting/tests_views.py

@@ -46,7 +46,7 @@ from ietf.meeting.helpers import send_interim_minutes_reminder, populate_importa
 from ietf.meeting.models import Session, TimeSlot, Meeting, SchedTimeSessAssignment, Schedule, SessionPresentation, SlideSubmission, SchedulingEvent, Room, Constraint, ConstraintName
 from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data
 from ietf.meeting.utils import finalize, condition_slide_order
-from ietf.meeting.utils import add_event_info_to_session_qs
+from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting
 from ietf.meeting.utils import create_recording, get_next_sequence
 from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule
 from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose
@@ -58,10 +58,12 @@ from ietf.utils.timezone import date_today, time_now
 
 from ietf.person.factories import PersonFactory
 from ietf.group.factories import GroupFactory, GroupEventFactory, RoleFactory
-from ietf.meeting.factories import ( SessionFactory, ScheduleFactory,
+from ietf.meeting.factories import (SessionFactory, ScheduleFactory,
     SessionPresentationFactory, MeetingFactory, FloorPlanFactory,
     TimeSlotFactory, SlideSubmissionFactory, RoomFactory,
-    ConstraintFactory, MeetingHostFactory, ProceedingsMaterialFactory )
+    ConstraintFactory, MeetingHostFactory, ProceedingsMaterialFactory,
+    AttendedFactory)
+from ietf.stats.factories import MeetingRegistrationFactory
 from ietf.doc.factories import DocumentFactory, WgDraftFactory
 from ietf.submit.tests import submission_file
 from ietf.utils.test_utils import assert_ical_response_is_valid
@@ -5967,16 +5969,10 @@ class IphoneAppJsonTests(TestCase):
             self.assertTrue(msessions.filter(group__acronym=s['group']['acronym']).exists())
 
 class FinalizeProceedingsTests(TestCase):
-    @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
+    def test_finalize_proceedings(self):
-    @requests_mock.Mocker()
-    def test_finalize_proceedings(self, mock):
         make_meeting_test_data()
         meeting = Meeting.objects.filter(type_id='ietf').order_by('id').last()
         meeting.session_set.filter(group__acronym='mars').first().sessionpresentation_set.create(document=Document.objects.filter(type='draft').first(),rev=None)
-        mock.get(
-            settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
-            text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
-        )
 
         url = urlreverse('ietf.meeting.views.finalize_proceedings',kwargs={'num':meeting.number})
         login_testing_unauthorized(self,"secretary",url)
@@ -7852,34 +7848,40 @@ class ProceedingsTests(BaseMeetingTestCase):
             0,
         )
 
-    @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
+    def test_proceedings_attendees(self):
-    @requests_mock.Mocker()
+        """Test proceedings attendee list. Check the following:
-    def test_proceedings_attendees(self, mock):
+           - assert onsite checkedin=True appears, not onsite checkedin=False
+           - assert remote attended appears, not remote not attended
+           - prefer onsite checkedin=True to remote attended when same person has both
+        """
+
         make_meeting_test_data()
-        meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="97")
+        meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016, 7, 14), number="97")
-        mock.get(
+        person_a = PersonFactory(name='Person A')
-            settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
+        person_b = PersonFactory(name='Person B')
-            text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
+        person_c = PersonFactory(name='Person C')
-        )
+        person_d = PersonFactory(name='Person D')
-        finalize(meeting)
+        MeetingRegistrationFactory(meeting=meeting, person=person_a, reg_type='onsite', checkedin=True)
-        url = urlreverse('ietf.meeting.views.proceedings_attendees',kwargs={'num':97})
+        MeetingRegistrationFactory(meeting=meeting, person=person_b, reg_type='onsite', checkedin=False)
+        MeetingRegistrationFactory(meeting=meeting, person=person_a, reg_type='remote')
+        AttendedFactory(session__meeting=meeting, session__type_id='plenary', person=person_a)
+        MeetingRegistrationFactory(meeting=meeting, person=person_c, reg_type='remote')
+        AttendedFactory(session__meeting=meeting, session__type_id='plenary', person=person_c)
+        MeetingRegistrationFactory(meeting=meeting, person=person_d, reg_type='remote')
+        url = urlreverse('ietf.meeting.views.proceedings_attendees',kwargs={'num': 97})
         response = self.client.get(url)
         self.assertContains(response, 'Attendee list')
         q = PyQuery(response.content)
-        self.assertEqual(1,len(q("#id_attendees tbody tr")))
+        self.assertEqual(2, len(q("#id_attendees tbody tr")))
+        text = q('#id_attendees tbody tr').text().replace('\n', ' ')
+        self.assertEqual(text, "A Person onsite C Person remote")
 
-    @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
+    def test_proceedings_overview(self):
-    @requests_mock.Mocker()
-    def test_proceedings_overview(self, mock):
         '''Test proceedings IETF Overview page.
         Note: old meetings aren't supported so need to add a new meeting then test.
         '''
         make_meeting_test_data()
         meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="97")
-        mock.get(
-            settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
-            text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
-        )
         finalize(meeting)
         url = urlreverse('ietf.meeting.views.proceedings_overview',kwargs={'num':97})
         response = self.client.get(url)
@@ -8266,3 +8268,20 @@ class ProceedingsTests(BaseMeetingTestCase):
         group = session.group
         sequence = get_next_sequence(group,meeting,'recording')
         self.assertEqual(sequence,1)
+
+    def test_participants_for_meeting(self):
+        person_a = PersonFactory()
+        person_b = PersonFactory()
+        person_c = PersonFactory()
+        person_d = PersonFactory()
+        m = MeetingFactory.create(type_id='ietf')
+        MeetingRegistrationFactory(meeting=m, person=person_a, reg_type='onsite', checkedin=True)
+        MeetingRegistrationFactory(meeting=m, person=person_b, reg_type='onsite', checkedin=False)
+        MeetingRegistrationFactory(meeting=m, person=person_c, reg_type='remote')
+        MeetingRegistrationFactory(meeting=m, person=person_d, reg_type='remote')
+        AttendedFactory(session__meeting=m, session__type_id='plenary', person=person_c)
+        checked_in, attended = participants_for_meeting(m)
+        self.assertTrue(person_a.pk in checked_in)
+        self.assertTrue(person_b.pk not in checked_in)
+        self.assertTrue(person_c.pk in attended)
+        self.assertTrue(person_d.pk not in attended)

ietf/meeting/utils.py

@@ -4,16 +4,14 @@ import datetime
 import itertools
 import os
 import pytz
-import requests
 import subprocess
 
 from collections import defaultdict
 from pathlib import Path
-from urllib.error import HTTPError
 
 from django.conf import settings
 from django.contrib import messages
-from django.template.loader import render_to_string
+from django.db.models import Q
 from django.utils import timezone
 from django.utils.encoding import smart_str
 
@@ -21,7 +19,7 @@ import debug                            # pyflakes:ignore
 
 from ietf.dbtemplate.models import DBTemplate
 from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot,
-    Constraint, SchedTimeSessAssignment, SessionPresentation)
+    Constraint, SchedTimeSessAssignment, SessionPresentation, Attended)
 from ietf.doc.models import Document, DocAlias, State, NewRevisionDocEvent
 from ietf.doc.models import DocEvent
 from ietf.group.models import Group
@@ -126,31 +124,7 @@ def sort_sessions(sessions):
     return sorted(sessions, key=lambda s: (s.meeting.number, s.group.acronym, session_time_for_sorting(s, use_meeting_date=False)))
 
 def create_proceedings_templates(meeting):
-    '''Create DBTemplates for meeting proceedings'''
+    '''Create DBTemplates for meeting proceedings'''  
-    # Get meeting attendees from registration system
-    url = settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number)
-    try:
-        attendees = requests.get(url, timeout=settings.DEFAULT_REQUESTS_TIMEOUT).json()
-    except (ValueError, HTTPError, requests.Timeout) as exc:
-        attendees = []
-        log(f'Failed to retrieve meeting attendees from [{url}]: {exc}')
-
-    if attendees:
-        attendees = sorted(attendees, key = lambda a: a['LastName'])
-        content = render_to_string('meeting/proceedings_attendees_table.html', {
-            'attendees':attendees})
-        try:
-            template = DBTemplate.objects.get(path='/meeting/proceedings/%s/attendees.html' % (meeting.number, ))
-            template.title='IETF %s Attendee List' % meeting.number
-            template.type_id='django'
-            template.content=content
-            template.save()
-        except DBTemplate.DoesNotExist:
-            DBTemplate.objects.create(
-                path='/meeting/proceedings/%s/attendees.html' % (meeting.number, ),
-                title='IETF %s Attendee List' % meeting.number,
-                type_id='django',
-                content=content)    
     # Make copy of default IETF Overview template
     if not meeting.overview:
         path = '/meeting/proceedings/%s/overview.rst' % (meeting.number, )
@@ -910,3 +884,14 @@ def post_process(doc):
             desc='Converted document to PDF',
         )
         doc.save_with_history([e])
+
+
+def participants_for_meeting(meeting):
+    """ Return a tuple (checked_in, attended)
+        checked_in = queryset of onsite, checkedin participants values_list('person')
+        attended = queryset of remote participants who attended a session values_list('person')
+    """
+    checked_in = meeting.meetingregistration_set.filter(reg_type='onsite', checkedin=True).values_list('person', flat=True).distinct()
+    sessions = meeting.session_set.filter(Q(type='plenary') | Q(group__type__in=['wg', 'rg']))
+    attended = Attended.objects.filter(session__in=sessions).values_list('person', flat=True).distinct()
+    return (checked_in, attended)

ietf/meeting/views.py

@@ -84,8 +84,10 @@ from ietf.meeting.utils import swap_meeting_schedule_timeslot_assignments, bulk_
 from ietf.meeting.utils import preprocess_meeting_important_dates
 from ietf.meeting.utils import new_doc_for_session, write_doc_for_session
 from ietf.meeting.utils import get_activity_stats, post_process, create_recording
+from ietf.meeting.utils import participants_for_meeting
 from ietf.message.utils import infer_message
 from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName
+from ietf.stats.models import MeetingRegistration
 from ietf.utils import markdown
 from ietf.utils.decorators import require_api_key
 from ietf.utils.hedgedoc import Note, NoteError
@@ -3858,14 +3860,19 @@ def proceedings_attendees(request, num=None):
     meeting = get_meeting(num)
     if meeting.proceedings_format_version == 1:
         return HttpResponseRedirect(f'{settings.PROCEEDINGS_V1_BASE_URL.format(meeting=meeting)}/attendee.html')
-    overview_template = '/meeting/proceedings/%s/attendees.html' % meeting.number
+
-    try:
+    checked_in, attended = participants_for_meeting(meeting)
-        template = render_to_string(overview_template, {})
+    regs = list(MeetingRegistration.objects.filter(meeting__number=num, reg_type='onsite', checkedin=True))
-    except TemplateDoesNotExist:
+
-        raise Http404
+    for mr in MeetingRegistration.objects.filter(meeting__number=num, reg_type='remote').select_related('person'):
+        if mr.person.pk in attended and mr.person.pk not in checked_in:
+            regs.append(mr)
+
+    meeting_registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name))
+
     return render(request, "meeting/proceedings_attendees.html", {
         'meeting': meeting,
-        'template': template,
+        'meeting_registrations': meeting_registrations,
     })
 
 def proceedings_overview(request, num=None):

ietf/nomcom/tests.py

@@ -2775,6 +2775,7 @@ class rfc9389EligibilityTests(TestCase):
         for person in ineligible_people:
             self.assertFalse(is_eligible(person,self.nomcom))
 
+
 class VolunteerTests(TestCase):
 
     def test_volunteer(self):

ietf/nomcom/utils.py

@@ -30,7 +30,8 @@ from ietf.doc.models import DocEvent, NewRevisionDocEvent
 from ietf.group.models import Group, Role
 from ietf.person.models import Email, Person
 from ietf.mailtrigger.utils import gather_address_lists
-from ietf.meeting.models import Meeting, Attended
+from ietf.meeting.models import Meeting
+from ietf.meeting.utils import participants_for_meeting
 from ietf.utils.pipe import pipe
 from ietf.utils.mail import send_mail_text, send_mail, get_payload_text
 from ietf.utils.log import log
@@ -689,9 +690,7 @@ def three_of_five_eligible_9389(previous_five, queryset=None):
 
     counts = defaultdict(lambda: 0)
     for meeting in previous_five:
-        checked_in = meeting.meetingregistration_set.filter(reg_type='onsite', checkedin=True).values_list('person', flat=True)
+        checked_in, attended = participants_for_meeting(meeting)
-        sessions = meeting.session_set.filter(Q(type='plenary') | Q(group__type__in=['wg', 'rg']))
-        attended = Attended.objects.filter(session__in=sessions).values_list('person', flat=True)
         for id in set(checked_in) | set(attended):
             counts[id] += 1
     return queryset.filter(pk__in=[id for id, count in counts.items() if count >= 3])

ietf/stats/migrations/0002_fix_meeting_registration_reg_type.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.6 on 2023-10-27 17:57
+
+'''
+MeetingRegistration reg_type and checkedin fields are not populated for meetings 
+prior to 108. For meetings 72-106, all records are for onsite checkedin participants.
+For meeting 107 all records are for remote paticipants. Set accordingly.
+'''
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    MeetingRegistration = apps.get_model("stats", "MeetingRegistration")
+    MeetingRegistration.objects.filter(meeting__number=107).update(reg_type='remote')
+    MeetingRegistration.objects.filter(meeting__number__lte=106, reg_type='').update(reg_type='onsite', checkedin=True)
+
+
+def reverse(apps, schema_editor):
+    MeetingRegistration = apps.get_model("stats", "MeetingRegistration")
+    MeetingRegistration.objects.filter(meeting__number=107).update(reg_type='')
+    MeetingRegistration.objects.filter(meeting__number__lte=106, reg_type='onsite').update(reg_type='', checkedin=False)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("stats", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]

ietf/templates/meeting/proceedings_attendees.html

@@ -14,7 +14,30 @@
         </a>
     </h1>
     <h2>Attendee list of IETF {{ meeting.number }} meeting</h2>
-    {{ template|safe }}
+    
+    <table id="id_attendees" class="table table-sm table-striped tablesorter">
+    <thead>
+        <tr>
+            <th scope="col" data-sort="last">Last Name</th>
+            <th scope="col" data-sort="first">First Name</th>
+            <th scope="col" data-sort="organization">Organization</th>
+            <th scope="col" data-sort="country">Country</th>
+            <th scope="col" data-sort="type">Registration Type</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for reg in meeting_registrations %}
+          <tr>
+            <td>{{ reg.last_name }}</td>
+            <td>{{ reg.first_name }}</td>
+            <td>{{ reg.affiliation }}</td>
+            <td>{{ reg.country_code }}</td>
+            <td>{{ reg.reg_type }}</td>
+          </tr>
+        {% endfor %}
+    </tbody>
+    </table>
+
 {% endblock %}
 {% block js %}
     <script src="{% static "ietf/js/list.js" %}"></script>

ietf/templates/meeting/proceedings_attendees_table.html

@@ -1,20 +0,0 @@
-<table id="id_attendees" class="table table-sm table-striped tablesorter">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="last">Last name</th>
-            <th scope="col" data-sort="first">First name</th>
-            <th scope="col" data-sort="organization">Organization</th>
-            <th scope="col" data-sort="country">Country</th>
-        </tr>
-    </thead>
-    <tbody>
-        {% for attendee in attendees %}
-            <tr>
-                <td>{{ attendee.LastName }}</td>
-                <td>{{ attendee.FirstName }}</td>
-                <td>{{ attendee.Company }}</td>
-                <td>{{ attendee.Country }}</td>
-            </tr>
-        {% endfor %}
-    </tbody>
-</table>