From 060320d766803e479d860832a2ae9ebeb5a9d4d3 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 5 Feb 2025 18:29:32 -0400 Subject: [PATCH] feat: keep proceedings cache up to date via celery (#8449) * refactor: better control proceedings caching * refactor: move methods from views to utils * chore: revert accidental settings change * fix: eliminate circular import get_schedule() with name=None should perhaps be an anti-pattern * feat: task to recompute proceedings daily * chore: proceedings cache lifetime = 1 day * fix: ensure finalization is immediately reflected * chore: update beat comments in docker-compose * style: undo a couple whitespace changes * test: update / refactor tests * test: test task * refactor: disallow positional arg to task * refactor: add trivial test of old task --- docker-compose.yml | 1 + ietf/meeting/tasks.py | 32 ++ ietf/meeting/tests_tasks.py | 51 +++ ietf/meeting/tests_views.py | 91 ++++- ietf/meeting/utils.py | 171 +++++++- ietf/meeting/views.py | 157 +------- ietf/templates/meeting/proceedings.html | 380 ++++++++---------- .../meeting/proceedings_wrapper.html | 27 ++ 8 files changed, 542 insertions(+), 368 deletions(-) create mode 100644 ietf/meeting/tests_tasks.py create mode 100644 ietf/templates/meeting/proceedings_wrapper.html diff --git a/docker-compose.yml b/docker-compose.yml index 65b28f54f..9988b10c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,6 +101,7 @@ services: # stop_grace_period: 1m # volumes: # - .:/workspace +# - app-assets:/assets volumes: postgresdb-data: diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py index 43cbb0a75..2b7c2fca9 100644 --- a/ietf/meeting/tasks.py +++ b/ietf/meeting/tasks.py @@ -3,10 +3,42 @@ # Celery task definitions # from celery import shared_task +from django.utils import timezone +from ietf.utils import log +from .models import Meeting +from .utils import generate_proceedings_content from .views import generate_agenda_data @shared_task def agenda_data_refresh(): generate_agenda_data(force_refresh=True) + + +@shared_task +def proceedings_content_refresh_task(*, all=False): + """Refresh meeting proceedings cache + + If `all` is `False`, then refreshes the cache for meetings whose numbers modulo + 24 equal the current hour number (0-23). Scheduling the task once per hour will + then result in all proceedings being recomputed daily, with no more than two per + hour (now) or a few per hour in the next decade. That keeps the computation time + to under a couple minutes on our current production system. + + If `all` is True, refreshes all meetings + """ + now = timezone.now() + + for meeting in Meeting.objects.filter(type_id="ietf").order_by("number"): + if meeting.proceedings_format_version == 1: + continue # skip v1 proceedings, they're stored externally + num = meeting.get_number() # convert str -> int + if num is None: + log.log( + f"Not refreshing proceedings for meeting {meeting.number}: " + f"type is 'ietf' but get_number() returned None" + ) + elif all or (num % 24 == now.hour): + log.log(f"Refreshing proceedings for meeting {meeting.number}...") + generate_proceedings_content(meeting, force_refresh=True) diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py new file mode 100644 index 000000000..c026a9983 --- /dev/null +++ b/ietf/meeting/tests_tasks.py @@ -0,0 +1,51 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +import datetime +from mock import patch, call +from ietf.utils.test_utils import TestCase +from .factories import MeetingFactory +from .tasks import proceedings_content_refresh_task, agenda_data_refresh + + +class TaskTests(TestCase): + @patch("ietf.meeting.tasks.generate_agenda_data") + def test_agenda_data_refresh(self, mock_generate): + agenda_data_refresh() + self.assertTrue(mock_generate.called) + self.assertEqual(mock_generate.call_args, call(force_refresh=True)) + + @patch("ietf.meeting.tasks.generate_proceedings_content") + def test_proceedings_content_refresh_task(self, mock_generate): + # Generate a couple of meetings + meeting120 = MeetingFactory(type_id="ietf", number="120") # 24 * 5 + meeting127 = MeetingFactory(type_id="ietf", number="127") # 24 * 5 + 7 + + # Times to be returned + now_utc = datetime.datetime.now(tz=datetime.timezone.utc) + hour_00_utc = now_utc.replace(hour=0) + hour_01_utc = now_utc.replace(hour=1) + hour_07_utc = now_utc.replace(hour=7) + + # hour 00 - should call meeting with number % 24 == 0 + with patch("ietf.meeting.tasks.timezone.now", return_value=hour_00_utc): + proceedings_content_refresh_task() + self.assertEqual(mock_generate.call_count, 1) + self.assertEqual(mock_generate.call_args, call(meeting120, force_refresh=True)) + mock_generate.reset_mock() + + # hour 01 - should call no meetings + with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): + proceedings_content_refresh_task() + self.assertEqual(mock_generate.call_count, 0) + + # hour 07 - should call meeting with number % 24 == 0 + with patch("ietf.meeting.tasks.timezone.now", return_value=hour_07_utc): + proceedings_content_refresh_task() + self.assertEqual(mock_generate.call_count, 1) + self.assertEqual(mock_generate.call_args, call(meeting127, force_refresh=True)) + mock_generate.reset_mock() + + # With all=True, all should be called regardless of time. Reuse hour_01_utc which called none before + with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): + proceedings_content_refresh_task(all=True) + self.assertEqual(mock_generate.call_count, 2) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 94f06dc89..581725dbc 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -32,6 +32,7 @@ from django.db.models import F, Max from django.http import QueryDict, FileResponse from django.template import Context, Template from django.utils import timezone +from django.utils.safestring import mark_safe from django.utils.text import slugify import debug # pyflakes:ignore @@ -46,7 +47,7 @@ from ietf.meeting.helpers import send_interim_meeting_cancellation_notice, send_ from ietf.meeting.helpers import send_interim_minutes_reminder, populate_important_dates, update_important_dates from ietf.meeting.models import Session, TimeSlot, Meeting, SchedTimeSessAssignment, Schedule, SessionPresentation, SlideSubmission, SchedulingEvent, Room, Constraint, ConstraintName from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data -from ietf.meeting.utils import condition_slide_order +from ietf.meeting.utils import condition_slide_order, generate_proceedings_content from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule @@ -8296,8 +8297,7 @@ class ProceedingsTests(BaseMeetingTestCase): path = Path(settings.BASE_DIR) / 'meeting/test_procmat.pdf' return path.open('rb') - def _assertMeetingHostsDisplayed(self, response, meeting): - pq = PyQuery(response.content) + def _assertMeetingHostsDisplayed(self, pq: PyQuery, meeting): host_divs = pq('div.host-logo') self.assertEqual(len(host_divs), meeting.meetinghosts.count(), 'Should have a logo for every meeting host') self.assertEqual( @@ -8313,12 +8313,11 @@ class ProceedingsTests(BaseMeetingTestCase): 'Correct image and name for each host should appear in the correct order' ) - def _assertProceedingsMaterialsDisplayed(self, response, meeting): + def _assertProceedingsMaterialsDisplayed(self, pq: PyQuery, meeting): """Checks that all (and only) active materials are linked with correct href and title""" expected_materials = [ m for m in meeting.proceedings_materials.order_by('type__order') if m.active() ] - pq = PyQuery(response.content) links = pq('div.proceedings-material a') self.assertEqual(len(links), len(expected_materials), 'Should have an entry for each active ProceedingsMaterial') self.assertEqual( @@ -8327,9 +8326,8 @@ class ProceedingsTests(BaseMeetingTestCase): 'Correct title and link for each ProceedingsMaterial should appear in the correct order' ) - def _assertGroupSessions(self, response, meeting): + def _assertGroupSessions(self, pq: PyQuery): """Checks that group/sessions are present""" - pq = PyQuery(response.content) sections = ["plenaries", "gen", "iab", "editorial", "irtf", "training"] for section in sections: self.assertEqual(len(pq(f"#{section}")), 1, f"{section} section should exists in proceedings") @@ -8337,10 +8335,9 @@ class ProceedingsTests(BaseMeetingTestCase): def test_proceedings(self): """Proceedings should be displayed correctly - Currently only tests that the view responds with a 200 response code and checks the ProceedingsMaterials - at the top of the proceedings. Ought to actually test the display of the individual group/session - materials as well. + Proceedings contents are tested in detail when testing generate_proceedings_content. """ + # number must be >97 (settings.PROCEEDINGS_VERSION_CHANGES) meeting = make_meeting_test_data(meeting=MeetingFactory(type_id='ietf', number='100')) session = Session.objects.filter(meeting=meeting, group__acronym="mars").first() GroupEventFactory(group=session.group,type='status_update') @@ -8365,16 +8362,72 @@ class ProceedingsTests(BaseMeetingTestCase): self._create_proceedings_materials(meeting) url = urlreverse("ietf.meeting.views.proceedings", kwargs=dict(num=meeting.number)) - r = self.client.get(url) + cached_content = mark_safe("

Fake proceedings content

") + with patch("ietf.meeting.views.generate_proceedings_content") as mock_gpc: + mock_gpc.return_value = cached_content + r = self.client.get(url) self.assertEqual(r.status_code, 200) + self.assertIn(cached_content, r.content.decode()) + self.assertTemplateUsed(r, "meeting/proceedings_wrapper.html") + self.assertTemplateNotUsed(r, "meeting/proceedings.html") + # These are rendered in proceedings_wrapper.html, so test them here if len(meeting.city) > 0: self.assertContains(r, meeting.city) if len(meeting.venue_name) > 0: self.assertContains(r, meeting.venue_name) + self._assertMeetingHostsDisplayed(PyQuery(r.content), meeting) + + @patch("ietf.meeting.utils.caches") + def test_generate_proceedings_content(self, mock_caches): + # number must be >97 (settings.PROCEEDINGS_VERSION_CHANGES) + meeting = make_meeting_test_data(meeting=MeetingFactory(type_id='ietf', number='100')) + + # First, check that by default a value in the cache is used without doing any other computation + mock_default_cache = mock_caches["default"] + mock_default_cache.get.return_value = "a cached value" + result = generate_proceedings_content(meeting) + self.assertEqual(result, "a cached value") + self.assertFalse(mock_default_cache.set.called) + self.assertTrue(mock_default_cache.get.called) + cache_key = mock_default_cache.get.call_args.args[0] + mock_default_cache.get.reset_mock() + + # Now set up for actual computation of the proceedings content. + session = Session.objects.filter(meeting=meeting, group__acronym="mars").first() + GroupEventFactory(group=session.group,type='status_update') + SessionPresentationFactory(document__type_id='recording',session=session) + SessionPresentationFactory(document__type_id='recording',session=session,document__title="Audio recording for tests") + + # Add various group sessions + groups = [] + parent_groups = [ + GroupFactory.create(type_id="area", acronym="gen"), + GroupFactory.create(acronym="iab"), + GroupFactory.create(acronym="irtf"), + ] + for parent in parent_groups: + groups.append(GroupFactory.create(parent=parent)) + for acronym in ["rsab", "edu"]: + groups.append(GroupFactory.create(acronym=acronym)) + for group in groups: + SessionFactory(meeting=meeting, group=group) + + self.write_materials_files(meeting, session) + self._create_proceedings_materials(meeting) + + # Now "empty" the mock cache and see that we compute the expected proceedings content. + mock_default_cache.get.return_value = None + proceedings_content = generate_proceedings_content(meeting) + self.assertTrue(mock_default_cache.get.called) + self.assertEqual(mock_default_cache.get.call_args.args[0], cache_key, "same cache key each time") + self.assertTrue(mock_default_cache.set.called) + self.assertEqual(mock_default_cache.set.call_args, call(cache_key, proceedings_content, timeout=86400)) + mock_default_cache.get.reset_mock() + mock_default_cache.set.reset_mock() # standard items on every proceedings - pq = PyQuery(r.content) + pq = PyQuery(proceedings_content) self.assertNotEqual( pq('a[href="{}"]'.format( urlreverse('ietf.meeting.views.proceedings_overview', kwargs=dict(num=meeting.number))) @@ -8405,9 +8458,17 @@ class ProceedingsTests(BaseMeetingTestCase): ) # configurable contents - self._assertMeetingHostsDisplayed(r, meeting) - self._assertProceedingsMaterialsDisplayed(r, meeting) - self._assertGroupSessions(r, meeting) + self._assertProceedingsMaterialsDisplayed(pq, meeting) + self._assertGroupSessions(pq) + + # Finally, repeat the first cache test, but now with force_refresh=True. The cached value + # should be ignored and we should recompute the proceedings as before. + mock_default_cache.get.return_value = "a cached value" + result = generate_proceedings_content(meeting, force_refresh=True) + self.assertEqual(result, proceedings_content) # should have recomputed the same thing + self.assertFalse(mock_default_cache.get.called, "don't bother reading cache when force_refresh is True") + self.assertTrue(mock_default_cache.set.called) + self.assertEqual(mock_default_cache.set.call_args, call(cache_key, proceedings_content, timeout=86400)) def test_named_session(self): """Session with a name should appear separately in the proceedings""" diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 6e681fdc3..92bae5ac2 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -3,6 +3,8 @@ import datetime import itertools import os +from hashlib import sha384 + import pytz import subprocess @@ -11,8 +13,9 @@ from pathlib import Path from django.conf import settings from django.contrib import messages +from django.core.cache import caches from django.core.files.base import ContentFile -from django.db.models import OuterRef, Subquery, TextField, Q, Value +from django.db.models import OuterRef, Subquery, TextField, Q, Value, Max from django.db.models.functions import Coalesce from django.template.loader import render_to_string from django.utils import timezone @@ -995,3 +998,169 @@ def participants_for_meeting(meeting): sessions = meeting.session_set.filter(Q(type='plenary') | Q(group__type__in=['wg', 'rg'])) attended = Attended.objects.filter(session__in=sessions).values_list('person', flat=True).distinct() return (checked_in, attended) + + +def generate_proceedings_content(meeting, force_refresh=False): + """Render proceedings content for a meeting and update cache + + :meeting: meeting whose proceedings should be rendered + :force_refresh: true to force regeneration and cache refresh + """ + cache = caches["default"] + cache_version = Document.objects.filter(session__meeting__number=meeting.number).aggregate(Max('time'))["time__max"] + # Include proceedings_final in the bare_key so we'll always reflect that accurately, even at the cost of + # a recomputation in the view + bare_key = f"proceedings.{meeting.number}.{cache_version}.final={meeting.proceedings_final}" + cache_key = sha384(bare_key.encode("utf8")).hexdigest() + if not force_refresh: + cached_content = cache.get(cache_key, None) + if cached_content is not None: + return cached_content + + def area_and_group_acronyms_from_session(s): + area = s.group_parent_at_the_time() + if area == None: + area = s.group.parent + group = s.group_at_the_time() + return (area.acronym, group.acronym) + + schedule = meeting.schedule + sessions = ( + meeting.session_set.with_current_status() + .filter(Q(timeslotassignments__schedule__in=[schedule, schedule.base if schedule else None]) + | Q(current_status='notmeet')) + .select_related() + .order_by('-current_status') + ) + + plenaries, _ = organize_proceedings_sessions( + sessions.filter(name__icontains='plenary') + .exclude(current_status='notmeet') + ) + irtf_meeting, irtf_not_meeting = organize_proceedings_sessions( + sessions.filter(group__parent__acronym = 'irtf').order_by('group__acronym') + ) + # per Colin (datatracker #5010) - don't report not meeting rags + irtf_not_meeting = [item for item in irtf_not_meeting if item["group"].type_id != "rag"] + irtf = {"meeting_groups":irtf_meeting, "not_meeting_groups":irtf_not_meeting} + + training, _ = organize_proceedings_sessions( + sessions.filter(group__acronym__in=['edu','iaoc'], type_id__in=['regular', 'other',]) + .exclude(current_status='notmeet') + ) + iab, _ = organize_proceedings_sessions( + sessions.filter(group__parent__acronym = 'iab') + .exclude(current_status='notmeet') + ) + editorial, _ = organize_proceedings_sessions( + sessions.filter(group__acronym__in=['rsab','rswg']) + .exclude(current_status='notmeet') + ) + + ietf = sessions.filter(group__parent__type__slug = 'area').exclude(group__acronym__in=['edu','iepg','tools']) + ietf = list(ietf) + ietf.sort(key=lambda s: area_and_group_acronyms_from_session(s)) + ietf_areas = [] + for area, area_sessions in itertools.groupby(ietf, key=lambda s: s.group_parent_at_the_time()): + meeting_groups, not_meeting_groups = organize_proceedings_sessions(area_sessions) + ietf_areas.append((area, meeting_groups, not_meeting_groups)) + + with timezone.override(meeting.tz()): + rendered_content = render_to_string( + "meeting/proceedings.html", + { + 'meeting': meeting, + 'plenaries': plenaries, + 'training': training, + 'irtf': irtf, + 'iab': iab, + 'editorial': editorial, + 'ietf_areas': ietf_areas, + 'meetinghost_logo': { + 'max_height': settings.MEETINGHOST_LOGO_MAX_DISPLAY_HEIGHT, + 'max_width': settings.MEETINGHOST_LOGO_MAX_DISPLAY_WIDTH, + } + }, + ) + cache.set( + cache_key, + rendered_content, + timeout=86400, # one day, in seconds + ) + return rendered_content + + +def organize_proceedings_sessions(sessions): + # Collect sessions by Group, then bin by session name (including sessions with blank names). + # If all of a group's sessions are 'notmeet', the processed data goes in not_meeting_sessions. + # Otherwise, the data goes in meeting_sessions. + meeting_groups = [] + not_meeting_groups = [] + for group_acronym, group_sessions in itertools.groupby(sessions, key=lambda s: s.group.acronym): + by_name = {} + is_meeting = False + all_canceled = True + group = None + for s in sorted( + group_sessions, + key=lambda gs: ( + gs.official_timeslotassignment().timeslot.time + if gs.official_timeslotassignment() else datetime.datetime(datetime.MAXYEAR, 1, 1) + ), + ): + group = s.group + if s.current_status != 'notmeet': + is_meeting = True + if s.current_status != 'canceled': + all_canceled = False + by_name.setdefault(s.name, []) + if s.current_status != 'notmeet' or s.presentations.exists(): + by_name[s.name].append(s) # for notmeet, only include sessions with materials + for sess_name, ss in by_name.items(): + session = ss[0] if ss else None + def _format_materials(items): + """Format session/material for template + + Input is a list of (session, materials) pairs. The materials value can be a single value or a list. + """ + material_times = {} # key is material, value is first timestamp it appeared + for s, mats in items: + tsa = s.official_timeslotassignment() + timestamp = tsa.timeslot.time if tsa else None + if not isinstance(mats, list): + mats = [mats] + for mat in mats: + if mat and mat not in material_times: + material_times[mat] = timestamp + n_mats = len(material_times) + result = [] + if n_mats == 1: + result.append({'material': list(material_times)[0]}) # no 'time' when only a single material + elif n_mats > 1: + for mat, timestamp in material_times.items(): + result.append({'material': mat, 'time': timestamp}) + return result + + entry = { + 'group': group, + 'name': sess_name, + 'session': session, + 'canceled': all_canceled, + 'has_materials': s.presentations.exists(), + 'agendas': _format_materials((s, s.agenda()) for s in ss), + 'minutes': _format_materials((s, s.minutes()) for s in ss), + 'bluesheets': _format_materials((s, s.bluesheets()) for s in ss), + 'recordings': _format_materials((s, s.recordings()) for s in ss), + 'meetecho_recordings': _format_materials((s, [s.session_recording_url()]) for s in ss), + 'chatlogs': _format_materials((s, s.chatlogs()) for s in ss), + 'slides': _format_materials((s, s.slides()) for s in ss), + 'drafts': _format_materials((s, s.drafts()) for s in ss), + 'last_update': session.last_update if hasattr(session, 'last_update') else None + } + if session and session.meeting.type_id == 'ietf' and not session.meeting.proceedings_final: + entry['attendances'] = _format_materials((s, s) for s in ss if Attended.objects.filter(session=s).exists()) + if is_meeting: + meeting_groups.append(entry) + else: + not_meeting_groups.append(entry) + return meeting_groups, not_meeting_groups diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 2f2464028..1226e30d6 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -75,7 +75,13 @@ from ietf.meeting.helpers import send_interim_meeting_cancellation_notice, send_ from ietf.meeting.helpers import send_interim_approval from ietf.meeting.helpers import send_interim_approval_request from ietf.meeting.helpers import send_interim_announcement_request, sessions_post_cancel -from ietf.meeting.utils import finalize, sort_accept_tuple, condition_slide_order +from ietf.meeting.utils import ( + condition_slide_order, + finalize, + generate_proceedings_content, + organize_proceedings_sessions, + sort_accept_tuple, +) from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.utils import session_time_for_sorting from ietf.meeting.utils import session_requested_by, SaveMaterialsError @@ -4128,93 +4134,10 @@ def upcoming_json(request): response = HttpResponse(json.dumps(data, indent=2, sort_keys=False), content_type='application/json;charset=%s'%settings.DEFAULT_CHARSET) return response -def organize_proceedings_sessions(sessions): - # Collect sessions by Group, then bin by session name (including sessions with blank names). - # If all of a group's sessions are 'notmeet', the processed data goes in not_meeting_sessions. - # Otherwise, the data goes in meeting_sessions. - meeting_groups = [] - not_meeting_groups = [] - for group_acronym, group_sessions in itertools.groupby(sessions, key=lambda s: s.group.acronym): - by_name = {} - is_meeting = False - all_canceled = True - group = None - for s in sorted( - group_sessions, - key=lambda gs: ( - gs.official_timeslotassignment().timeslot.time - if gs.official_timeslotassignment() else datetime.datetime(datetime.MAXYEAR, 1, 1) - ), - ): - group = s.group - if s.current_status != 'notmeet': - is_meeting = True - if s.current_status != 'canceled': - all_canceled = False - by_name.setdefault(s.name, []) - if s.current_status != 'notmeet' or s.presentations.exists(): - by_name[s.name].append(s) # for notmeet, only include sessions with materials - for sess_name, ss in by_name.items(): - session = ss[0] if ss else None - def _format_materials(items): - """Format session/material for template - - Input is a list of (session, materials) pairs. The materials value can be a single value or a list. - """ - material_times = {} # key is material, value is first timestamp it appeared - for s, mats in items: - tsa = s.official_timeslotassignment() - timestamp = tsa.timeslot.time if tsa else None - if not isinstance(mats, list): - mats = [mats] - for mat in mats: - if mat and mat not in material_times: - material_times[mat] = timestamp - n_mats = len(material_times) - result = [] - if n_mats == 1: - result.append({'material': list(material_times)[0]}) # no 'time' when only a single material - elif n_mats > 1: - for mat, timestamp in material_times.items(): - result.append({'material': mat, 'time': timestamp}) - return result - - entry = { - 'group': group, - 'name': sess_name, - 'session': session, - 'canceled': all_canceled, - 'has_materials': s.presentations.exists(), - 'agendas': _format_materials((s, s.agenda()) for s in ss), - 'minutes': _format_materials((s, s.minutes()) for s in ss), - 'bluesheets': _format_materials((s, s.bluesheets()) for s in ss), - 'recordings': _format_materials((s, s.recordings()) for s in ss), - 'meetecho_recordings': _format_materials((s, [s.session_recording_url()]) for s in ss), - 'chatlogs': _format_materials((s, s.chatlogs()) for s in ss), - 'slides': _format_materials((s, s.slides()) for s in ss), - 'drafts': _format_materials((s, s.drafts()) for s in ss), - 'last_update': session.last_update if hasattr(session, 'last_update') else None - } - if session and session.meeting.type_id == 'ietf' and not session.meeting.proceedings_final: - entry['attendances'] = _format_materials((s, s) for s in ss if Attended.objects.filter(session=s).exists()) - if is_meeting: - meeting_groups.append(entry) - else: - not_meeting_groups.append(entry) - return meeting_groups, not_meeting_groups - def proceedings(request, num=None): - - def area_and_group_acronyms_from_session(s): - area = s.group_parent_at_the_time() - if area == None: - area = s.group.parent - group = s.group_at_the_time() - return (area.acronym, group.acronym) - meeting = get_meeting(num) - + # Early proceedings were hosted on www.ietf.org rather than the datatracker if meeting.proceedings_format_version == 1: return HttpResponseRedirect(settings.PROCEEDINGS_V1_BASE_URL.format(meeting=meeting)) @@ -4225,72 +4148,12 @@ def proceedings(request, num=None): kwargs['num'] = num return redirect('ietf.meeting.views.materials', **kwargs) - begin_date = meeting.get_submission_start_date() - cut_off_date = meeting.get_submission_cut_off_date() - cor_cut_off_date = meeting.get_submission_correction_date() - today_utc = date_today(datetime.timezone.utc) - - schedule = get_schedule(meeting, None) - sessions = ( - meeting.session_set.with_current_status() - .filter(Q(timeslotassignments__schedule__in=[schedule, schedule.base if schedule else None]) - | Q(current_status='notmeet')) - .select_related() - .order_by('-current_status') - ) - - plenaries, _ = organize_proceedings_sessions( - sessions.filter(name__icontains='plenary') - .exclude(current_status='notmeet') - ) - irtf_meeting, irtf_not_meeting = organize_proceedings_sessions( - sessions.filter(group__parent__acronym = 'irtf').order_by('group__acronym') - ) - # per Colin (datatracker #5010) - don't report not meeting rags - irtf_not_meeting = [item for item in irtf_not_meeting if item["group"].type_id != "rag"] - irtf = {"meeting_groups":irtf_meeting, "not_meeting_groups":irtf_not_meeting} - - training, _ = organize_proceedings_sessions( - sessions.filter(group__acronym__in=['edu','iaoc'], type_id__in=['regular', 'other',]) - .exclude(current_status='notmeet') - ) - iab, _ = organize_proceedings_sessions( - sessions.filter(group__parent__acronym = 'iab') - .exclude(current_status='notmeet') - ) - editorial, _ = organize_proceedings_sessions( - sessions.filter(group__acronym__in=['rsab','rswg']) - .exclude(current_status='notmeet') - ) - - ietf = sessions.filter(group__parent__type__slug = 'area').exclude(group__acronym__in=['edu','iepg','tools']) - ietf = list(ietf) - ietf.sort(key=lambda s: area_and_group_acronyms_from_session(s)) - ietf_areas = [] - for area, area_sessions in itertools.groupby(ietf, key=lambda s: s.group_parent_at_the_time()): - meeting_groups, not_meeting_groups = organize_proceedings_sessions(area_sessions) - ietf_areas.append((area, meeting_groups, not_meeting_groups)) - - cache_version = Document.objects.filter(session__meeting__number=meeting.number).aggregate(Max('time'))["time__max"] with timezone.override(meeting.tz()): - return render(request, "meeting/proceedings.html", { + return render(request, "meeting/proceedings_wrapper.html", { 'meeting': meeting, - 'plenaries': plenaries, - 'training': training, - 'irtf': irtf, - 'iab': iab, - 'editorial': editorial, - 'ietf_areas': ietf_areas, - 'cut_off_date': cut_off_date, - 'cor_cut_off_date': cor_cut_off_date, - 'submission_started': today_utc > begin_date, - 'cache_version': cache_version, 'attendance': meeting.get_attendance(), - 'meetinghost_logo': { - 'max_height': settings.MEETINGHOST_LOGO_MAX_DISPLAY_HEIGHT, - 'max_width': settings.MEETINGHOST_LOGO_MAX_DISPLAY_WIDTH, - } + 'proceedings_content': generate_proceedings_content(meeting), }) @role_required('Secretariat') diff --git a/ietf/templates/meeting/proceedings.html b/ietf/templates/meeting/proceedings.html index b5d4a6198..0aa8197fe 100644 --- a/ietf/templates/meeting/proceedings.html +++ b/ietf/templates/meeting/proceedings.html @@ -1,184 +1,160 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin %} -{% load ietf_filters static %} -{% block pagehead %} - -{% endblock %} -{% block title %} - IETF {{ meeting.number }} - {% if not meeting.proceedings_final %}Draft{% endif %} - Proceedings -{% endblock %} -{% block content %} - {% origin %} - {% include 'meeting/proceedings/title.html' with meeting=meeting attendance=attendance only %} - {% if user|has_role:"Secretariat" and not meeting.proceedings_final %} - - Finalize proceedings - - {% endif %} - {# cache for 15 minutes, as long as there's no proceedings activity. takes 4-8 seconds to generate. #} - {% load cache %} - {% cache 900 ietf_meeting_proceedings meeting.number cache_version %} - {% include 'meeting/proceedings/introduction.html' with meeting=meeting only %} - - {% if plenaries %} -

Plenaries

+{% include 'meeting/proceedings/introduction.html' with meeting=meeting only %} + +{% if plenaries %} +

Plenaries

+ + + + + + + + + + + + {% for entry in plenaries %} + {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} + {% endfor %} + +
GroupArtifactsRecordingsSlidesInternet-Drafts
+{% endif %} + +{% for area, meeting_groups, not_meeting_groups in ietf_areas %} +

+ {{ area.acronym|upper }} {{ area.name }} +

+ {% if meeting_groups %} - - - - - - - + + + + + + + - {% for entry in plenaries %} - {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} - {% endfor %} + {% for entry in meeting_groups %} + {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} + {% endfor %}
GroupArtifactsRecordingsSlidesInternet-Drafts
GroupArtifactsRecordingsSlidesInternet-Drafts
{% endif %} - - {% for area, meeting_groups, not_meeting_groups in ietf_areas %} -

- {{ area.acronym|upper }} {{ area.name }} -

- {% if meeting_groups %} - - - - - - - - - - - - {% for entry in meeting_groups %} - {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} - {% endfor %} - -
GroupArtifactsRecordingsSlidesInternet-Drafts
- {% endif %} - {% if not_meeting_groups %} -

- {{ area.name }} groups not meeting: - {% for entry in not_meeting_groups %} - {% if entry.name == "" %}{# do not show named sessions in this list #} - - {{ entry.group.acronym }} - {% if not forloop.last %},{% endif %} - {% endif %} - {% endfor %} -

- - - - - - - - - - - - {% for entry in not_meeting_groups %}{% if entry.has_materials %} - {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} - {% endif %}{% endfor %} - -
- {% endif %} - {% endfor %} - - {% if training %} -

Training

- + {% if not_meeting_groups %} +

+ {{ area.name }} groups not meeting: + {% for entry in not_meeting_groups %} + {% if entry.name == "" %}{# do not show named sessions in this list #} + + {{ entry.group.acronym }} + {% if not forloop.last %},{% endif %} + {% endif %} + {% endfor %} +

+
- - - - - - - + + + + + + + - {% for entry in training %} - {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=False only %} - {% endfor %} + {% for entry in not_meeting_groups %}{% if entry.has_materials %} + {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} + {% endif %}{% endfor %}
GroupArtifactsRecordingsSlidesInternet-Drafts
{% endif %} - - {% if iab %} -

- IAB Internet Architecture Board -

- - - - - - - - - - - - {% for entry in iab %} - {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} - {% endfor %} - -
- Group - - Artifacts - - Recordings - - Slides - - Internet-Drafts -
- {% endif %} - - {% if irtf.meeting_groups %} -

- IRTF Internet Research Task Force -

- - - - - - - - - - - - {% for entry in irtf.meeting_groups %} - {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} - {% endfor %} - -
- Group - - Artifacts - - Recordings - - Slides - - Internet-Drafts -
- {% if irtf.not_meeting_groups %} +{% endfor %} + +{% if training %} +

Training

+ + + + + + + + + + + + {% for entry in training %} + {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=False only %} + {% endfor %} + +
GroupArtifactsRecordingsSlidesInternet-Drafts
+{% endif %} + +{% if iab %} +

+ IAB Internet Architecture Board +

+ + + + + + + + + + + + {% for entry in iab %} + {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} + {% endfor %} + +
+ Group + + Artifacts + + Recordings + + Slides + + Internet-Drafts +
+{% endif %} + +{% if irtf.meeting_groups %} +

+ IRTF Internet Research Task Force +

+ + + + + + + + + + + + {% for entry in irtf.meeting_groups %} + {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} + {% endfor %} + +
+ Group + + Artifacts + + Recordings + + Slides + + Internet-Drafts +
+ {% if irtf.not_meeting_groups %}

IRTF groups not meeting: {% for entry in irtf.not_meeting_groups %} @@ -191,18 +167,18 @@

- - - - - - - + + + + + + + - {% for entry in irtf.not_meeting %}{% if entry.has_materials %} - {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} - {% endif %}{% endfor %} + {% for entry in irtf.not_meeting %}{% if entry.has_materials %} + {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} + {% endif %}{% endfor %}
{% endif %} @@ -211,35 +187,29 @@

Editorial Stream

- - - - - - - + + + + + + + - {% for entry in editorial %} - {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} - {% endfor %} + {% for entry in editorial %} + {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %} + {% endfor %}
- Group - - Artifacts - - Recordings - - Slides - - Internet-Drafts -
+ Group + + Artifacts + + Recordings + + Slides + + Internet-Drafts +
{% endif %} - {% endif %} -{% endcache %} -{% endblock %} -{% block js %} - -{% endblock %} +{% endif %} diff --git a/ietf/templates/meeting/proceedings_wrapper.html b/ietf/templates/meeting/proceedings_wrapper.html new file mode 100644 index 000000000..a20291a69 --- /dev/null +++ b/ietf/templates/meeting/proceedings_wrapper.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015, All Rights Reserved #} +{% load origin %} +{% load ietf_filters static %} +{% block pagehead %} + +{% endblock %} +{% block title %} + IETF {{ meeting.number }} + {% if not meeting.proceedings_final %}Draft{% endif %} + Proceedings +{% endblock %} +{% block content %} + {% origin %} + {% include 'meeting/proceedings/title.html' with meeting=meeting attendance=attendance only %} + {% if user|has_role:"Secretariat" and not meeting.proceedings_final %} + + Finalize proceedings + + {% endif %} + {{ proceedings_content }} +{% endblock %} +{% block js %} + +{% endblock %}