From c3d4cc1aea27f9cd13b7619641991b7ef39279f9 Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Mon, 10 Oct 2016 21:21:02 +0000 Subject: [PATCH] Add meeting proceedings introduction pages: Progress Report and Attendees. Commit ready for merge. - Legacy-Id: 12111 --- ietf/meeting/tests_views.py | 76 +++++++++- ietf/meeting/urls.py | 2 + ietf/meeting/utils.py | 28 +++- ietf/meeting/views.py | 30 +++- ietf/secr/proceedings/proc_utils.py | 132 +++++++----------- ietf/settings.py | 2 +- ietf/static/ietf/css/ietf.css | 6 + ietf/templates/meeting/proceedings.html | 8 +- .../meeting/proceedings_attendees.html | 14 ++ .../meeting/proceedings_attendees_table.html | 18 +++ .../meeting/proceedings_progress_report.html | 53 +++++++ requirements.txt | 1 + 12 files changed, 282 insertions(+), 88 deletions(-) create mode 100644 ietf/templates/meeting/proceedings_attendees.html create mode 100644 ietf/templates/meeting/proceedings_attendees_table.html create mode 100644 ietf/templates/meeting/proceedings_progress_report.html diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 87da4149c..d0eeedea7 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -10,7 +10,9 @@ import debug # pyflakes:ignore from django.core.urlresolvers import reverse as urlreverse from django.conf import settings from django.contrib.auth.models import User +from django.http import HttpRequest +from mock import patch, MagicMock from pyquery import PyQuery from StringIO import StringIO @@ -22,6 +24,7 @@ from ietf.meeting.helpers import send_interim_cancellation_notice from ietf.meeting.helpers import send_interim_minutes_reminder from ietf.meeting.models import Session, TimeSlot, Meeting from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting +from ietf.meeting.utils import finalize from ietf.name.models import SessionStatusName from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.utils.mail import outbox @@ -33,6 +36,7 @@ from ietf.meeting.factories import ( SessionFactory, SessionPresentationFactory, MeetingFactory, FloorPlanFactory ) from ietf.doc.factories import DocumentFactory + class MeetingTests(TestCase): def setUp(self): self.materials_dir = os.path.abspath(settings.TEST_MATERIALS_DIR) @@ -270,6 +274,44 @@ class MeetingTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) + @patch('urllib2.urlopen') + def test_proceedings_attendees(self, mock_urlopen): + mock_urlopen.return_value = StringIO('[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') + make_meeting_test_data() + + # add recent meeting + date = datetime.date(2016,4,3) + Meeting.objects.create(type_id='ietf',date=date,number=95) + url = urlreverse('ietf.secr.meetings.views.add') + post_data = dict(number='96',city='Berlin',date='2016-07-14',country='DE', + time_zone='Europe/Berlin',venue_name='Intercontinental Berlin', + venue_addr='', + idsubmit_cutoff_day_offset_00=13, + idsubmit_cutoff_day_offset_01=20, + idsubmit_cutoff_time_utc =datetime.timedelta(hours=23, minutes=59, seconds=59), + idsubmit_cutoff_warning_days =datetime.timedelta(days=21), + submission_start_day_offset=90, + submission_cutoff_day_offset=26, + submission_correction_day_offset=50, + ) + self.client.login(username='secretary', password='secretary+password') + response = self.client.post(url, post_data) + self.assertRedirects(response,urlreverse('ietf.secr.meetings.views.main')) + self.assertTrue(Meeting.objects.filter(number=96).exists()) + meeting = Meeting.objects.get(number=96) + + # finalize the meeting proceedings + finalize(HttpRequest(),meeting) + + # check attendees + url = urlreverse('ietf.meeting.views.proceedings_attendees',kwargs={'num':96}) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertTrue('Attendee List' in response.content) + q = PyQuery(response.content) + self.assertEqual(1,len(q("#id_attendees tbody tr"))) + + def test_proceedings_overview(self): '''Test proceedings IETF Overview page. Note: old meetings aren't supported so need to add a new meeting then test. @@ -299,6 +341,36 @@ class MeetingTests(TestCase): self.assertEqual(response.status_code, 200) self.assertTrue('The Internet Engineering Task Force' in response.content) + def test_proceedings_progress_report(self): + make_meeting_test_data() + + # add recent meeting + date = datetime.date(2016,4,3) + Meeting.objects.create(type_id='ietf',date=date,number=95) + url = urlreverse('ietf.secr.meetings.views.add') + post_data = dict(number='96',city='Berlin',date='2016-07-14',country='DE', + time_zone='Europe/Berlin',venue_name='Intercontinental Berlin', + venue_addr='', + idsubmit_cutoff_day_offset_00=13, + idsubmit_cutoff_day_offset_01=20, + idsubmit_cutoff_time_utc =datetime.timedelta(hours=23, minutes=59, seconds=59), + idsubmit_cutoff_warning_days =datetime.timedelta(days=21), + submission_start_day_offset=90, + submission_cutoff_day_offset=26, + submission_correction_day_offset=50, + ) + self.client.login(username='secretary', password='secretary+password') + response = self.client.post(url, post_data) + self.assertRedirects(response,urlreverse('ietf.secr.meetings.views.main')) + self.assertTrue(Meeting.objects.filter(number=96).exists()) + meeting = Meeting.objects.get(number=96) + + # check progress report + url = urlreverse('ietf.meeting.views.proceedings_progress_report',kwargs={'num':96}) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + self.assertTrue('Progress Report' in response.content) + def test_feed(self): meeting = make_meeting_test_data() session = Session.objects.filter(meeting=meeting, group__acronym="mars").first() @@ -1252,7 +1324,9 @@ class IphoneAppJsonTests(TestCase): self.assertEqual(r.status_code,200) class FinalizeProceedingsTests(TestCase): - def test_finalize_proceedings(self): + @patch('urllib2.urlopen') + def test_finalize_proceedings(self, mock_urlopen): + mock_urlopen.return_value = StringIO('[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') make_meeting_test_data() meeting = Meeting.objects.filter(type_id='ietf').order_by('id').last() meeting.session_set.filter(group__acronym='mars').first().sessionpresentation_set.create(document=Document.objects.filter(type='draft').first(),rev=None) diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py index 295ddb410..ef30e5891 100644 --- a/ietf/meeting/urls.py +++ b/ietf/meeting/urls.py @@ -78,7 +78,9 @@ type_ietf_only_patterns_id_optional = [ url(r'^proceedings(?:.html)?/?$', views.proceedings), url(r'^proceedings(?:.html)?/finalize/?$', views.finalize_proceedings), url(r'^proceedings/acknowledgements/$', views.proceedings_acknowledgements), + url(r'^proceedings/attendees/$', views.proceedings_attendees), url(r'^proceedings/overview/$', views.proceedings_overview), + url(r'^proceedings/progress-report/$', views.proceedings_progress_report), ] urlpatterns = [ diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 8a5a08ed2..f35a9f476 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -1,5 +1,13 @@ import datetime +import json +import urllib2 +import urlparse +from django.conf import settings +from django.contrib import messages +from django.template.loader import render_to_string + +from ietf.dbtemplate.models import DBTemplate from ietf.meeting.models import Session from ietf.group.utils import can_manage_materials @@ -72,7 +80,7 @@ def sort_sessions(sessions): return meeting_sorted -def finalize(meeting): +def finalize(request, meeting): end_date = meeting.end_date() end_time = datetime.datetime.combine(end_date, datetime.datetime.min.time())+datetime.timedelta(days=1) for session in meeting.session_set.all(): @@ -83,6 +91,24 @@ def finalize(meeting): else: sp.rev = '00' sp.save() + # get attendees + url = urlparse.urljoin(settings.REGISTRATION_ATTENDEES_BASE_URL,meeting.number) + try: + attendees = json.load(urllib2.urlopen(url)) + except (ValueError, urllib2.HTTPError): + messages.warning(request,'Could not retrieve attendee list from registration system (%s)' % url, fail_silently=True) + attendees = [] + + if attendees: + attendees = sorted(attendees, key = lambda a: a['LastName']) + content = render_to_string('meeting/proceedings_attendees_table.html', { + 'attendees':attendees}) + template = DBTemplate.objects.create( + path='/meeting/proceedings/%s/attendees.html' % meeting.number, + title='IETF %s Attendee List' % meeting.number, + type_id='django', + content=content) + meeting.proceedings_final = True meeting.save() return diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 0303e6c2a..b7b5add8e 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -56,6 +56,7 @@ from ietf.meeting.helpers import send_interim_approval_request from ietf.meeting.helpers import send_interim_announcement_request from ietf.meeting.utils import finalize from ietf.secr.proceedings.utils import handle_upload_file +from ietf.secr.proceedings.proc_utils import get_progress_stats from ietf.utils.mail import send_mail_message from ietf.utils.pipe import pipe from ietf.utils.pdf import pdf_pages @@ -1984,7 +1985,7 @@ def finalize_proceedings(request, num=None): raise Http404 if request.method=='POST': - finalize(meeting) + finalize(request, meeting) return HttpResponseRedirect(reverse('ietf.meeting.views.proceedings',kwargs={'num':meeting.number})) return render(request, "meeting/finalize.html", {'meeting':meeting,}) @@ -1999,6 +2000,22 @@ def proceedings_acknowledgements(request, num=None): 'meeting': meeting, }) + +@role_required('Secretariat') +def proceedings_attendees(request, num=None): + + meeting = get_meeting(num) + if meeting.number < 95: + return HttpResponseRedirect( 'https://www.ietf.org/proceedings/%s/attendees.html' % num ) + overview_template = '/meeting/proceedings/%s/attendees.html' % meeting.number + template = render_to_string(overview_template, {}) + + return render(request, "meeting/proceedings_attendees.html", { + 'meeting': meeting, + 'template': template, + }) + + @role_required('Secretariat') def proceedings_overview(request, num=None): '''Display Overview for given meeting''' @@ -2013,6 +2030,17 @@ def proceedings_overview(request, num=None): 'template': template, }) +@role_required('Secretariat') +def proceedings_progress_report(request, num=None): + meeting = get_meeting(num) + if meeting.number < 95: + return HttpResponseRedirect( 'https://www.ietf.org/proceedings/%s/progress-report.html' % num ) + sdate = meeting.previous_meeting().date + edate = meeting.date + context = get_progress_stats(sdate,edate) + context['meeting'] = meeting + return render(request, "meeting/proceedings_progress_report.html", context) + class OldUploadRedirect(RedirectView): def get_redirect_url(self, **kwargs): return reverse_lazy('ietf.meeting.views.session_details',kwargs=self.kwargs) diff --git a/ietf/secr/proceedings/proc_utils.py b/ietf/secr/proceedings/proc_utils.py index 30e8299e0..932ee0c6c 100644 --- a/ietf/secr/proceedings/proc_utils.py +++ b/ietf/secr/proceedings/proc_utils.py @@ -106,97 +106,67 @@ def mycomp(timeslot): def get_progress_stats(sdate,edate): ''' - This function takes a date range and produces a dictionary of statistics / objects for use - in a progress report. Generally the end date will be the date of the last meeting + This function takes a date range and produces a dictionary of statistics / objects for + use in a progress report. Generally the end date will be the date of the last meeting and the start date will be the date of the meeting before that. ''' data = {} data['sdate'] = sdate data['edate'] = edate - # Activty Report Section - new_docs = Document.objects.filter(type='draft').filter(docevent__type='new_revision', - docevent__newrevisiondocevent__rev='00', - docevent__time__gte=sdate, - docevent__time__lt=edate) - data['new'] = new_docs.count() - data['updated'] = 0 - data['updated_more'] = 0 - for d in new_docs: - updates = d.docevent_set.filter(type='new_revision',time__gte=sdate,time__lt=edate).count() - if updates > 1: - data['updated'] += 1 - if updates > 2: - data['updated_more'] +=1 - - # calculate total documents updated, not counting new, rev=00 - result = set() events = DocEvent.objects.filter(doc__type='draft',time__gte=sdate,time__lt=edate) - for e in events.filter(type='new_revision').exclude(newrevisiondocevent__rev='00'): - result.add(e.doc) - data['total_updated'] = len(result) + + data['actions_count'] = events.filter(type='iesg_approved').count() + data['last_calls_count'] = events.filter(type='sent_last_call').count() + new_draft_events = events.filter(newrevisiondocevent__rev='00') + new_drafts = list(set([ e.doc_id for e in new_draft_events ])) + data['new_drafts_count'] = len(new_drafts) + data['new_drafts_updated_count'] = events.filter(doc__in=new_drafts,newrevisiondocevent__rev='01').count() + data['new_drafts_updated_more_count'] = events.filter(doc__in=new_drafts,newrevisiondocevent__rev='02').count() + + update_events = events.filter(type='new_revision').exclude(doc__in=new_drafts) + data['updated_drafts_count'] = len(set([ e.doc_id for e in update_events ])) + + # Calculate Final Four Weeks stats (ffw) + ffwdate = edate - datetime.timedelta(days=28) + ffw_new_count = events.filter(time__gte=ffwdate,newrevisiondocevent__rev='00').count() + try: + ffw_new_percent = format(ffw_new_count / float(data['new_drafts_count']),'.0%') + except ZeroDivisionError: + ffw_new_percent = 0 + + data['ffw_new_count'] = ffw_new_count + data['ffw_new_percent'] = ffw_new_percent + + ffw_update_events = events.filter(time__gte=ffwdate,type='new_revision').exclude(doc__in=new_drafts) + ffw_update_count = len(set([ e.doc_id for e in ffw_update_events ])) + try: + ffw_update_percent = format(ffw_update_count / float(data['updated_drafts_count']),'.0%') + except ZeroDivisionError: + ffw_update_percent = 0 + + data['ffw_update_count'] = ffw_update_count + data['ffw_update_percent'] = ffw_update_percent - # calculate sent last call - data['last_call'] = events.filter(type='sent_last_call').count() + rfcs = events.filter(type='published_rfc') + data['rfcs'] = rfcs.select_related('doc').select_related('doc__group').select_related('doc__intended_std_level') - # calculate approved - data['approved'] = events.filter(type='iesg_approved').count() + data['counts'] = {'std':rfcs.filter(doc__intended_std_level__in=('ps','ds','std')).count(), + 'bcp':rfcs.filter(doc__intended_std_level='bcp').count(), + 'exp':rfcs.filter(doc__intended_std_level='exp').count(), + 'inf':rfcs.filter(doc__intended_std_level='inf').count()} - # get 4 weeks - ff1_date = edate - datetime.timedelta(days=28) - ff_docs = Document.objects.filter(type='draft').filter(docevent__type='new_revision', - docevent__newrevisiondocevent__rev='00', - docevent__time__gte=ff1_date, - docevent__time__lt=edate) - ff_new_count = ff_docs.count() - ff_new_percent = format(ff_new_count / float(data['new']),'.0%') - - # calculate total documents updated in final four weeks, not counting new, rev=00 - result = set() - events = DocEvent.objects.filter(doc__type='draft',time__gte=ff1_date,time__lt=edate) - for e in events.filter(type='new_revision').exclude(newrevisiondocevent__rev='00'): - result.add(e.doc) - ff_update_count = len(result) - ff_update_percent = format(ff_update_count / float(data['total_updated']),'.0%') - - data['ff_new_count'] = ff_new_count - data['ff_new_percent'] = ff_new_percent - data['ff_update_count'] = ff_update_count - data['ff_update_percent'] = ff_update_percent - - # Progress Report Section - data['docevents'] = DocEvent.objects.filter(doc__type='draft',time__gte=sdate,time__lt=edate) - data['action_events'] = data['docevents'].filter(type='iesg_approved') - data['lc_events'] = data['docevents'].filter(type='sent_last_call') - - data['new_groups'] = Group.objects.filter(type='wg', - groupevent__changestategroupevent__state='active', - groupevent__time__gte=sdate, - groupevent__time__lt=edate) - - data['concluded_groups'] = Group.objects.filter(type='wg', - groupevent__changestategroupevent__state='conclude', - groupevent__time__gte=sdate, - groupevent__time__lt=edate) - - data['new_docs'] = Document.objects.filter(type='draft').filter(docevent__type='new_revision', - docevent__time__gte=sdate, - docevent__time__lt=edate).distinct() - - data['rfcs'] = DocEvent.objects.filter(type='published_rfc', - doc__type='draft', - time__gte=sdate, - time__lt=edate) - - # attach the ftp URL for use in the template - for event in data['rfcs']: - num = get_rfc_num(event.doc) - event.ftp_url = 'ftp://ftp.ietf.org/rfc/rfc%s.txt' % num - - data['counts'] = {'std':data['rfcs'].filter(doc__intended_std_level__in=('ps','ds','std')).count(), - 'bcp':data['rfcs'].filter(doc__intended_std_level='bcp').count(), - 'exp':data['rfcs'].filter(doc__intended_std_level='exp').count(), - 'inf':data['rfcs'].filter(doc__intended_std_level='inf').count()} + data['new_groups'] = Group.objects.filter( + type='wg', + groupevent__changestategroupevent__state='active', + groupevent__time__gte=sdate, + groupevent__time__lt=edate) + + data['concluded_groups'] = Group.objects.filter( + type='wg', + groupevent__changestategroupevent__state='conclude', + groupevent__time__gte=sdate, + groupevent__time__lt=edate) return data diff --git a/ietf/settings.py b/ietf/settings.py index 54a7ec1f6..6e4b9c901 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -599,7 +599,7 @@ SECR_INTERIM_LISTING_DIR = '/a/www/www6/meeting/interim' SECR_MAX_UPLOAD_SIZE = 40960000 SECR_PROCEEDINGS_DIR = '/a/www/www6s/proceedings/' SECR_PPT2PDF_COMMAND = ['/usr/bin/soffice','--headless','--convert-to','pdf','--outdir'] - +REGISTRATION_ATTENDEES_BASE_URL = 'https://ietf.org/registration/attendees/' USE_ETAGS=True PRODUCTION_TIMEZONE = "America/Los_Angeles" diff --git a/ietf/static/ietf/css/ietf.css b/ietf/static/ietf/css/ietf.css index c08097b4a..76a504810 100644 --- a/ietf/static/ietf/css/ietf.css +++ b/ietf/static/ietf/css/ietf.css @@ -544,6 +544,12 @@ ul.errorlist { color: #555; } +/* === Proceedings ===================================================== */ +ul.progress-section { + list-style-type: none; + margin-bottom: 2em; +} + /* === Font-Awesome ========================================================= */ .btn .fa-stack { width: 1em; height: 1em; } diff --git a/ietf/templates/meeting/proceedings.html b/ietf/templates/meeting/proceedings.html index ea6e6eb4b..f97955a0f 100644 --- a/ietf/templates/meeting/proceedings.html +++ b/ietf/templates/meeting/proceedings.html @@ -33,13 +33,15 @@ {% load cache %} {% cache 900 ietf_meeting_proceedings meeting.number cache_version %} + {% if meeting.proceedings_final %}

Introduction

Acknowledgements
IETF Overview
- Progress Report
- Attendees
-
+ Progress Report
+ Attendees
+ + {% endif %} {% with "True" as show_agenda %} diff --git a/ietf/templates/meeting/proceedings_attendees.html b/ietf/templates/meeting/proceedings_attendees.html new file mode 100644 index 000000000..fede3b14a --- /dev/null +++ b/ietf/templates/meeting/proceedings_attendees.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015, All Rights Reserved #} +{% load origin markup_tags %} + +{% block title %}IETF {{ meeting.number }} Proceedings {% endblock %} + +{% block content %} + {% origin %} +

IETF {{ meeting.number }} Proceedings

+

Attendee List of IETF {{ meeting.number }} Meeting

+ + {{ template|safe }} + +{% endblock %} diff --git a/ietf/templates/meeting/proceedings_attendees_table.html b/ietf/templates/meeting/proceedings_attendees_table.html new file mode 100644 index 000000000..5cba89484 --- /dev/null +++ b/ietf/templates/meeting/proceedings_attendees_table.html @@ -0,0 +1,18 @@ + + + + + + + + + + {% for attendee in attendees %} + + + + + + {% endfor %} + +
Last NameFirst NameOrganization
{{ attendee.LastName }}{{ attendee.FirstName }}{{ attendee.Company }}
\ No newline at end of file diff --git a/ietf/templates/meeting/proceedings_progress_report.html b/ietf/templates/meeting/proceedings_progress_report.html new file mode 100644 index 000000000..dd1f530d7 --- /dev/null +++ b/ietf/templates/meeting/proceedings_progress_report.html @@ -0,0 +1,53 @@ +{% extends "base.html" %} +{% load ams_filters %} + +{% block title %}IETF {{ meeting.number }} Proceedings - Progress Report{% endblock %} + +{% block content %} + +

IETF {{ meeting.number }} Proceedings

+

IETF Progress Report

+ +

{{ sdate|date:"d-F-y" }} to {{ edate|date:"d-F-y" }}

+ + +

{{ new_groups.count }} New Working Group(s) formed this period

+ + +

{{ concluded_groups.count }} Working Group(s) concluded this period

+ + +

{{ rfcs.count }} RFCs published this period

+

{{ counts.std }} Standards Track; {{ counts.bcp }} BCP; {{ counts.exp }} Experimental; {{ counts.inf }} Informational

+ + + {% for rfc in rfcs %} + + + + + + + + {% endfor %} +
{{ rfc.doc.canonical_name|upper }}{{ rfc.doc.intended_std_level.name|abbr_status }}({{ rfc.doc.group.acronym }}){{ rfc.time|date:"F Y" }}{{ rfc.doc.title }}
+ +{% endblock %} diff --git a/requirements.txt b/requirements.txt index 42558dbbb..1f22ede4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ html5lib>=0.90,<0.99999999 # ietf.utils.html needs a rewrite for html5lib 1.x -- jsonfield>=1.0.3 # for SubmissionCheck. This is https://github.com/bradjasper/django-jsonfield/. #lxml>=3.4.0 # from PyQuery; mimeparse>=0.1.3 # from TastyPie +mock>=2.0.0 MySQL-python>=1.2.5 pathlib>=1.0 Pillow>=3.0