From b6b45166bfc37cfca923eda379c6637207d97070 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Wed, 31 May 2017 23:08:23 +0000 Subject: [PATCH] Added email address to the MeetingRegistration model, and also to the fields imported from ietf.org/registration/attendees/NN . Changed the bin counting code for attendee-related stats to use names plus email as unique string, rather than only names. - Legacy-Id: 13481 --- ietf/bin/fetch-meeting-registration-data | 4 +-- ietf/stats/backfill_data.py | 2 ++ ietf/stats/models.py | 1 + ietf/stats/resources.py | 1 + ietf/stats/tests.py | 6 ++-- ietf/stats/utils.py | 16 ++++++++-- ietf/stats/views.py | 37 ++++++++++++++---------- 7 files changed, 43 insertions(+), 24 deletions(-) diff --git a/ietf/bin/fetch-meeting-registration-data b/ietf/bin/fetch-meeting-registration-data index 6d854d7a3..5f75df3af 100755 --- a/ietf/bin/fetch-meeting-registration-data +++ b/ietf/bin/fetch-meeting-registration-data @@ -39,8 +39,8 @@ else: sys.exit(1) for meeting in meetings: - total = get_meeting_registration_data(meeting) - msg = "Fetched data for meeting {}: {} registrations added".format(meeting.number, total) + added, processed, total = get_meeting_registration_data(meeting) + msg = "Fetched data for meeting {}: {} processed, {} added, {} in table".format(meeting.number, processed, added, total) if sys.stdout.isatty(): print(msg) # make debugging a bit easier else: diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py index 44ce6d9a3..52d66137a 100644 --- a/ietf/stats/backfill_data.py +++ b/ietf/stats/backfill_data.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import sys, os, argparse basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) diff --git a/ietf/stats/models.py b/ietf/stats/models.py index 051af882d..441d7e0cf 100644 --- a/ietf/stats/models.py +++ b/ietf/stats/models.py @@ -50,6 +50,7 @@ class MeetingRegistration(models.Model): affiliation = models.CharField(blank=True, max_length=255) country_code = models.CharField(max_length=2) # ISO 3166 person = models.ForeignKey(Person, blank=True, null=True) + email = models.EmailField(blank=True, null=True) def __unicode__(self): return u"{} {}".format(self.first_name, self.last_name) diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py index 979863762..ab5f50803 100644 --- a/ietf/stats/resources.py +++ b/ietf/stats/resources.py @@ -63,6 +63,7 @@ class MeetingRegistrationResource(ModelResource): "last_name": ALL, "affiliation": ALL, "country_code": ALL, + "email": ALL, "person": ALL_WITH_RELATIONS } api.stats.register(MeetingRegistrationResource()) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 1fbc2c70c..42a067d87 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -102,9 +102,9 @@ class StatisticsTests(TestCase): # create some data for the statistics make_test_data() meeting = MeetingFactory(type_id='ietf', date=datetime.date.today(), number="96") - MeetingRegistration.objects.create(first_name='John', last_name='Smith', country_code='US', meeting=meeting) + MeetingRegistration.objects.create(first_name='John', last_name='Smith', country_code='US', email="john.smith@example.us", meeting=meeting) CountryAlias.objects.get_or_create(alias="US", country=CountryName.objects.get(slug="US")) - MeetingRegistration.objects.create(first_name='Jaume', last_name='Guillaume', country_code='FR', meeting=meeting) + MeetingRegistration.objects.create(first_name='Jaume', last_name='Guillaume', country_code='FR', email="jaume.guillaume@example.fr", meeting=meeting) CountryAlias.objects.get_or_create(alias="FR", country=CountryName.objects.get(slug="FR")) # check redirect @@ -193,7 +193,7 @@ class StatisticsTests(TestCase): def test_get_meeting_registration_data(self, mock_get): response = Response() response.status_code = 200 - response._content = '[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]' + response._content = '[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US","Email":"john.doe@example.us"}]' mock_get.return_value = response meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96") get_meeting_registration_data(meeting) diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py index 9b0f4cd70..d700395d6 100644 --- a/ietf/stats/utils.py +++ b/ietf/stats/utils.py @@ -214,13 +214,17 @@ def get_meeting_registration_data(meeting): """"Retrieve registration attendee data and summary statistics. Returns number of Registration records created.""" num_created = 0 + num_processed = 0 response = requests.get(settings.REGISTRATION_ATTENDEES_BASE_URL + meeting.number) if response.status_code == 200: decoded = [] try: decoded = response.json() except ValueError: - pass + if response.content.strip() == 'Invalid meeting': + pass + else: + raise RuntimeError("Could not decode response from registrations API: '%s...'" % (response.content[:64], )) for registration in decoded: object, created = MeetingRegistration.objects.get_or_create( @@ -228,10 +232,16 @@ def get_meeting_registration_data(meeting): first_name=registration['FirstName'], last_name=registration['LastName'], affiliation=registration['Company'], - country_code=registration['Country']) + country_code=registration['Country'], + email=registration['Email'], + ) if created: num_created += 1 - return num_created + num_processed += 1 + else: + raise RuntimeError("Bad response from registrations API: %s, '%s'" % (response.status_code, response.content)) + num_total = MeetingRegistration.objects.filter(meeting_id=meeting.pk).count() + return num_created, num_processed, num_total diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 71dc9896f..549f74bae 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,8 +1,10 @@ +import os +import calendar import datetime +import email.utils import itertools import json -import calendar -import os + from collections import defaultdict from django.shortcuts import render @@ -780,6 +782,9 @@ def meeting_stats(request, num=None, stats_type=None): if alias.alias.isupper() } + def reg_name(r): + return email.utils.formataddr(((r.first_name + u" " + r.last_name).strip(), r.email)) + if meeting and any(stats_type == t[0] for t in possible_stats_types): registrations = MeetingRegistration.objects.filter(meeting=meeting) @@ -794,7 +799,7 @@ def meeting_stats(request, num=None, stats_type=None): eu_countries = set(CountryName.objects.filter(in_eu=True)) for r in registrations: - name = (r.first_name + " " + r.last_name).strip() + name = reg_name(r) c = country_mapping.get(r.country_code) bins[c.name if c else ""].add(name) @@ -831,7 +836,7 @@ def meeting_stats(request, num=None, stats_type=None): country_mapping = get_country_mapping(registrations) for r in registrations: - name = (r.first_name + " " + r.last_name).strip() + name = reg_name(r) c = country_mapping.get(r.country_code) bins[c.continent.name if c else ""].add(name) @@ -853,16 +858,16 @@ def meeting_stats(request, num=None, stats_type=None): elif not meeting and any(stats_type == t[0] for t in possible_stats_types): template_name = "overview" - registrations = MeetingRegistration.objects.filter(meeting__type="ietf") + registrations = MeetingRegistration.objects.filter(meeting__type="ietf").select_related('meeting') if stats_type == "overview": stats_title = "Number of registrations per meeting" bins = defaultdict(set) - for first_name, last_name, meeting_number in registrations.values_list("first_name", "last_name", "meeting__number"): - meeting_number = int(meeting_number) - name = (first_name + " " + last_name).strip() + for r in registrations: + meeting_number = int(r.meeting.number) + name = reg_name(r) bins[meeting_number].add(name) @@ -891,10 +896,10 @@ def meeting_stats(request, num=None, stats_type=None): bins = defaultdict(set) - for first_name, last_name, country_code, meeting_number in registrations.values_list("first_name", "last_name", "country_code", "meeting__number"): - meeting_number = int(meeting_number) - name = (first_name + " " + last_name).strip() - c = country_mapping.get(country_code) + for r in registrations: + meeting_number = int(r.meeting.number) + name = reg_name(r) + c = country_mapping.get(r.country_code) if c: bins[(meeting_number, c.name)].add(name) @@ -911,10 +916,10 @@ def meeting_stats(request, num=None, stats_type=None): bins = defaultdict(set) - for first_name, last_name, country_code, meeting_number in registrations.values_list("first_name", "last_name", "country_code", "meeting__number"): - meeting_number = int(meeting_number) - name = (first_name + " " + last_name).strip() - c = country_mapping.get(country_code) + for r in registrations: + meeting_number = int(r.meeting.number) + name = reg_name(r) + c = country_mapping.get(r.country_code) if c: bins[(meeting_number, c.continent.name)].add(name)