diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 8ddad4e9f..ee0d27bb8 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -212,9 +212,9 @@ class ToOneField(tastypie.fields.ToOneField): if not foreign_obj: if not self.null: if callable(self.attribute): - raise ApiFieldError("The related resource for resource %s could not be found." % (previous_obj)) + raise ApiFieldError(u"The related resource for resource %s could not be found." % (previous_obj)) else: - raise ApiFieldError("The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr)) + raise ApiFieldError(u"The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr)) return None fk_resource = self.get_related_resource(foreign_obj) diff --git a/ietf/bin/fetch-meeting-registration-data b/ietf/bin/fetch-meeting-registration-data new file mode 100755 index 000000000..6d854d7a3 --- /dev/null +++ b/ietf/bin/fetch-meeting-registration-data @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys, os, argparse + +basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +sys.path = [ basedir ] + sys.path +os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" + +virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py") +if os.path.exists(virtualenv_activation): + execfile(virtualenv_activation, dict(__file__=virtualenv_activation)) + +import django +django.setup() + +parser = argparse.ArgumentParser() +parser.add_argument("--meeting", help="meeting to fetch data for") +parser.add_argument("--all", action="store_true", help="fetch data for all meetings") +parser.add_argument("--latest", type=int, help="fetch data for latest N meetings") +args = parser.parse_args() + + +import syslog + +from ietf.meeting.models import Meeting + +from ietf.stats.utils import get_meeting_registration_data + +meetings = Meeting.objects.none() +if args.meeting: + meetings = Meeting.objects.filter(number=args.meeting, type="ietf") +elif args.all: + meetings = Meeting.objects.filter(type="ietf").order_by("date") +elif args.latest: + meetings = Meeting.objects.filter(type="ietf").order_by("-date")[:args.latest] +else: + print("Please use one of --meeting, --all or --latest") + sys.exit(1) + +for meeting in meetings: + total = get_meeting_registration_data(meeting) + msg = "Fetched data for meeting {}: {} registrations added".format(meeting.number, total) + if sys.stdout.isatty(): + print(msg) # make debugging a bit easier + else: + syslog.syslog(msg) diff --git a/ietf/bin/find-submission-confirmation-email-in-postfix-log b/ietf/bin/find-submission-confirmation-email-in-postfix-log index 3df3c78be..69bdcc331 100755 --- a/ietf/bin/find-submission-confirmation-email-in-postfix-log +++ b/ietf/bin/find-submission-confirmation-email-in-postfix-log @@ -66,7 +66,7 @@ if "<" in from_email: submission = Submission.objects.filter(name=draft).latest('submission_date') document = Document.objects.get(name=draft) -emails = [ author.address for author in document.authors.all() ] +emails = [ author.email.address for author in document.documentauthor_set.all() if author.email ] timestrings = [] for file in [ Path(settings.INTERNET_DRAFT_PATH) / ("%s-%s.txt"%(draft, submission.rev)), diff --git a/ietf/bin/generate-draft-aliases b/ietf/bin/generate-draft-aliases index 7216aa2bc..5852e341b 100755 --- a/ietf/bin/generate-draft-aliases +++ b/ietf/bin/generate-draft-aliases @@ -65,7 +65,7 @@ def get_draft_authors_emails(draft): " Get list of authors for the given draft." # This feels 'correct'; however, it creates fairly large delta - return [email.email_address() for email in draft.authors.all()] + return [author.email.email_address() for author in draft.documentauthor_set.all() if author.email.email_address()] # This gives fairly small delta compared to current state, # however, it seems to be wrong (doesn't check for emails being diff --git a/ietf/community/migrations/0004_cleanup_data.py b/ietf/community/migrations/0004_cleanup_data.py index 6ee14c951..ab1c24f4e 100644 --- a/ietf/community/migrations/0004_cleanup_data.py +++ b/ietf/community/migrations/0004_cleanup_data.py @@ -57,7 +57,7 @@ def port_rules_to_typed_system(apps, schema_editor): elif rule.rule_type in ["author", "author_rfc"]: - found_persons = list(try_to_uniquify_person(rule, Person.objects.filter(email__documentauthor__id__gte=1).filter(name__icontains=rule.value).distinct())) + found_persons = list(try_to_uniquify_person(rule, Person.objects.filter(documentauthor__id__gte=1).filter(name__icontains=rule.value).distinct())) if found_persons: rule.person = found_persons[0] diff --git a/ietf/community/tests.py b/ietf/community/tests.py index 9efba6d17..77899544e 100644 --- a/ietf/community/tests.py +++ b/ietf/community/tests.py @@ -35,7 +35,7 @@ class CommunityListTests(TestCase): rule_state_iesg = SearchRule.objects.create(rule_type="state_iesg", state=State.objects.get(type="draft-iesg", slug="lc"), community_list=clist) - rule_author = SearchRule.objects.create(rule_type="author", state=State.objects.get(type="draft", slug="active"), person=Person.objects.filter(email__documentauthor__document=draft).first(), community_list=clist) + rule_author = SearchRule.objects.create(rule_type="author", state=State.objects.get(type="draft", slug="active"), person=Person.objects.filter(documentauthor__document=draft).first(), community_list=clist) rule_ad = SearchRule.objects.create(rule_type="ad", state=State.objects.get(type="draft", slug="active"), person=draft.ad, community_list=clist) @@ -117,7 +117,7 @@ class CommunityListTests(TestCase): r = self.client.post(url, { "action": "add_rule", "rule_type": "author_rfc", - "author_rfc-person": Person.objects.filter(email__documentauthor__document=draft).first().pk, + "author_rfc-person": Person.objects.filter(documentauthor__document=draft).first().pk, "author_rfc-state": State.objects.get(type="draft", slug="rfc").pk, }) self.assertEqual(r.status_code, 302) diff --git a/ietf/community/utils.py b/ietf/community/utils.py index ddd62d9c4..0a573c195 100644 --- a/ietf/community/utils.py +++ b/ietf/community/utils.py @@ -90,7 +90,7 @@ def docs_matching_community_list_rule(rule): elif rule.rule_type.startswith("state_"): return docs.filter(states=rule.state) elif rule.rule_type in ["author", "author_rfc"]: - return docs.filter(states=rule.state, documentauthor__author__person=rule.person) + return docs.filter(states=rule.state, documentauthor__person=rule.person) elif rule.rule_type == "ad": return docs.filter(states=rule.state, ad=rule.person) elif rule.rule_type == "shepherd": @@ -123,7 +123,7 @@ def community_list_rules_matching_doc(doc): rules |= SearchRule.objects.filter( rule_type__in=["author", "author_rfc"], state__in=states, - person__in=list(Person.objects.filter(email__documentauthor__document=doc)), + person__in=list(Person.objects.filter(documentauthor__document=doc)), ) if doc.ad_id: diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index db753e070..58222b24e 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -26,7 +26,7 @@ class DocAliasInline(admin.TabularInline): class DocAuthorInline(admin.TabularInline): model = DocumentAuthor - raw_id_fields = ['author', ] + raw_id_fields = ['person', 'email'] extra = 1 class RelatedDocumentInline(admin.TabularInline): @@ -100,7 +100,7 @@ class DocumentAdmin(admin.ModelAdmin): list_display = ['name', 'rev', 'group', 'pages', 'intended_std_level', 'author_list', 'time'] search_fields = ['name'] list_filter = ['type'] - raw_id_fields = ['authors', 'group', 'shepherd', 'ad'] + raw_id_fields = ['group', 'shepherd', 'ad'] inlines = [DocAliasInline, DocAuthorInline, RelatedDocumentInline, ] form = DocumentForm @@ -123,7 +123,7 @@ class DocHistoryAdmin(admin.ModelAdmin): list_display = ['doc', 'rev', 'state', 'group', 'pages', 'intended_std_level', 'author_list', 'time'] search_fields = ['doc__name'] ordering = ['time', 'doc', 'rev'] - raw_id_fields = ['doc', 'authors', 'group', 'shepherd', 'ad'] + raw_id_fields = ['doc', 'group', 'shepherd', 'ad'] def state(self, instance): return instance.get_state() @@ -175,8 +175,8 @@ class BallotPositionDocEventAdmin(DocEventAdmin): admin.site.register(BallotPositionDocEvent, BallotPositionDocEventAdmin) class DocumentAuthorAdmin(admin.ModelAdmin): - list_display = ['id', 'document', 'author', 'order'] - search_fields = [ 'document__name', 'author__address', ] - raw_id_fields = ['document', 'author', ] + list_display = ['id', 'document', 'person', 'email', 'affiliation', 'country', 'order'] + search_fields = ['document__docalias__name', 'person__name', 'email__address', 'affiliation', 'country'] + raw_id_fields = ["document", "person", "email"] admin.site.register(DocumentAuthor, DocumentAuthorAdmin) diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index f0802fa8a..569e3e5d1 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -46,8 +46,8 @@ class DocumentFactory(factory.DjangoModelFactory): def authors(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument if create and extracted: order = 0 - for email in extracted: - DocumentAuthor.objects.create(document=obj, author=email, order=order) + for person in extracted: + DocumentAuthor.objects.create(document=obj, person=person, order=order) order += 1 @classmethod diff --git a/ietf/doc/migrations/0026_author_revamp_and_extra_attributes.py b/ietf/doc/migrations/0026_author_revamp_and_extra_attributes.py new file mode 100644 index 000000000..e530b37c8 --- /dev/null +++ b/ietf/doc/migrations/0026_author_revamp_and_extra_attributes.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0019_formallanguagename'), + ('doc', '0025_auto_20170307_0146'), + ] + + operations = [ + migrations.AddField( + model_name='dochistory', + name='words', + field=models.IntegerField(null=True, blank=True), + ), + migrations.AddField( + model_name='document', + name='words', + field=models.IntegerField(null=True, blank=True), + ), + migrations.AddField( + model_name='dochistory', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), + migrations.AddField( + model_name='document', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), + migrations.RemoveField( + model_name='dochistory', + name='authors', + ), + migrations.RemoveField( + model_name='document', + name='authors', + ), + migrations.AddField( + model_name='dochistoryauthor', + name='affiliation', + field=models.CharField(help_text=b'Organization/company used by author for submission', max_length=100, blank=True), + ), + migrations.AddField( + model_name='dochistoryauthor', + name='country', + field=models.CharField(blank=True, help_text=b'Country used by author for submission', max_length=255), + ), + migrations.RenameField( + model_name='dochistoryauthor', + old_name='author', + new_name='email', + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='email', + field=models.ForeignKey(blank=True, to='person.Email', help_text=b'Email address used by author for submission', null=True), + ), + migrations.AddField( + model_name='dochistoryauthor', + name='person', + field=models.ForeignKey(blank=True, to='person.Person', null=True), + ), + migrations.AddField( + model_name='documentauthor', + name='affiliation', + field=models.CharField(help_text=b'Organization/company used by author for submission', max_length=100, blank=True), + ), + migrations.AddField( + model_name='documentauthor', + name='country', + field=models.CharField(blank=True, help_text=b'Country used by author for submission', max_length=255), + ), + migrations.RenameField( + model_name='documentauthor', + old_name='author', + new_name='email', + ), + migrations.AlterField( + model_name='documentauthor', + name='email', + field=models.ForeignKey(blank=True, to='person.Email', help_text=b'Email address used by author for submission', null=True), + ), + migrations.AddField( + model_name='documentauthor', + name='person', + field=models.ForeignKey(blank=True, to='person.Person', null=True), + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='document', + field=models.ForeignKey(related_name='documentauthor_set', to='doc.DocHistory'), + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='order', + field=models.IntegerField(default=1), + ), + migrations.RunSQL("update doc_documentauthor a inner join person_email e on a.email_id = e.address set a.person_id = e.person_id;", migrations.RunSQL.noop), + migrations.RunSQL("update doc_dochistoryauthor a inner join person_email e on a.email_id = e.address set a.person_id = e.person_id;", migrations.RunSQL.noop), + migrations.AlterField( + model_name='documentauthor', + name='person', + field=models.ForeignKey(to='person.Person'), + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='person', + field=models.ForeignKey(to='person.Person'), + ), + ] diff --git a/ietf/doc/migrations/0027_remove_fake_email_adresses.py b/ietf/doc/migrations/0027_remove_fake_email_adresses.py new file mode 100644 index 000000000..7aff8dd41 --- /dev/null +++ b/ietf/doc/migrations/0027_remove_fake_email_adresses.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def fix_invalid_emails(apps, schema_editor): + Email = apps.get_model("person", "Email") + Role = apps.get_model("group", "Role") + RoleHistory = apps.get_model("group", "RoleHistory") + DocumentAuthor = apps.get_model("doc", "DocumentAuthor") + DocHistoryAuthor = apps.get_model("doc", "DocHistoryAuthor") + + e = Email.objects.filter(address="unknown-email-Gigi-Karmous-Edwards").first() + if e: + # according to ftp://ietf.org/ietf/97dec/adsl-minutes-97dec.txt + new_e, _ = Email.objects.get_or_create( + address="GiGi.Karmous-Edwards@pulse.com", + primary=e.primary, + active=e.active, + person=e.person, + ) + Role.objects.filter(email=e).update(email=new_e) + RoleHistory.objects.filter(email=e).update(email=new_e) + e.delete() + + e = Email.objects.filter(address="unknown-email-Pat-Thaler").first() + if e: + # current chair email + new_e = Email.objects.get(address="pat.thaler@broadcom.com") + Role.objects.filter(email=e).update(email=new_e) + RoleHistory.objects.filter(email=e).update(email=new_e) + e.delete() + + e = Email.objects.filter(address="unknown-email-Greg->").first() + if e: + # current email + new_e = Email.objects.get(address="gregimirsky@gmail.com") + DocumentAuthor.objects.filter(email=e).update(email=new_e) + DocHistoryAuthor.objects.filter(email=e).update(email=new_e) + e.delete() + + DocumentAuthor.objects.filter(email__address__startswith="unknown-email-").exclude(email__address__contains="@").update(email=None) + DocHistoryAuthor.objects.filter(email__address__startswith="unknown-email-").exclude(email__address__contains="@").update(email=None) + Email.objects.exclude(address__contains="@").filter(address__startswith="unknown-email-").delete() + + assert not Email.objects.filter(address__startswith="unknown-email-") + +class Migration(migrations.Migration): + + dependencies = [ + ('doc', '0026_author_revamp_and_extra_attributes'), + ('person', '0014_auto_20160613_0751'), + ('group', '0009_auto_20150930_0758'), + ] + + operations = [ + migrations.RunPython(fix_invalid_emails, migrations.RunPython.noop), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 6604e263d..c5c9dc49d 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -18,12 +18,13 @@ import debug # pyflakes:ignore from ietf.group.models import Group from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName, - DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName ) + DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, FormalLanguageName ) from ietf.person.models import Email, Person from ietf.utils import log from ietf.utils.admin import admin_link from ietf.utils.rfcmarkup import markup from ietf.utils.validators import validate_no_control_chars +from ietf.utils.mail import formataddr logger = logging.getLogger('django') @@ -82,6 +83,8 @@ class DocumentInfo(models.Model): abstract = models.TextField(blank=True) rev = models.CharField(verbose_name="revision", max_length=16, blank=True) pages = models.IntegerField(blank=True, null=True) + words = models.IntegerField(blank=True, null=True) + formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document") order = models.IntegerField(default=1, blank=True) # This is probably obviated by SessionPresentaion.order intended_std_level = models.ForeignKey(IntendedStdLevelName, verbose_name="Intended standardization level", blank=True, null=True) std_level = models.ForeignKey(StdLevelName, verbose_name="Standardization level", blank=True, null=True) @@ -319,7 +322,7 @@ class DocumentInfo(models.Model): return self._cached_is_rfc def author_list(self): - return ", ".join(email.address for email in self.authors.all()) + return u", ".join(author.email_id for author in self.documentauthor_set.all() if author.email_id) # This, and several other ballot related functions here, assume that there is only one active ballot for a document at any point in time. # If that assumption is violated, they will only expose the most recently created ballot @@ -509,20 +512,33 @@ class RelatedDocument(models.Model): return False -class DocumentAuthor(models.Model): - document = models.ForeignKey('Document') - author = models.ForeignKey(Email, help_text="Email address used by author for submission") +class DocumentAuthorInfo(models.Model): + person = models.ForeignKey(Person) + # email should only be null for some historic documents + email = models.ForeignKey(Email, help_text="Email address used by author for submission", blank=True, null=True) + affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission") + country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission") order = models.IntegerField(default=1) - def __unicode__(self): - return u"%s %s (%s)" % (self.document.name, self.author.get_name(), self.order) + def formatted_email(self): + + if self.email: + return formataddr((self.person.plain_ascii(), self.email.address)) + else: + return "" class Meta: + abstract = True ordering = ["document", "order"] - + +class DocumentAuthor(DocumentAuthorInfo): + document = models.ForeignKey('Document') + + def __unicode__(self): + return u"%s %s (%s)" % (self.document.name, self.person, self.order) + class Document(DocumentInfo): name = models.CharField(max_length=255, primary_key=True) # immutable - authors = models.ManyToManyField(Email, through=DocumentAuthor, blank=True) def __unicode__(self): return self.name @@ -757,16 +773,13 @@ class RelatedDocHistory(models.Model): def __unicode__(self): return u"%s %s %s" % (self.source.doc.name, self.relationship.name.lower(), self.target.name) -class DocHistoryAuthor(models.Model): - document = models.ForeignKey('DocHistory') - author = models.ForeignKey(Email) - order = models.IntegerField() +class DocHistoryAuthor(DocumentAuthorInfo): + # use same naming convention as non-history version to make it a bit + # easier to write generic code + document = models.ForeignKey('DocHistory', related_name="documentauthor_set") def __unicode__(self): - return u"%s %s (%s)" % (self.document.doc.name, self.author.get_name(), self.order) - - class Meta: - ordering = ["document", "order"] + return u"%s %s (%s)" % (self.document.doc.name, self.person, self.order) class DocHistory(DocumentInfo): doc = models.ForeignKey(Document, related_name="history_set") @@ -775,7 +788,7 @@ class DocHistory(DocumentInfo): # canonical_name and replace the function on Document with a # property name = models.CharField(max_length=255) - authors = models.ManyToManyField(Email, through=DocHistoryAuthor, blank=True) + def __unicode__(self): return unicode(self.doc.name) diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index f91d03af8..ac6efd312 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -99,7 +99,6 @@ class DocumentResource(ModelResource): shepherd = ToOneField(EmailResource, 'shepherd', null=True) states = ToManyField(StateResource, 'states', null=True) tags = ToManyField(DocTagNameResource, 'tags', null=True) - authors = ToManyField(EmailResource, 'authors', null=True) rfc = CharField(attribute='rfc_number', null=True) class Meta: cache = SimpleCache() @@ -128,14 +127,14 @@ class DocumentResource(ModelResource): "shepherd": ALL_WITH_RELATIONS, "states": ALL_WITH_RELATIONS, "tags": ALL_WITH_RELATIONS, - "authors": ALL_WITH_RELATIONS, } api.doc.register(DocumentResource()) -from ietf.person.resources import EmailResource +from ietf.person.resources import PersonResource, EmailResource class DocumentAuthorResource(ModelResource): + person = ToOneField(PersonResource, 'person') + email = ToOneField(EmailResource, 'email', null=True) document = ToOneField(DocumentResource, 'document') - author = ToOneField(EmailResource, 'author') class Meta: cache = SimpleCache() queryset = DocumentAuthor.objects.all() @@ -143,9 +142,12 @@ class DocumentAuthorResource(ModelResource): #resource_name = 'documentauthor' filtering = { "id": ALL, + "affiliation": ALL, + "country": ALL, "order": ALL, + "person": ALL_WITH_RELATIONS, + "email": ALL_WITH_RELATIONS, "document": ALL_WITH_RELATIONS, - "author": ALL_WITH_RELATIONS, } api.doc.register(DocumentAuthorResource()) @@ -209,7 +211,6 @@ class DocHistoryResource(ModelResource): doc = ToOneField(DocumentResource, 'doc') states = ToManyField(StateResource, 'states', null=True) tags = ToManyField(DocTagNameResource, 'tags', null=True) - authors = ToManyField(EmailResource, 'authors', null=True) class Meta: cache = SimpleCache() queryset = DocHistory.objects.all() @@ -239,7 +240,6 @@ class DocHistoryResource(ModelResource): "doc": ALL_WITH_RELATIONS, "states": ALL_WITH_RELATIONS, "tags": ALL_WITH_RELATIONS, - "authors": ALL_WITH_RELATIONS, } api.doc.register(DocHistoryResource()) @@ -412,10 +412,11 @@ class InitialReviewDocEventResource(ModelResource): } api.doc.register(InitialReviewDocEventResource()) -from ietf.person.resources import EmailResource +from ietf.person.resources import PersonResource, EmailResource class DocHistoryAuthorResource(ModelResource): + person = ToOneField(PersonResource, 'person') + email = ToOneField(EmailResource, 'email', null=True) document = ToOneField(DocHistoryResource, 'document') - author = ToOneField(EmailResource, 'author') class Meta: cache = SimpleCache() queryset = DocHistoryAuthor.objects.all() @@ -423,9 +424,12 @@ class DocHistoryAuthorResource(ModelResource): #resource_name = 'dochistoryauthor' filtering = { "id": ALL, + "affiliation": ALL, + "country": ALL, "order": ALL, + "person": ALL_WITH_RELATIONS, + "email": ALL_WITH_RELATIONS, "document": ALL_WITH_RELATIONS, - "author": ALL_WITH_RELATIONS, } api.doc.register(DocHistoryAuthorResource()) diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 070133cde..51d31afaa 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -190,6 +190,11 @@ def rfcnospace(string): else: return string +@register.filter +def prettystdname(string): + from ietf.doc.utils import prettify_std_name + return prettify_std_name(unicode(string or "")) + @register.filter(name='rfcurl') def rfclink(string): """ diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index ac524afaa..58465c271 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -87,7 +87,7 @@ class SearchTests(TestCase): self.assertTrue(draft.title in unicontent(r)) # find by author - r = self.client.get(base_url + "?activedrafts=on&by=author&author=%s" % draft.authors.all()[0].person.name_parts()[1]) + r = self.client.get(base_url + "?activedrafts=on&by=author&author=%s" % draft.documentauthor_set.first().person.name_parts()[1]) self.assertEqual(r.status_code, 200) self.assertTrue(draft.title in unicontent(r)) @@ -1237,7 +1237,7 @@ class ChartTests(ResourceTestCaseMixin, TestCase): person = PersonFactory.create() DocumentFactory.create( states=[('draft','active')], - authors=[person.email(), ], + authors=[person, ], ) conf_url = urlreverse('ietf.doc.views_stats.chart_conf_person_drafts', kwargs=dict(id=person.id)) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index e1df40ed8..e91cb8aaa 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -377,7 +377,10 @@ class EditInfoTests(TestCase): DocumentAuthor.objects.create( document=draft, - author=Email.objects.get(address="aread@ietf.org"), + person=Person.objects.get(email__address="aread@ietf.org"), + email=Email.objects.get(address="aread@ietf.org"), + country="US", + affiliation="", order=1 ) @@ -1348,7 +1351,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.basea.documentauthor_set.create(author=Email.objects.create(address="basea_author@example.com"),order=1) + p = Person.objects.create(address="basea_author") + e = Email.objects.create(address="basea_author@example.com", person=p) + self.basea.documentauthor_set.create(person=p, email=e, order=1) self.baseb = Document.objects.create( name="draft-test-base-b", @@ -1359,7 +1364,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() - datetime.timedelta(days = 365 - settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.baseb.documentauthor_set.create(author=Email.objects.create(address="baseb_author@example.com"),order=1) + p = Person.objects.create(name="baseb_author") + e = Email.objects.create(address="baseb_author@example.com", person=p) + self.baseb.documentauthor_set.create(person=p, email=e, order=1) self.replacea = Document.objects.create( name="draft-test-replace-a", @@ -1370,7 +1377,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() + datetime.timedelta(days = settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.replacea.documentauthor_set.create(author=Email.objects.create(address="replacea_author@example.com"),order=1) + p = Person.objects.create(name="replacea_author") + e = Email.objects.create(address="replacea_author@example.com", person=p) + self.replacea.documentauthor_set.create(person=p, email=e, order=1) self.replaceboth = Document.objects.create( name="draft-test-replace-both", @@ -1381,7 +1390,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() + datetime.timedelta(days = settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.replaceboth.documentauthor_set.create(author=Email.objects.create(address="replaceboth_author@example.com"),order=1) + p = Person.objects.create(name="replaceboth_author") + e = Email.objects.create(address="replaceboth_author@example.com", person=p) + self.replaceboth.documentauthor_set.create(person=p, email=e, order=1) self.basea.set_state(State.objects.get(used=True, type="draft", slug="active")) self.baseb.set_state(State.objects.get(used=True, type="draft", slug="expired")) diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index 4b91c2427..07dc1b842 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -260,10 +260,7 @@ class ReviewTests(TestCase): # set up some reviewer-suitability factors reviewer_email = Email.objects.get(person__user__username="reviewer") - DocumentAuthor.objects.create( - author=reviewer_email, - document=doc, - ) + DocumentAuthor.objects.create(person=reviewer_email.person, email=reviewer_email, document=doc) doc.rev = "10" doc.save_with_history([DocEvent.objects.create(doc=doc, rev=doc.rev, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")]) @@ -572,7 +569,7 @@ class ReviewTests(TestCase): self.assertTrue(settings.MAILING_LIST_ARCHIVE_URL in review_req.review.external_url) # Check that the review has the reviewer as author - self.assertEqual(review_req.reviewer, review_req.review.authors.first()) + self.assertEqual(review_req.reviewer, review_req.review.documentauthor_set.first().email) # Check that we have a copy of the outgoing message msgid = outbox[0]["Message-ID"] diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 7d504d0c4..ab1b1bf0c 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -56,7 +56,6 @@ from ietf.group.models import Role from ietf.group.utils import can_manage_group_type, can_manage_materials from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, user_is_person, role_required from ietf.name.models import StreamName, BallotPositionName -from ietf.person.models import Email from ietf.utils.history import find_history_active_at from ietf.doc.forms import TelechatForm, NotifyForm from ietf.doc.mails import email_comment @@ -167,7 +166,7 @@ def document_main(request, name, rev=None): can_edit_replaces = has_role(request.user, ("Area Director", "Secretariat", "IRTF Chair", "WG Chair", "RG Chair", "WG Secretary", "RG Secretary")) - is_author = unicode(request.user) in set([email.address for email in doc.authors.all()]) + is_author = request.user.is_authenticated() and doc.documentauthor_set.filter(person__user=request.user).exists() can_view_possibly_replaces = can_edit_replaces or is_author rfc_number = name[3:] if name.startswith("") else None @@ -984,11 +983,11 @@ def document_json(request, name, rev=None): data["intended_std_level"] = extract_name(doc.intended_std_level) data["std_level"] = extract_name(doc.std_level) data["authors"] = [ - dict(name=e.person.name, - email=e.address, - affiliation=e.person.affiliation) - for e in Email.objects.filter(documentauthor__document=doc).select_related("person").order_by("documentauthor__order") - ] + dict(name=author.person.name, + email=author.email.address, + affiliation=author.affiliation) + for author in doc.documentauthor_set.all().select_related("person", "email").order_by("order") + ] data["shepherd"] = doc.shepherd.formatted_email() if doc.shepherd else None data["ad"] = doc.ad.role_email("ad").formatted_email() if doc.ad else None diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index 9e010fba0..1db51ec9c 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -567,7 +567,7 @@ def complete_review(request, name, request_id): else: author_email = request.user.person.email() frm = request.user.person.formatted_email() - author, created = DocumentAuthor.objects.get_or_create(document=review, author=author_email) + author, created = DocumentAuthor.objects.get_or_create(document=review, email=author_email, person=request.user.person) if need_to_email_review: # email the review diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index b46b7a373..ed81b5682 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -167,7 +167,7 @@ def retrieve_search_results(form, all_types=False): # radio choices by = query["by"] if by == "author": - docs = docs.filter(authors__person__alias__name__icontains=query["author"]) + docs = docs.filter(documentauthor__person__alias__name__icontains=query["author"]) elif by == "group": docs = docs.filter(group__acronym=query["group"]) elif by == "area": diff --git a/ietf/doc/views_stats.py b/ietf/doc/views_stats.py index 639a9bb93..54249774b 100644 --- a/ietf/doc/views_stats.py +++ b/ietf/doc/views_stats.py @@ -171,7 +171,7 @@ def chart_data_person_drafts(request, id): if not person: data = [] else: - data = model_to_timeline_data(DocEvent, doc__authors__person=person, type='new_revision') + data = model_to_timeline_data(DocEvent, doc__documentauthor__person=person, type='new_revision') return JsonResponse(data, safe=False) diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py index 2601f9888..22a2845ef 100644 --- a/ietf/idindex/index.py +++ b/ietf/idindex/index.py @@ -117,15 +117,15 @@ def all_id2_txt(): file_types = file_types_for_drafts() authors = {} - for a in DocumentAuthor.objects.filter(document__name__startswith="draft-").order_by("order").select_related("author", "author__person").iterator(): + for a in DocumentAuthor.objects.filter(document__name__startswith="draft-").order_by("order").select_related("email", "person").iterator(): if a.document_id not in authors: l = authors[a.document_id] = [] else: l = authors[a.document_id] - if "@" in a.author.address: - l.append(u'%s <%s>' % (a.author.person.plain_name().replace("@", ""), a.author.address.replace(",", ""))) + if a.email: + l.append(u'%s <%s>' % (a.person.plain_name().replace("@", ""), a.email.address.replace(",", ""))) else: - l.append(a.author.person.plain_name()) + l.append(a.person.plain_name()) shepherds = dict((e.pk, e.formatted_ascii_email().replace('"', '')) for e in Email.objects.filter(shepherd_document_set__type="draft").select_related("person").distinct()) @@ -236,12 +236,12 @@ def active_drafts_index_by_group(extra_values=()): d["initial_rev_time"] = time # add authors - for a in DocumentAuthor.objects.filter(document__states=active_state).order_by("order").select_related("author__person"): + for a in DocumentAuthor.objects.filter(document__states=active_state).order_by("order").select_related("person"): d = docs_dict.get(a.document_id) if d: if "authors" not in d: d["authors"] = [] - d["authors"].append(a.author.person.plain_ascii()) # This should probably change to .plain_name() when non-ascii names are permitted + d["authors"].append(a.person.plain_ascii()) # This should probably change to .plain_name() when non-ascii names are permitted # put docs into groups for d in docs_dict.itervalues(): diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py index e06739237..28408cfff 100644 --- a/ietf/idindex/tests.py +++ b/ietf/idindex/tests.py @@ -97,7 +97,7 @@ class IndexTests(TestCase): self.assertEqual(t[12], ".pdf,.txt") self.assertEqual(t[13], draft.title) author = draft.documentauthor_set.order_by("order").get() - self.assertEqual(t[14], u"%s <%s>" % (author.author.person.name, author.author.address)) + self.assertEqual(t[14], u"%s <%s>" % (author.person.name, author.email.address)) self.assertEqual(t[15], u"%s <%s>" % (draft.shepherd.person.name, draft.shepherd.address)) self.assertEqual(t[16], u"%s <%s>" % (draft.ad.plain_ascii(), draft.ad.email_address())) diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index f1709c4c3..2abf1ba72 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -64,15 +64,15 @@ def get_document_emails(ipr): messages = [] for rel in ipr.iprdocrel_set.all(): doc = rel.document.document - authors = doc.authors.all() - + if is_draft(doc): doc_info = 'Internet-Draft entitled "{}" ({})'.format(doc.title,doc.name) else: doc_info = 'RFC entitled "{}" (RFC{})'.format(doc.title,get_rfc_num(doc)) addrs = gather_address_lists('ipr_posted_on_doc',doc=doc).as_strings(compact=False) - author_names = ', '.join([a.person.name for a in authors]) + + author_names = ', '.join(a.person.name for a in doc.documentauthor_set.select_related("person")) context = dict( doc_info=doc_info, diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py index 24e721463..16952064f 100644 --- a/ietf/mailtrigger/models.py +++ b/ietf/mailtrigger/models.py @@ -181,7 +181,7 @@ class Recipient(models.Model): addrs = [] if 'submission' in kwargs: submission = kwargs['submission'] - addrs.extend(["%s <%s>" % (author["name"], author["email"]) for author in submission.authors_parsed() if author["email"]]) + addrs.extend(["%s <%s>" % (author["name"], author["email"]) for author in submission.authors if author.get("email")]) return addrs def gather_submission_group_chairs(self, **kwargs): @@ -203,10 +203,14 @@ class Recipient(models.Model): submission = kwargs['submission'] doc=submission.existing_document() if doc: - old_authors = [i.author.formatted_email() for i in doc.documentauthor_set.all() if not i.author.invalid_address()] - new_authors = [ formataddr((author["name"], author["email"])) for author in submission.authors_parsed() if author["email"]] - addrs.extend(old_authors) - if doc.group and set(old_authors)!=set(new_authors): + old_authors = [ author for author in doc.documentauthor_set.all() if author.email ] + + addrs.extend([ author.formatted_email() for author in old_authors]) + + old_author_email_set = set(author.email.address for author in old_authors) + new_author_email_set = set(author["email"] for author in submission.authors if author.get("email")) + + if doc.group and old_author_email_set != new_author_email_set: if doc.group.type_id in ['wg','rg','ag']: addrs.extend(Recipient.objects.get(slug='group_chairs').gather(**{'group':doc.group})) elif doc.group.type_id in ['area']: @@ -216,8 +220,8 @@ class Recipient(models.Model): if doc.stream_id and doc.stream_id not in ['ietf']: addrs.extend(Recipient.objects.get(slug='stream_managers').gather(**{'streams':[doc.stream_id]})) else: - addrs.extend([formataddr((author["name"], author["email"])) for author in submission.authors_parsed() if author["email"]]) - if submission.submitter_parsed()["email"]: + addrs.extend([formataddr((author["name"], author["email"])) for author in submission.authors if author.get("email")]) + if submission.submitter_parsed()["email"]: addrs.append(submission.submitter) return addrs diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index d20275259..dd782a29f 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -6,6 +6,8 @@ from django import forms from django.db.models import Q from django.forms import BaseInlineFormSet +import debug # pyflakes:ignore + from ietf.doc.models import Document, DocAlias, State, NewRevisionDocEvent from ietf.doc.utils import get_document_content from ietf.group.models import Group @@ -84,7 +86,7 @@ class InterimSessionInlineFormSet(BaseInlineFormSet): dates.sort() last_date = dates[0] for date in dates[1:]: - if last_date.day + 1 != date.day: + if date - last_date != datetime.timedelta(days=1): raise forms.ValidationError('For Multi-Day meetings, days must be consecutive') last_date = date diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index df7ccc8bf..667744b6e 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -280,7 +280,7 @@ class MeetingTests(TestCase): r = self.client.get(urlreverse("ietf.meeting.views.materials", kwargs=dict(num=meeting.number))) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - row = q('#content td div:contains("%s")' % str(session.group.acronym)).closest("tr") + row = q('#content #%s' % str(session.group.acronym)).closest("tr") self.assertTrue(row.find('a:contains("Agenda")')) self.assertTrue(row.find('a:contains("Minutes")')) self.assertTrue(row.find('a:contains("Slideshow")')) @@ -970,7 +970,7 @@ class InterimTests(TestCase): 'session_set-INITIAL_FORMS':0} r = self.client.post(urlreverse("ietf.meeting.views.interim_request"),data) - + self.assertRedirects(r,urlreverse('ietf.meeting.views.upcoming')) meeting = Meeting.objects.order_by('id').last() self.assertEqual(meeting.type_id,'interim') diff --git a/ietf/name/admin.py b/ietf/name/admin.py index 684e930ce..c4d2caa35 100644 --- a/ietf/name/admin.py +++ b/ietf/name/admin.py @@ -1,17 +1,21 @@ from django.contrib import admin from ietf.name.models import ( - BallotPositionName, ConstraintName, DBTemplateTypeName, DocRelationshipName, + BallotPositionName, ConstraintName, ContinentName, CountryName, + DBTemplateTypeName, DocRelationshipName, DocReminderTypeName, DocTagName, DocTypeName, DraftSubmissionStateName, - FeedbackTypeName, GroupMilestoneStateName, GroupStateName, GroupTypeName, + FeedbackTypeName, FormalLanguageName, GroupMilestoneStateName, GroupStateName, GroupTypeName, IntendedStdLevelName, IprDisclosureStateName, IprEventTypeName, IprLicenseTypeName, LiaisonStatementEventTypeName, LiaisonStatementPurposeName, LiaisonStatementState, LiaisonStatementTagName, MeetingTypeName, NomineePositionStateName, ReviewRequestStateName, ReviewResultName, ReviewTypeName, RoleName, RoomResourceName, SessionStatusName, StdLevelName, StreamName, TimeSlotTypeName, ) +from ietf.stats.models import CountryAlias + class NameAdmin(admin.ModelAdmin): list_display = ["slug", "name", "desc", "used"] + search_fields = ["slug", "name"] prepopulate_from = { "slug": ("name",) } class DocRelationshipNameAdmin(NameAdmin): @@ -26,12 +30,24 @@ class GroupTypeNameAdmin(NameAdmin): list_display = ["slug", "name", "verbose_name", "desc", "used"] admin.site.register(GroupTypeName, GroupTypeNameAdmin) +class CountryAliasInline(admin.TabularInline): + model = CountryAlias + extra = 1 + +class CountryNameAdmin(NameAdmin): + list_display = ["slug", "name", "continent", "in_eu"] + list_filter = ["continent", "in_eu"] + inlines = [CountryAliasInline] +admin.site.register(CountryName, CountryNameAdmin) + admin.site.register(BallotPositionName, NameAdmin) admin.site.register(ConstraintName, NameAdmin) +admin.site.register(ContinentName, NameAdmin) admin.site.register(DBTemplateTypeName, NameAdmin) admin.site.register(DocReminderTypeName, NameAdmin) admin.site.register(DocTagName, NameAdmin) admin.site.register(DraftSubmissionStateName, NameAdmin) +admin.site.register(FormalLanguageName, NameAdmin) admin.site.register(FeedbackTypeName, NameAdmin) admin.site.register(GroupMilestoneStateName, NameAdmin) admin.site.register(GroupStateName, NameAdmin) diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index d7e154042..4aa927b32 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -4016,6 +4016,3064 @@ "model": "name.constraintname", "pk": "conflict" }, + { + "fields": { + "desc": "", + "name": "Africa", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "africa" + }, + { + "fields": { + "desc": "", + "name": "Antarctica", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "antarctica" + }, + { + "fields": { + "desc": "", + "name": "Asia", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "asia" + }, + { + "fields": { + "desc": "", + "name": "Europe", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "europe" + }, + { + "fields": { + "desc": "", + "name": "North America", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "north-america" + }, + { + "fields": { + "desc": "", + "name": "Oceania", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "oceania" + }, + { + "fields": { + "desc": "", + "name": "South America", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "south-america" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Andorra", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AD" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "United Arab Emirates", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AE" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Afghanistan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AF" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Antigua and Barbuda", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AG" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Anguilla", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AI" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Albania", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AL" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Armenia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AM" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Angola", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AO" + }, + { + "fields": { + "continent": "antarctica", + "desc": "", + "in_eu": false, + "name": "Antarctica", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AQ" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Argentina", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AR" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "American Samoa", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Austria", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AT" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Australia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AU" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Aruba", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AW" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Åland Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AX" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Azerbaijan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AZ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Bosnia and Herzegovina", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BA" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Barbados", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BB" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Bangladesh", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BD" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Belgium", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BE" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Burkina Faso", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BF" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Bulgaria", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BG" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Bahrain", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BH" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Burundi", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BI" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Benin", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BJ" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Barthélemy", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BL" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Bermuda", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BM" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Brunei", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BN" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Bolivia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BO" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Bonaire, Sint Eustatius and Saba", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BQ" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Brazil", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BR" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Bahamas", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BS" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Bhutan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BT" + }, + { + "fields": { + "continent": "antarctica", + "desc": "", + "in_eu": false, + "name": "Bouvet Island", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BV" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Botswana", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BW" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Belarus", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BY" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Belize", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BZ" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Canada", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CA" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Cocos (Keeling) Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CC" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Congo (the Democratic Republic of the)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CD" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Central African Republic", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CF" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Congo", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CG" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Switzerland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CH" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Côte d'Ivoire", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CI" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Cook Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CK" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Chile", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CL" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Cameroon", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CM" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "China", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CN" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Colombia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CO" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Costa Rica", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CR" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Cuba", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CU" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Cabo Verde", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CV" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Curaçao", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CW" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Christmas Island", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CX" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": true, + "name": "Cyprus", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CY" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Czech Republic", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CZ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Germany", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DE" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Djibouti", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DJ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Denmark", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DK" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Dominica", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DM" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Dominican Republic", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DO" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Algeria", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DZ" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Ecuador", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "EC" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Estonia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "EE" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Egypt", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "EG" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Western Sahara", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "EH" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Eritrea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ER" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Spain", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ES" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Ethiopia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ET" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Finland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FI" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Fiji", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FJ" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Falkland Islands [Malvinas]", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FK" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Micronesia (Federated States of)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FM" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Faroe Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FO" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "France", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FR" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Gabon", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GA" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "United Kingdom", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GB" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Grenada", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GD" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Georgia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GE" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "French Guiana", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GF" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Guernsey", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GG" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Ghana", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GH" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Gibraltar", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GI" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Greenland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GL" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Gambia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GM" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Guinea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GN" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Guadeloupe", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GP" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Equatorial Guinea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GQ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Greece", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GR" + }, + { + "fields": { + "continent": "antarctica", + "desc": "", + "in_eu": false, + "name": "South Georgia and the South Sandwich Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GS" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Guatemala", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GT" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Guam", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GU" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Guinea-Bissau", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GW" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Guyana", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GY" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Hong Kong", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HK" + }, + { + "fields": { + "continent": "antarctica", + "desc": "", + "in_eu": false, + "name": "Heard Island and McDonald Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HM" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Honduras", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HN" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Croatia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HR" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Haiti", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HT" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Hungary", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HU" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Indonesia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ID" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Ireland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IE" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Israel", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IL" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Isle of Man", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IM" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "India", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IN" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "British Indian Ocean Territory", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IO" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Iraq", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IQ" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Iran", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IR" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Iceland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Italy", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IT" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Jersey", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "JE" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Jamaica", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "JM" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Jordan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "JO" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Japan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "JP" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Kenya", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KE" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Kyrgyzstan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KG" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Cambodia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KH" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Kiribati", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KI" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Comoros", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KM" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Kitts and Nevis", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KN" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "North Korea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KP" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "South Korea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KR" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Kuwait", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KW" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Cayman Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KY" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Kazakhstan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KZ" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Laos", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LA" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Lebanon", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LB" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Lucia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LC" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Liechtenstein", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LI" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Sri Lanka", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LK" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Liberia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LR" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Lesotho", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Lithuania", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LT" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Luxembourg", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LU" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Latvia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LV" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Libya", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LY" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Morocco", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MA" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Monaco", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MC" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Moldova", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MD" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Montenegro", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ME" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Martin (French part)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MF" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Madagascar", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MG" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Marshall Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MH" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Macedonia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MK" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Mali", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ML" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Myanmar", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MM" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Mongolia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MN" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Macao", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MO" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Northern Mariana Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MP" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Martinique", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MQ" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Mauritania", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MR" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Montserrat", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Malta", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MT" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Mauritius", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MU" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Maldives", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MV" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Malawi", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MW" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Mexico", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MX" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Malaysia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MY" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Mozambique", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MZ" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Namibia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NA" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "New Caledonia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NC" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Niger", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NE" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Norfolk Island", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NF" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Nigeria", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NG" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Nicaragua", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NI" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Netherlands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NL" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Norway", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NO" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Nepal", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NP" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Nauru", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NR" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Niue", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NU" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "New Zealand", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NZ" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Oman", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "OM" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Panama", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PA" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Peru", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PE" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "French Polynesia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PF" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Papua New Guinea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PG" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Philippines", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PH" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Pakistan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PK" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Poland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PL" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Pierre and Miquelon", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PM" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Pitcairn", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PN" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Puerto Rico", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PR" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Palestine, State of", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Portugal", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PT" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Palau", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PW" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Paraguay", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PY" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Qatar", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "QA" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Réunion", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "RE" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Romania", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "RO" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Serbia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "RS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Russia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "RU" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Rwanda", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "RW" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Saudi Arabia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SA" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Solomon Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SB" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Seychelles", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SC" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Sudan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SD" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Sweden", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SE" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Singapore", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SG" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Saint Helena, Ascension and Tristan da Cunha", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SH" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Slovenia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SI" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Svalbard and Jan Mayen", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SJ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Slovakia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SK" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Sierra Leone", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SL" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "San Marino", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SM" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Senegal", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SN" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Somalia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SO" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Suriname", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SR" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "South Sudan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SS" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Sao Tome and Principe", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ST" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "El Salvador", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SV" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Sint Maarten (Dutch part)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SX" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Syria", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SY" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Swaziland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SZ" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Turks and Caicos Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TC" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Chad", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TD" + }, + { + "fields": { + "continent": "antarctica", + "desc": "", + "in_eu": false, + "name": "French Southern Territories", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TF" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Togo", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TG" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Thailand", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TH" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Tajikistan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TJ" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Tokelau", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TK" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Timor-Leste", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TL" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Turkmenistan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TM" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Tunisia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TN" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Tonga", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TO" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Turkey", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TR" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Trinidad and Tobago", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TT" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Tuvalu", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TV" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Taiwan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TW" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Tanzania", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TZ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Ukraine", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "UA" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Uganda", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "UG" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "United States Minor Outlying Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "UM" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "United States of America", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "US" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Uruguay", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "UY" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Uzbekistan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "UZ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Holy See", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VA" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Vincent and the Grenadines", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VC" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Venezuela", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VE" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Virgin Islands (British)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VG" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Virgin Islands (U.S.)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VI" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Vietnam", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VN" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Vanuatu", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VU" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Wallis and Futuna", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "WF" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Samoa", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "WS" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Yemen", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "YE" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Mayotte", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "YT" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "South Africa", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ZA" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Zambia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ZM" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Zimbabwe", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ZW" + }, { "fields": { "desc": "", @@ -4844,6 +7902,66 @@ "model": "name.feedbacktypename", "pk": "questio" }, + { + "fields": { + "desc": "Augmented Backus-Naur Form", + "name": "ABNF", + "order": 1, + "used": true + }, + "model": "name.formallanguagename", + "pk": "abnf" + }, + { + "fields": { + "desc": "Abstract Syntax Notation One", + "name": "ASN.1", + "order": 2, + "used": true + }, + "model": "name.formallanguagename", + "pk": "asn1" + }, + { + "fields": { + "desc": "Concise Binary Object Representation", + "name": "CBOR", + "order": 3, + "used": true + }, + "model": "name.formallanguagename", + "pk": "cbor" + }, + { + "fields": { + "desc": "Code in the C Programming Language", + "name": "C Code", + "order": 4, + "used": true + }, + "model": "name.formallanguagename", + "pk": "ccode" + }, + { + "fields": { + "desc": "Javascript Object Notation", + "name": "JSON", + "order": 5, + "used": true + }, + "model": "name.formallanguagename", + "pk": "json" + }, + { + "fields": { + "desc": "Extensible Markup Language", + "name": "XML", + "order": 6, + "used": true + }, + "model": "name.formallanguagename", + "pk": "xml" + }, { "fields": { "desc": "", diff --git a/ietf/name/migrations/0019_formallanguagename.py b/ietf/name/migrations/0019_formallanguagename.py new file mode 100644 index 000000000..884e77784 --- /dev/null +++ b/ietf/name/migrations/0019_formallanguagename.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0018_iab_programs'), + ] + + operations = [ + migrations.CreateModel( + name='FormalLanguageName', + fields=[ + ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('desc', models.TextField(blank=True)), + ('used', models.BooleanField(default=True)), + ('order', models.IntegerField(default=0)), + ], + options={ + 'ordering': ['order', 'name'], + 'abstract': False, + }, + ), + ] diff --git a/ietf/name/migrations/0020_add_formlang_names.py b/ietf/name/migrations/0020_add_formlang_names.py new file mode 100644 index 000000000..2bf378b2e --- /dev/null +++ b/ietf/name/migrations/0020_add_formlang_names.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def insert_initial_formal_language_names(apps, schema_editor): + FormalLanguageName = apps.get_model("name", "FormalLanguageName") + FormalLanguageName.objects.get_or_create(slug="abnf", name="ABNF", desc="Augmented Backus-Naur Form", order=1) + FormalLanguageName.objects.get_or_create(slug="asn1", name="ASN.1", desc="Abstract Syntax Notation One", order=2) + FormalLanguageName.objects.get_or_create(slug="cbor", name="CBOR", desc="Concise Binary Object Representation", order=3) + FormalLanguageName.objects.get_or_create(slug="ccode", name="C Code", desc="Code in the C Programming Language", order=4) + FormalLanguageName.objects.get_or_create(slug="json", name="JSON", desc="Javascript Object Notation", order=5) + FormalLanguageName.objects.get_or_create(slug="xml", name="XML", desc="Extensible Markup Language", order=6) + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0019_formallanguagename'), + ] + + operations = [ + migrations.RunPython(insert_initial_formal_language_names, migrations.RunPython.noop) + ] diff --git a/ietf/name/migrations/0021_continentname_countryname.py b/ietf/name/migrations/0021_continentname_countryname.py new file mode 100644 index 000000000..a5a254350 --- /dev/null +++ b/ietf/name/migrations/0021_continentname_countryname.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0020_add_formlang_names'), + ] + + operations = [ + migrations.CreateModel( + name='ContinentName', + fields=[ + ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('desc', models.TextField(blank=True)), + ('used', models.BooleanField(default=True)), + ('order', models.IntegerField(default=0)), + ], + options={ + 'ordering': ['order', 'name'], + 'abstract': False, + }, + ), + migrations.CreateModel( + name='CountryName', + fields=[ + ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('desc', models.TextField(blank=True)), + ('used', models.BooleanField(default=True)), + ('order', models.IntegerField(default=0)), + ('in_eu', models.BooleanField(default=False, verbose_name='In EU')), + ('continent', models.ForeignKey(to='name.ContinentName')), + ], + options={ + 'ordering': ['order', 'name'], + 'abstract': False, + }, + ), + ] diff --git a/ietf/name/migrations/0022_add_country_continent_names.py b/ietf/name/migrations/0022_add_country_continent_names.py new file mode 100644 index 000000000..f05c73dea --- /dev/null +++ b/ietf/name/migrations/0022_add_country_continent_names.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def insert_initial_country_continent_names(apps, schema_editor): + ContinentName = apps.get_model("name", "ContinentName") + africa, _ = ContinentName.objects.get_or_create(slug="africa", name="Africa") + antarctica, _ = ContinentName.objects.get_or_create(slug="antarctica", name="Antarctica") + asia, _ = ContinentName.objects.get_or_create(slug="asia", name="Asia") + europe, _ = ContinentName.objects.get_or_create(slug="europe", name="Europe") + north_america, _ = ContinentName.objects.get_or_create(slug="north-america", name="North America") + oceania, _ = ContinentName.objects.get_or_create(slug="oceania", name="Oceania") + south_america, _ = ContinentName.objects.get_or_create(slug="south-america", name="South America") + + CountryName = apps.get_model("name", "CountryName") + CountryName.objects.get_or_create(slug="AD", name=u"Andorra", continent=europe) + CountryName.objects.get_or_create(slug="AE", name=u"United Arab Emirates", continent=asia) + CountryName.objects.get_or_create(slug="AF", name=u"Afghanistan", continent=asia) + CountryName.objects.get_or_create(slug="AG", name=u"Antigua and Barbuda", continent=north_america) + CountryName.objects.get_or_create(slug="AI", name=u"Anguilla", continent=north_america) + CountryName.objects.get_or_create(slug="AL", name=u"Albania", continent=europe) + CountryName.objects.get_or_create(slug="AM", name=u"Armenia", continent=asia) + CountryName.objects.get_or_create(slug="AO", name=u"Angola", continent=africa) + CountryName.objects.get_or_create(slug="AQ", name=u"Antarctica", continent=antarctica) + CountryName.objects.get_or_create(slug="AR", name=u"Argentina", continent=south_america) + CountryName.objects.get_or_create(slug="AS", name=u"American Samoa", continent=oceania) + CountryName.objects.get_or_create(slug="AT", name=u"Austria", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="AU", name=u"Australia", continent=oceania) + CountryName.objects.get_or_create(slug="AW", name=u"Aruba", continent=north_america) + CountryName.objects.get_or_create(slug="AX", name=u"Åland Islands", continent=europe) + CountryName.objects.get_or_create(slug="AZ", name=u"Azerbaijan", continent=asia) + CountryName.objects.get_or_create(slug="BA", name=u"Bosnia and Herzegovina", continent=europe) + CountryName.objects.get_or_create(slug="BB", name=u"Barbados", continent=north_america) + CountryName.objects.get_or_create(slug="BD", name=u"Bangladesh", continent=asia) + CountryName.objects.get_or_create(slug="BE", name=u"Belgium", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="BF", name=u"Burkina Faso", continent=africa) + CountryName.objects.get_or_create(slug="BG", name=u"Bulgaria", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="BH", name=u"Bahrain", continent=asia) + CountryName.objects.get_or_create(slug="BI", name=u"Burundi", continent=africa) + CountryName.objects.get_or_create(slug="BJ", name=u"Benin", continent=africa) + CountryName.objects.get_or_create(slug="BL", name=u"Saint Barthélemy", continent=north_america) + CountryName.objects.get_or_create(slug="BM", name=u"Bermuda", continent=north_america) + CountryName.objects.get_or_create(slug="BN", name=u"Brunei", continent=asia) + CountryName.objects.get_or_create(slug="BO", name=u"Bolivia", continent=south_america) + CountryName.objects.get_or_create(slug="BQ", name=u"Bonaire, Sint Eustatius and Saba", continent=north_america) + CountryName.objects.get_or_create(slug="BR", name=u"Brazil", continent=south_america) + CountryName.objects.get_or_create(slug="BS", name=u"Bahamas", continent=north_america) + CountryName.objects.get_or_create(slug="BT", name=u"Bhutan", continent=asia) + CountryName.objects.get_or_create(slug="BV", name=u"Bouvet Island", continent=antarctica) + CountryName.objects.get_or_create(slug="BW", name=u"Botswana", continent=africa) + CountryName.objects.get_or_create(slug="BY", name=u"Belarus", continent=europe) + CountryName.objects.get_or_create(slug="BZ", name=u"Belize", continent=north_america) + CountryName.objects.get_or_create(slug="CA", name=u"Canada", continent=north_america) + CountryName.objects.get_or_create(slug="CC", name=u"Cocos (Keeling) Islands", continent=asia) + CountryName.objects.get_or_create(slug="CD", name=u"Congo (the Democratic Republic of the)", continent=africa) + CountryName.objects.get_or_create(slug="CF", name=u"Central African Republic", continent=africa) + CountryName.objects.get_or_create(slug="CG", name=u"Congo", continent=africa) + CountryName.objects.get_or_create(slug="CH", name=u"Switzerland", continent=europe) + CountryName.objects.get_or_create(slug="CI", name=u"Côte d'Ivoire", continent=africa) + CountryName.objects.get_or_create(slug="CK", name=u"Cook Islands", continent=oceania) + CountryName.objects.get_or_create(slug="CL", name=u"Chile", continent=south_america) + CountryName.objects.get_or_create(slug="CM", name=u"Cameroon", continent=africa) + CountryName.objects.get_or_create(slug="CN", name=u"China", continent=asia) + CountryName.objects.get_or_create(slug="CO", name=u"Colombia", continent=south_america) + CountryName.objects.get_or_create(slug="CR", name=u"Costa Rica", continent=north_america) + CountryName.objects.get_or_create(slug="CU", name=u"Cuba", continent=north_america) + CountryName.objects.get_or_create(slug="CV", name=u"Cabo Verde", continent=africa) + CountryName.objects.get_or_create(slug="CW", name=u"Curaçao", continent=north_america) + CountryName.objects.get_or_create(slug="CX", name=u"Christmas Island", continent=asia) + CountryName.objects.get_or_create(slug="CY", name=u"Cyprus", continent=asia, in_eu=True) + CountryName.objects.get_or_create(slug="CZ", name=u"Czech Republic", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="DE", name=u"Germany", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="DJ", name=u"Djibouti", continent=africa) + CountryName.objects.get_or_create(slug="DK", name=u"Denmark", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="DM", name=u"Dominica", continent=north_america) + CountryName.objects.get_or_create(slug="DO", name=u"Dominican Republic", continent=north_america) + CountryName.objects.get_or_create(slug="DZ", name=u"Algeria", continent=africa) + CountryName.objects.get_or_create(slug="EC", name=u"Ecuador", continent=south_america) + CountryName.objects.get_or_create(slug="EE", name=u"Estonia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="EG", name=u"Egypt", continent=africa) + CountryName.objects.get_or_create(slug="EH", name=u"Western Sahara", continent=africa) + CountryName.objects.get_or_create(slug="ER", name=u"Eritrea", continent=africa) + CountryName.objects.get_or_create(slug="ES", name=u"Spain", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="ET", name=u"Ethiopia", continent=africa) + CountryName.objects.get_or_create(slug="FI", name=u"Finland", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="FJ", name=u"Fiji", continent=oceania) + CountryName.objects.get_or_create(slug="FK", name=u"Falkland Islands [Malvinas]", continent=south_america) + CountryName.objects.get_or_create(slug="FM", name=u"Micronesia (Federated States of)", continent=oceania) + CountryName.objects.get_or_create(slug="FO", name=u"Faroe Islands", continent=europe) + CountryName.objects.get_or_create(slug="FR", name=u"France", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="GA", name=u"Gabon", continent=africa) + CountryName.objects.get_or_create(slug="GB", name=u"United Kingdom", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="GD", name=u"Grenada", continent=north_america) + CountryName.objects.get_or_create(slug="GE", name=u"Georgia", continent=asia) + CountryName.objects.get_or_create(slug="GF", name=u"French Guiana", continent=south_america) + CountryName.objects.get_or_create(slug="GG", name=u"Guernsey", continent=europe) + CountryName.objects.get_or_create(slug="GH", name=u"Ghana", continent=africa) + CountryName.objects.get_or_create(slug="GI", name=u"Gibraltar", continent=europe) + CountryName.objects.get_or_create(slug="GL", name=u"Greenland", continent=north_america) + CountryName.objects.get_or_create(slug="GM", name=u"Gambia", continent=africa) + CountryName.objects.get_or_create(slug="GN", name=u"Guinea", continent=africa) + CountryName.objects.get_or_create(slug="GP", name=u"Guadeloupe", continent=north_america) + CountryName.objects.get_or_create(slug="GQ", name=u"Equatorial Guinea", continent=africa) + CountryName.objects.get_or_create(slug="GR", name=u"Greece", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="GS", name=u"South Georgia and the South Sandwich Islands", continent=antarctica) + CountryName.objects.get_or_create(slug="GT", name=u"Guatemala", continent=north_america) + CountryName.objects.get_or_create(slug="GU", name=u"Guam", continent=oceania) + CountryName.objects.get_or_create(slug="GW", name=u"Guinea-Bissau", continent=africa) + CountryName.objects.get_or_create(slug="GY", name=u"Guyana", continent=south_america) + CountryName.objects.get_or_create(slug="HK", name=u"Hong Kong", continent=asia) + CountryName.objects.get_or_create(slug="HM", name=u"Heard Island and McDonald Islands", continent=antarctica) + CountryName.objects.get_or_create(slug="HN", name=u"Honduras", continent=north_america) + CountryName.objects.get_or_create(slug="HR", name=u"Croatia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="HT", name=u"Haiti", continent=north_america) + CountryName.objects.get_or_create(slug="HU", name=u"Hungary", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="ID", name=u"Indonesia", continent=asia) + CountryName.objects.get_or_create(slug="IE", name=u"Ireland", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="IL", name=u"Israel", continent=asia) + CountryName.objects.get_or_create(slug="IM", name=u"Isle of Man", continent=europe) + CountryName.objects.get_or_create(slug="IN", name=u"India", continent=asia) + CountryName.objects.get_or_create(slug="IO", name=u"British Indian Ocean Territory", continent=asia) + CountryName.objects.get_or_create(slug="IQ", name=u"Iraq", continent=asia) + CountryName.objects.get_or_create(slug="IR", name=u"Iran", continent=asia) + CountryName.objects.get_or_create(slug="IS", name=u"Iceland", continent=europe) + CountryName.objects.get_or_create(slug="IT", name=u"Italy", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="JE", name=u"Jersey", continent=europe) + CountryName.objects.get_or_create(slug="JM", name=u"Jamaica", continent=north_america) + CountryName.objects.get_or_create(slug="JO", name=u"Jordan", continent=asia) + CountryName.objects.get_or_create(slug="JP", name=u"Japan", continent=asia) + CountryName.objects.get_or_create(slug="KE", name=u"Kenya", continent=africa) + CountryName.objects.get_or_create(slug="KG", name=u"Kyrgyzstan", continent=asia) + CountryName.objects.get_or_create(slug="KH", name=u"Cambodia", continent=asia) + CountryName.objects.get_or_create(slug="KI", name=u"Kiribati", continent=oceania) + CountryName.objects.get_or_create(slug="KM", name=u"Comoros", continent=africa) + CountryName.objects.get_or_create(slug="KN", name=u"Saint Kitts and Nevis", continent=north_america) + CountryName.objects.get_or_create(slug="KP", name=u"North Korea", continent=asia) + CountryName.objects.get_or_create(slug="KR", name=u"South Korea", continent=asia) + CountryName.objects.get_or_create(slug="KW", name=u"Kuwait", continent=asia) + CountryName.objects.get_or_create(slug="KY", name=u"Cayman Islands", continent=north_america) + CountryName.objects.get_or_create(slug="KZ", name=u"Kazakhstan", continent=asia) + CountryName.objects.get_or_create(slug="LA", name=u"Laos", continent=asia) + CountryName.objects.get_or_create(slug="LB", name=u"Lebanon", continent=asia) + CountryName.objects.get_or_create(slug="LC", name=u"Saint Lucia", continent=north_america) + CountryName.objects.get_or_create(slug="LI", name=u"Liechtenstein", continent=europe) + CountryName.objects.get_or_create(slug="LK", name=u"Sri Lanka", continent=asia) + CountryName.objects.get_or_create(slug="LR", name=u"Liberia", continent=africa) + CountryName.objects.get_or_create(slug="LS", name=u"Lesotho", continent=africa) + CountryName.objects.get_or_create(slug="LT", name=u"Lithuania", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="LU", name=u"Luxembourg", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="LV", name=u"Latvia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="LY", name=u"Libya", continent=africa) + CountryName.objects.get_or_create(slug="MA", name=u"Morocco", continent=africa) + CountryName.objects.get_or_create(slug="MC", name=u"Monaco", continent=europe) + CountryName.objects.get_or_create(slug="MD", name=u"Moldova", continent=europe) + CountryName.objects.get_or_create(slug="ME", name=u"Montenegro", continent=europe) + CountryName.objects.get_or_create(slug="MF", name=u"Saint Martin (French part)", continent=north_america) + CountryName.objects.get_or_create(slug="MG", name=u"Madagascar", continent=africa) + CountryName.objects.get_or_create(slug="MH", name=u"Marshall Islands", continent=oceania) + CountryName.objects.get_or_create(slug="MK", name=u"Macedonia", continent=europe) + CountryName.objects.get_or_create(slug="ML", name=u"Mali", continent=africa) + CountryName.objects.get_or_create(slug="MM", name=u"Myanmar", continent=asia) + CountryName.objects.get_or_create(slug="MN", name=u"Mongolia", continent=asia) + CountryName.objects.get_or_create(slug="MO", name=u"Macao", continent=asia) + CountryName.objects.get_or_create(slug="MP", name=u"Northern Mariana Islands", continent=oceania) + CountryName.objects.get_or_create(slug="MQ", name=u"Martinique", continent=north_america) + CountryName.objects.get_or_create(slug="MR", name=u"Mauritania", continent=africa) + CountryName.objects.get_or_create(slug="MS", name=u"Montserrat", continent=north_america) + CountryName.objects.get_or_create(slug="MT", name=u"Malta", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="MU", name=u"Mauritius", continent=africa) + CountryName.objects.get_or_create(slug="MV", name=u"Maldives", continent=asia) + CountryName.objects.get_or_create(slug="MW", name=u"Malawi", continent=africa) + CountryName.objects.get_or_create(slug="MX", name=u"Mexico", continent=north_america) + CountryName.objects.get_or_create(slug="MY", name=u"Malaysia", continent=asia) + CountryName.objects.get_or_create(slug="MZ", name=u"Mozambique", continent=africa) + CountryName.objects.get_or_create(slug="NA", name=u"Namibia", continent=africa) + CountryName.objects.get_or_create(slug="NC", name=u"New Caledonia", continent=oceania) + CountryName.objects.get_or_create(slug="NE", name=u"Niger", continent=africa) + CountryName.objects.get_or_create(slug="NF", name=u"Norfolk Island", continent=oceania) + CountryName.objects.get_or_create(slug="NG", name=u"Nigeria", continent=africa) + CountryName.objects.get_or_create(slug="NI", name=u"Nicaragua", continent=north_america) + CountryName.objects.get_or_create(slug="NL", name=u"Netherlands", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="NO", name=u"Norway", continent=europe) + CountryName.objects.get_or_create(slug="NP", name=u"Nepal", continent=asia) + CountryName.objects.get_or_create(slug="NR", name=u"Nauru", continent=oceania) + CountryName.objects.get_or_create(slug="NU", name=u"Niue", continent=oceania) + CountryName.objects.get_or_create(slug="NZ", name=u"New Zealand", continent=oceania) + CountryName.objects.get_or_create(slug="OM", name=u"Oman", continent=asia) + CountryName.objects.get_or_create(slug="PA", name=u"Panama", continent=north_america) + CountryName.objects.get_or_create(slug="PE", name=u"Peru", continent=south_america) + CountryName.objects.get_or_create(slug="PF", name=u"French Polynesia", continent=oceania) + CountryName.objects.get_or_create(slug="PG", name=u"Papua New Guinea", continent=oceania) + CountryName.objects.get_or_create(slug="PH", name=u"Philippines", continent=asia) + CountryName.objects.get_or_create(slug="PK", name=u"Pakistan", continent=asia) + CountryName.objects.get_or_create(slug="PL", name=u"Poland", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="PM", name=u"Saint Pierre and Miquelon", continent=north_america) + CountryName.objects.get_or_create(slug="PN", name=u"Pitcairn", continent=oceania) + CountryName.objects.get_or_create(slug="PR", name=u"Puerto Rico", continent=north_america) + CountryName.objects.get_or_create(slug="PS", name=u"Palestine, State of", continent=asia) + CountryName.objects.get_or_create(slug="PT", name=u"Portugal", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="PW", name=u"Palau", continent=oceania) + CountryName.objects.get_or_create(slug="PY", name=u"Paraguay", continent=south_america) + CountryName.objects.get_or_create(slug="QA", name=u"Qatar", continent=asia) + CountryName.objects.get_or_create(slug="RE", name=u"Réunion", continent=africa) + CountryName.objects.get_or_create(slug="RO", name=u"Romania", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="RS", name=u"Serbia", continent=europe) + CountryName.objects.get_or_create(slug="RU", name=u"Russia", continent=europe) + CountryName.objects.get_or_create(slug="RW", name=u"Rwanda", continent=africa) + CountryName.objects.get_or_create(slug="SA", name=u"Saudi Arabia", continent=asia) + CountryName.objects.get_or_create(slug="SB", name=u"Solomon Islands", continent=oceania) + CountryName.objects.get_or_create(slug="SC", name=u"Seychelles", continent=africa) + CountryName.objects.get_or_create(slug="SD", name=u"Sudan", continent=africa) + CountryName.objects.get_or_create(slug="SE", name=u"Sweden", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="SG", name=u"Singapore", continent=asia) + CountryName.objects.get_or_create(slug="SH", name=u"Saint Helena, Ascension and Tristan da Cunha", continent=africa) + CountryName.objects.get_or_create(slug="SI", name=u"Slovenia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="SJ", name=u"Svalbard and Jan Mayen", continent=europe) + CountryName.objects.get_or_create(slug="SK", name=u"Slovakia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="SL", name=u"Sierra Leone", continent=africa) + CountryName.objects.get_or_create(slug="SM", name=u"San Marino", continent=europe) + CountryName.objects.get_or_create(slug="SN", name=u"Senegal", continent=africa) + CountryName.objects.get_or_create(slug="SO", name=u"Somalia", continent=africa) + CountryName.objects.get_or_create(slug="SR", name=u"Suriname", continent=south_america) + CountryName.objects.get_or_create(slug="SS", name=u"South Sudan", continent=africa) + CountryName.objects.get_or_create(slug="ST", name=u"Sao Tome and Principe", continent=africa) + CountryName.objects.get_or_create(slug="SV", name=u"El Salvador", continent=north_america) + CountryName.objects.get_or_create(slug="SX", name=u"Sint Maarten (Dutch part)", continent=north_america) + CountryName.objects.get_or_create(slug="SY", name=u"Syria", continent=asia) + CountryName.objects.get_or_create(slug="SZ", name=u"Swaziland", continent=africa) + CountryName.objects.get_or_create(slug="TC", name=u"Turks and Caicos Islands", continent=north_america) + CountryName.objects.get_or_create(slug="TD", name=u"Chad", continent=africa) + CountryName.objects.get_or_create(slug="TF", name=u"French Southern Territories", continent=antarctica) + CountryName.objects.get_or_create(slug="TG", name=u"Togo", continent=africa) + CountryName.objects.get_or_create(slug="TH", name=u"Thailand", continent=asia) + CountryName.objects.get_or_create(slug="TJ", name=u"Tajikistan", continent=asia) + CountryName.objects.get_or_create(slug="TK", name=u"Tokelau", continent=oceania) + CountryName.objects.get_or_create(slug="TL", name=u"Timor-Leste", continent=asia) + CountryName.objects.get_or_create(slug="TM", name=u"Turkmenistan", continent=asia) + CountryName.objects.get_or_create(slug="TN", name=u"Tunisia", continent=africa) + CountryName.objects.get_or_create(slug="TO", name=u"Tonga", continent=oceania) + CountryName.objects.get_or_create(slug="TR", name=u"Turkey", continent=europe) + CountryName.objects.get_or_create(slug="TT", name=u"Trinidad and Tobago", continent=north_america) + CountryName.objects.get_or_create(slug="TV", name=u"Tuvalu", continent=oceania) + CountryName.objects.get_or_create(slug="TW", name=u"Taiwan", continent=asia) + CountryName.objects.get_or_create(slug="TZ", name=u"Tanzania", continent=africa) + CountryName.objects.get_or_create(slug="UA", name=u"Ukraine", continent=europe) + CountryName.objects.get_or_create(slug="UG", name=u"Uganda", continent=africa) + CountryName.objects.get_or_create(slug="UM", name=u"United States Minor Outlying Islands", continent=oceania) + CountryName.objects.get_or_create(slug="US", name=u"United States of America", continent=north_america) + CountryName.objects.get_or_create(slug="UY", name=u"Uruguay", continent=south_america) + CountryName.objects.get_or_create(slug="UZ", name=u"Uzbekistan", continent=asia) + CountryName.objects.get_or_create(slug="VA", name=u"Holy See", continent=europe) + CountryName.objects.get_or_create(slug="VC", name=u"Saint Vincent and the Grenadines", continent=north_america) + CountryName.objects.get_or_create(slug="VE", name=u"Venezuela", continent=south_america) + CountryName.objects.get_or_create(slug="VG", name=u"Virgin Islands (British)", continent=north_america) + CountryName.objects.get_or_create(slug="VI", name=u"Virgin Islands (U.S.)", continent=north_america) + CountryName.objects.get_or_create(slug="VN", name=u"Vietnam", continent=asia) + CountryName.objects.get_or_create(slug="VU", name=u"Vanuatu", continent=oceania) + CountryName.objects.get_or_create(slug="WF", name=u"Wallis and Futuna", continent=oceania) + CountryName.objects.get_or_create(slug="WS", name=u"Samoa", continent=oceania) + CountryName.objects.get_or_create(slug="YE", name=u"Yemen", continent=asia) + CountryName.objects.get_or_create(slug="YT", name=u"Mayotte", continent=africa) + CountryName.objects.get_or_create(slug="ZA", name=u"South Africa", continent=africa) + CountryName.objects.get_or_create(slug="ZM", name=u"Zambia", continent=africa) + CountryName.objects.get_or_create(slug="ZW", name=u"Zimbabwe", continent=africa) + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0021_continentname_countryname'), + ] + + operations = [ + migrations.RunPython(insert_initial_country_continent_names, migrations.RunPython.noop) + ] diff --git a/ietf/name/models.py b/ietf/name/models.py index c1c69b4c6..15161f331 100644 --- a/ietf/name/models.py +++ b/ietf/name/models.py @@ -46,6 +46,8 @@ class StdLevelName(NameModel): class IntendedStdLevelName(NameModel): """Proposed Standard, (Draft Standard), Internet Standard, Experimental, Informational, Best Current Practice, Historic, ...""" +class FormalLanguageName(NameModel): + """ABNF, ASN.1, C code, CBOR, JSON, XML, ...""" class DocReminderTypeName(NameModel): "Stream state" class BallotPositionName(NameModel): @@ -97,4 +99,10 @@ class ReviewResultName(NameModel): """Almost ready, Has issues, Has nits, Not Ready, On the right track, Ready, Ready with issues, Ready with nits, Serious Issues""" +class ContinentName(NameModel): + "Africa, Antarctica, Asia, ..." +class CountryName(NameModel): + "Afghanistan, Aaland Islands, Albania, ..." + continent = models.ForeignKey(ContinentName) + in_eu = models.BooleanField(verbose_name="In EU", default=False) diff --git a/ietf/name/resources.py b/ietf/name/resources.py index 15bc6d378..287963395 100644 --- a/ietf/name/resources.py +++ b/ietf/name/resources.py @@ -14,7 +14,8 @@ from ietf.name.models import (TimeSlotTypeName, GroupStateName, DocTagName, Inte ConstraintName, MeetingTypeName, DocRelationshipName, RoomResourceName, IprLicenseTypeName, LiaisonStatementTagName, FeedbackTypeName, LiaisonStatementState, StreamName, BallotPositionName, DBTemplateTypeName, NomineePositionStateName, - ReviewRequestStateName, ReviewTypeName, ReviewResultName) + ReviewRequestStateName, ReviewTypeName, ReviewResultName, + FormalLanguageName, ContinentName, CountryName) class TimeSlotTypeNameResource(ModelResource): @@ -456,3 +457,55 @@ class ReviewResultNameResource(ModelResource): } api.name.register(ReviewResultNameResource()) + + +class FormalLanguageNameResource(ModelResource): + class Meta: + queryset = FormalLanguageName.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'formallanguagename' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(FormalLanguageNameResource()) + + + +class ContinentNameResource(ModelResource): + class Meta: + queryset = ContinentName.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'continentname' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(ContinentNameResource()) + +class CountryNameResource(ModelResource): + continent = ToOneField(ContinentNameResource, 'continent') + class Meta: + queryset = CountryName.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'countryname' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + "in_eu": ALL, + "continent": ALL_WITH_RELATIONS, + } +api.name.register(CountryNameResource()) + diff --git a/ietf/person/admin.py b/ietf/person/admin.py index 8c5ce62c0..e51427afa 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -32,4 +32,3 @@ class PersonAdmin(admin.ModelAdmin): inlines = [ EmailInline, AliasInline, ] # actions = None admin.site.register(Person, PersonAdmin) - diff --git a/ietf/person/models.py b/ietf/person/models.py index 6539e7365..c2c0d8e2e 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -16,7 +16,7 @@ from django.utils.text import slugify import debug # pyflakes:ignore -from ietf.person.name import name_parts, initials +from ietf.person.name import name_parts, initials, plain_name from ietf.utils.mail import send_mail_preformatted from ietf.utils.storage import NoLocationMigrationFileSystemStorage from ietf.utils.mail import formataddr @@ -51,8 +51,7 @@ class PersonInfo(models.Model): return (first and first[0]+"." or "")+(middle or "")+" "+last+(suffix and " "+suffix or "") def plain_name(self): if not hasattr(self, '_cached_plain_name'): - prefix, first, middle, last, suffix = name_parts(self.name) - self._cached_plain_name = u" ".join([first, last]) + self._cached_plain_name = plain_name(self.name) return self._cached_plain_name def ascii_name(self): if not hasattr(self, '_cached_ascii_name'): @@ -137,18 +136,18 @@ class PersonInfo(models.Model): def has_drafts(self): from ietf.doc.models import Document - return Document.objects.filter(authors__person=self, type='draft').exists() + return Document.objects.filter(documentauthor__person=self, type='draft').exists() def rfcs(self): from ietf.doc.models import Document - rfcs = list(Document.objects.filter(authors__person=self, type='draft', states__slug='rfc')) + rfcs = list(Document.objects.filter(documentauthor__person=self, type='draft', states__slug='rfc')) rfcs.sort(key=lambda d: d.canonical_name() ) return rfcs def active_drafts(self): from ietf.doc.models import Document - return Document.objects.filter(authors__person=self, type='draft', states__slug='active').order_by('-time') + return Document.objects.filter(documentauthor__person=self, type='draft', states__slug='active').order_by('-time') def expired_drafts(self): from ietf.doc.models import Document - return Document.objects.filter(authors__person=self, type='draft', states__slug__in=['repl', 'expired', 'auth-rm', 'ietf-rm']).order_by('-time') + return Document.objects.filter(documentauthor__person=self, type='draft', states__slug__in=['repl', 'expired', 'auth-rm', 'ietf-rm']).order_by('-time') class Meta: abstract = True @@ -260,15 +259,11 @@ class Email(models.Model): else: return self.address - def invalid_address(self): - # we have some legacy authors with unknown email addresses - return self.address.startswith("unknown-email") and "@" not in self.address - def email_address(self): """Get valid, current email address; in practise, for active, non-invalid addresses it is just the address field. In other cases, we default to person's email address.""" - if self.invalid_address() or not self.active: + if not self.active: if self.person: return self.person.email_address() return diff --git a/ietf/person/name.py b/ietf/person/name.py index a6401a2dc..14fba0a56 100644 --- a/ietf/person/name.py +++ b/ietf/person/name.py @@ -54,6 +54,10 @@ def initials(name): initials = u" ".join([ n[0]+'.' for n in given.split() ]) return initials +def plain_name(name): + prefix, first, middle, last, suffix = name_parts(name) + return u" ".join([first, last]) + if __name__ == "__main__": import sys name = u" ".join(sys.argv[1:]) diff --git a/ietf/person/resources.py b/ietf/person/resources.py index e0c8608fc..11ef747bf 100644 --- a/ietf/person/resources.py +++ b/ietf/person/resources.py @@ -6,7 +6,7 @@ from tastypie.cache import SimpleCache from ietf import api -from ietf.person.models import Person, Email, Alias, PersonHistory +from ietf.person.models import (Person, Email, Alias, PersonHistory) from ietf.utils.resources import UserResource @@ -82,4 +82,3 @@ class PersonHistoryResource(ModelResource): "user": ALL_WITH_RELATIONS, } api.person.register(PersonHistoryResource()) - diff --git a/ietf/review/utils.py b/ietf/review/utils.py index d346589c0..65da48042 100644 --- a/ietf/review/utils.py +++ b/ietf/review/utils.py @@ -754,7 +754,7 @@ def make_assignment_choices(email_queryset, review_req): connections[r.person_id] = "is group {}".format(r.name) if doc.shepherd: connections[doc.shepherd.person_id] = "is shepherd of document" - for author in DocumentAuthor.objects.filter(document=doc, author__person__in=possible_person_ids).values_list("author__person", flat=True): + for author in DocumentAuthor.objects.filter(document=doc, person__in=possible_person_ids).values_list("person", flat=True): connections[author] = "is author of document" # unavailable periods diff --git a/ietf/secr/drafts/email.py b/ietf/secr/drafts/email.py index 07b706888..49583d990 100644 --- a/ietf/secr/drafts/email.py +++ b/ietf/secr/drafts/email.py @@ -48,12 +48,12 @@ def get_authors(draft): Takes a draft object and returns a list of authors suitable for a tombstone document """ authors = [] - for a in draft.authors.all(): + for a in draft.documentauthor_set.all(): initial = '' prefix, first, middle, last, suffix = a.person.name_parts() if first: initial = first + '. ' - entry = '%s%s <%s>' % (initial,last,a.address) + entry = '%s%s <%s>' % (initial,last,a.email.address) authors.append(entry) return authors @@ -64,10 +64,10 @@ def get_abbr_authors(draft): """ initial = '' result = '' - authors = DocumentAuthor.objects.filter(document=draft) + authors = DocumentAuthor.objects.filter(document=draft).order_by("order") if authors: - prefix, first, middle, last, suffix = authors[0].author.person.name_parts() + prefix, first, middle, last, suffix = authors[0].person.name_parts() if first: initial = first[0] + '. ' result = '%s%s' % (initial,last) @@ -140,9 +140,9 @@ def get_fullcc_list(draft): """ emails = {} # get authors - for author in draft.authors.all(): - if author.address not in emails: - emails[author.address] = '"%s"' % (author.person.name) + for author in draft.documentauthor_set.all(): + if author.email and author.email.address not in emails: + emails[author.email.address] = '"%s"' % (author.person.name) if draft.group.acronym != 'none': # add chairs diff --git a/ietf/secr/drafts/forms.py b/ietf/secr/drafts/forms.py index 801355748..f05cebb6a 100644 --- a/ietf/secr/drafts/forms.py +++ b/ietf/secr/drafts/forms.py @@ -104,6 +104,8 @@ class AuthorForm(forms.Form): ''' person = forms.CharField(max_length=50,widget=forms.TextInput(attrs={'class':'name-autocomplete'}),help_text="To see a list of people type the first name, or last name, or both.") email = forms.CharField(widget=forms.Select(),help_text="Select an email.") + affiliation = forms.CharField(max_length=100, required=False, help_text="Affiliation") + country = forms.CharField(max_length=255, required=False, help_text="Country") # check for id within parenthesis to ensure name was selected from the list def clean_person(self): diff --git a/ietf/secr/drafts/views.py b/ietf/secr/drafts/views.py index 281227cb8..3fa8ced44 100644 --- a/ietf/secr/drafts/views.py +++ b/ietf/secr/drafts/views.py @@ -540,7 +540,7 @@ def approvals(request): @role_required('Secretariat') def author_delete(request, id, oid): ''' - This view deletes the specified author(email) from the draft + This view deletes the specified author from the draft ''' DocumentAuthor.objects.get(id=oid).delete() messages.success(request, 'The author was deleted successfully') @@ -574,13 +574,17 @@ def authors(request, id): return redirect('ietf.secr.drafts.views.view', id=id) if form.is_valid(): - author = form.cleaned_data['email'] + person = form.cleaned_data['person'] + email = form.cleaned_data['email'] + affiliation = form.cleaned_data.get('affiliation') or "" + country = form.cleaned_data.get('country') or "" + authors = draft.documentauthor_set.all() if authors: order = authors.aggregate(Max('order')).values()[0] + 1 else: order = 1 - DocumentAuthor.objects.create(document=draft,author=author,order=order) + DocumentAuthor.objects.create(document=draft, person=person, email=email, affiliation=affiliation, country=country, order=order) messages.success(request, 'Author added successfully!') return redirect('ietf.secr.drafts.views.authors', id=id) diff --git a/ietf/secr/templates/drafts/authors.html b/ietf/secr/templates/drafts/authors.html index c5a4cb9ce..c47cb3983 100644 --- a/ietf/secr/templates/drafts/authors.html +++ b/ietf/secr/templates/drafts/authors.html @@ -24,6 +24,8 @@ Name Email + Affiliation + Country Order Action @@ -31,8 +33,10 @@ {% for author in draft.documentauthor_set.all %} - {{ author.author.person }} - {{ author.author }} + {{ author.person }} + {{ author.email }} + {{ author.affiliation }} + {{ author.country.name }} {{ author.order }} Delete @@ -49,6 +53,10 @@ {{ form.person.errors }}{{ form.person }}{% if form.person.help_text %}
{{ form.person.help_text }}{% endif %} {{ form.email.errors }}{{ form.email }}{% if form.email.help_text %}
{{ form.email.help_text }}{% endif %} + + + {{ form.affiliation.errors }}{{ form.affiliation }}{% if form.affiliation.help_text %}
{{ form.affiliation.help_text }}{% endif %} + {{ form.country.errors }}{{ form.country }}{% if form.country.help_text %}
{{ form.country.help_text }}{% endif %} diff --git a/ietf/secr/templates/drafts/makerfc.html b/ietf/secr/templates/drafts/makerfc.html index 962f76093..f5945d558 100644 --- a/ietf/secr/templates/drafts/makerfc.html +++ b/ietf/secr/templates/drafts/makerfc.html @@ -52,7 +52,7 @@

Author(s)

- {% for author in draft.authors.all %} + {% for author in draft.documentauthor_set.all %} {% endfor %}
{{ author.person.name }}
diff --git a/ietf/secr/templates/drafts/view.html b/ietf/secr/templates/drafts/view.html index 627832271..2f90838eb 100644 --- a/ietf/secr/templates/drafts/view.html +++ b/ietf/secr/templates/drafts/view.html @@ -73,7 +73,7 @@

Author(s)

{% for author in draft.documentauthor_set.all %} - + {% endfor %}
{{ author.author.person.name }}
{{ author.person.name }}
diff --git a/ietf/settings.py b/ietf/settings.py index 726d666b0..a7ace3ac5 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -397,6 +397,7 @@ INSTALLED_APPS = ( 'ietf.redirects', 'ietf.release', 'ietf.review', + 'ietf.stats', 'ietf.submit', 'ietf.sync', 'ietf.utils', @@ -488,6 +489,7 @@ TEST_CODE_COVERAGE_EXCLUDE = [ "ietf/utils/test_runner.py", "name/generate_fixtures.py", "review/import_from_review_tool.py", + "stats/backfill_data.py", ] # These are filename globs. They are used by test_parse_templates() and @@ -556,6 +558,8 @@ INTERNET_DRAFT_ARCHIVE_DIR = '/a/www/www6s/draft-archive' INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/a/www/www6s/archive/id' MEETING_RECORDINGS_DIR = '/a/www/audio' +DOCUMENT_FORMAT_BLACKLIST = ["tar", "dtd", "p7s"] + # Mailing list info URL for lists hosted on the IETF servers MAILING_LIST_INFO_URL = "https://www.ietf.org/mailman/listinfo/%(list_addr)s" MAILING_LIST_ARCHIVE_URL = "https://mailarchive.ietf.org" @@ -618,6 +622,8 @@ AUDIO_IMPORT_EMAIL = ['agenda@ietf.org','ietf@meetecho.com'] IANA_EVAL_EMAIL = "drafts-eval@icann.org" SESSION_REQUEST_FROM_EMAIL = 'IETF Meeting Session Request Tool ' +SECRETARIAT_TICKET_EMAIL = "ietf-action@ietf.org" + # Put real password in settings_local.py IANA_SYNC_PASSWORD = "secret" IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" diff --git a/ietf/static/ietf/css/ietf.css b/ietf/static/ietf/css/ietf.css index 0cda09bad..8d014938f 100644 --- a/ietf/static/ietf/css/ietf.css +++ b/ietf/static/ietf/css/ietf.css @@ -580,6 +580,15 @@ table.simple-table td:last-child { width: 7em; } +.document-stats .popover .element { + padding-left: 1em; + text-indent: -1em; +} + +.document-stats #chart { + height: 25em; +} + .stats-time-graph { height: 15em; } diff --git a/ietf/static/ietf/js/stats.js b/ietf/static/ietf/js/stats.js new file mode 100644 index 000000000..931e9ebf8 --- /dev/null +++ b/ietf/static/ietf/js/stats.js @@ -0,0 +1,60 @@ +$(document).ready(function () { + if (window.chartConf) { + window.chartConf.credits = { + enabled: false + }; + window.chartConf.exporting = { + fallbackToExportServer: false + }; + + if (!window.chartConf.legend) + window.chartConf.legend = { + enabled: false + }; + + var chart = Highcharts.chart('chart', window.chartConf); + } + + if (window.pieChartConf) { + window.pieChartConf.credits = { + enabled: false + }; + var pieChart = Highcharts.chart('pie-chart', window.pieChartConf); + } + /* + $(".popover-details").each(function () { + var stdNameRegExp = new RegExp("^(rfc|bcp|fyi|std)[0-9]+$", 'i'); + var draftRegExp = new RegExp("^draft-", 'i'); + + var html = []; + $.each(($(this).data("elements") || "").split("|"), function (i, element) { + if (!$.trim(element)) + return; + + if (draftRegExp.test(element) || stdNameRegExp.test(element)) { + var displayName = element; + + if (stdNameRegExp.test(element)) + displayName = element.slice(0, 3).toUpperCase() + " " + element.slice(3); + + html.push('
' + displayName + '
'); + } + else { + html.push('
' + element + '
'); + } + }); + + if ($(this).data("sliced")) + html.push('
…
'); + + $(this).popover({ + trigger: "focus", + template: '', + content: html.join(""), + placement: "top", + html: true + }).on("click", function (e) { + e.preventDefault(); + }); + });*/ +}); diff --git a/ietf/stats/admin.py b/ietf/stats/admin.py new file mode 100644 index 000000000..57f489746 --- /dev/null +++ b/ietf/stats/admin.py @@ -0,0 +1,22 @@ +from django.contrib import admin + +from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias + + +class AffiliationAliasAdmin(admin.ModelAdmin): + list_filter = ["name"] + list_display = ["alias", "name"] + search_fields = ["alias", "name"] +admin.site.register(AffiliationAlias, AffiliationAliasAdmin) + +class AffiliationIgnoredEndingAdmin(admin.ModelAdmin): + list_display = ["ending"] + search_fields = ["ending"] +admin.site.register(AffiliationIgnoredEnding, AffiliationIgnoredEndingAdmin) + +class CountryAliasAdmin(admin.ModelAdmin): + list_filter = ["country"] + list_display = ["alias", "country"] + search_fields = ["alias", "country__name"] +admin.site.register(CountryAlias, CountryAliasAdmin) + diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py new file mode 100644 index 000000000..44ce6d9a3 --- /dev/null +++ b/ietf/stats/backfill_data.py @@ -0,0 +1,133 @@ +import sys, os, argparse + +basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +sys.path = [ basedir ] + sys.path +os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" + +virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py") +if os.path.exists(virtualenv_activation): + execfile(virtualenv_activation, dict(__file__=virtualenv_activation)) + +import django +django.setup() + +from django.conf import settings + +from ietf.doc.models import Document +from ietf.name.models import FormalLanguageName +from ietf.utils.draft import Draft + +parser = argparse.ArgumentParser() +parser.add_argument("--document", help="specific document name") +parser.add_argument("--words", action="store_true", help="fill in word count") +parser.add_argument("--formlang", action="store_true", help="fill in formal languages") +parser.add_argument("--authors", action="store_true", help="fill in author info") +args = parser.parse_args() + +formal_language_dict = { l.pk: l for l in FormalLanguageName.objects.all() } + +docs_qs = Document.objects.filter(type="draft") + +if args.document: + docs_qs = docs_qs.filter(docalias__name=args.document) + +for doc in docs_qs.prefetch_related("docalias_set", "formal_languages", "documentauthor_set", "documentauthor_set__person", "documentauthor_set__person__alias_set"): + canonical_name = doc.name + for n in doc.docalias_set.all(): + if n.name.startswith("rfc"): + canonical_name = n.name + + if canonical_name.startswith("rfc"): + path = os.path.join(settings.RFC_PATH, canonical_name + ".txt") + else: + path = os.path.join(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR, canonical_name + "-" + doc.rev + ".txt") + + if not os.path.exists(path): + print "skipping", doc.name, "no txt file found at", path + continue + + with open(path, 'r') as f: + d = Draft(f.read(), path) + + updated = False + + updates = {} + + if args.words: + words = d.get_wordcount() + if words != doc.words: + updates["words"] = words + + if args.formlang: + langs = d.get_formal_languages() + + new_formal_languages = set(formal_language_dict[l] for l in langs) + old_formal_languages = set(doc.formal_languages.all()) + + if new_formal_languages != old_formal_languages: + for l in new_formal_languages - old_formal_languages: + doc.formal_languages.add(l) + updated = True + for l in old_formal_languages - new_formal_languages: + doc.formal_languages.remove(l) + updated = True + + if args.authors: + old_authors = doc.documentauthor_set.all() + old_authors_by_name = {} + old_authors_by_email = {} + for author in old_authors: + for alias in author.person.alias_set.all(): + old_authors_by_name[alias.name] = author + old_authors_by_name[author.person.plain_name()] = author + + if author.email_id: + old_authors_by_email[author.email_id] = author + + # the draft parser sometimes has a problem when + # affiliation isn't in the second line and it then thinks + # it's an extra author - skip those extra authors + seen = set() + for full, _, _, _, _, email, country, company in d.get_author_list(): + if email in seen: + continue + seen.add(email) + + old_author = None + if email: + old_author = old_authors_by_email.get(email) + if not old_author: + old_author = old_authors_by_name.get(full) + + if not old_author: + print "UNKNOWN AUTHOR", doc.name, full, email, country, company + continue + + if old_author.affiliation != company: + print "new affiliation", canonical_name, "[", full, "]", old_author.affiliation, "->", company + old_author.affiliation = company + old_author.save(update_fields=["affiliation"]) + updated = True + + if country is None: + country = "" + + try: + country = country.decode("utf-8") + except UnicodeDecodeError: + country = country.decode("latin-1") + + if old_author.country != country: + print "new country", canonical_name ,"[", full, email, "]", old_author.country.encode("utf-8"), "->", country.encode("utf-8") + old_author.country = country + old_author.save(update_fields=["country"]) + updated = True + + + if updates: + Document.objects.filter(pk=doc.pk).update(**updates) + updated = True + + if updated: + print "updated", canonical_name + diff --git a/ietf/stats/migrations/0001_initial.py b/ietf/stats/migrations/0001_initial.py new file mode 100644 index 000000000..121b45b41 --- /dev/null +++ b/ietf/stats/migrations/0001_initial.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0022_add_country_continent_names'), + ] + + operations = [ + migrations.CreateModel( + name='AffiliationAlias', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('alias', models.CharField(help_text=b"Note that aliases will be matched case-insensitive and both before and after some clean-up.", max_length=255, unique=True)), + ('name', models.CharField(max_length=255)), + ], + options={'verbose_name_plural': 'affiliation aliases'}, + ), + migrations.CreateModel( + name='AffiliationIgnoredEnding', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('ending', models.CharField(help_text=b"Regexp with ending, e.g. 'Inc\\.?' - remember to escape .!", max_length=255)), + ], + ), + migrations.CreateModel( + name='CountryAlias', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('alias', models.CharField(help_text=b"Note that lower-case aliases are matched case-insensitive while aliases with at least one uppercase letter is matched case-sensitive. So 'United States' is best entered as 'united states' so it both matches 'United States' and 'United states' and 'UNITED STATES', whereas 'US' is best entered as 'US' so it doesn't accidentally match an ordinary word like 'us'.", max_length=255)), + ('country', models.ForeignKey(to='name.CountryName', max_length=255)), + ], + options={'verbose_name_plural': 'country aliases'}, + ), + ] diff --git a/ietf/stats/migrations/0002_add_initial_aliases.py b/ietf/stats/migrations/0002_add_initial_aliases.py new file mode 100644 index 000000000..d317267d8 --- /dev/null +++ b/ietf/stats/migrations/0002_add_initial_aliases.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def add_affiliation_info(apps, schema_editor): + AffiliationAlias = apps.get_model("stats", "AffiliationAlias") + + AffiliationAlias.objects.get_or_create(alias="cisco", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco system", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco systems (india) private limited", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco systems india pvt", name="Cisco Systems") + + AffiliationIgnoredEnding = apps.get_model("stats", "AffiliationIgnoredEnding") + AffiliationIgnoredEnding.objects.get_or_create(ending="LLC\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="Ltd\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="Inc\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="GmbH\.?") + + CountryAlias = apps.get_model("stats", "CountryAlias") + for iso_country_code in ['AD', 'AE', 'AF', 'AG', 'AI', 'AL', 'AM', 'AO', 'AQ', 'AR', 'AS', 'AT', 'AU', 'AW', + 'AX', 'AZ', 'BA', 'BB', 'BD', 'BE', 'BF', 'BG', 'BH', 'BI', 'BJ', 'BL', 'BM', 'BN', + 'BO', 'BQ', 'BR', 'BS', 'BT', 'BV', 'BW', 'BY', 'BZ', 'CA', 'CC', 'CD', 'CF', 'CG', + 'CH', 'CI', 'CK', 'CL', 'CM', 'CN', 'CO', 'CR', 'CU', 'CV', 'CW', 'CX', 'CY', 'CZ', + 'DE', 'DJ', 'DK', 'DM', 'DO', 'DZ', 'EC', 'EE', 'EG', 'EH', 'ER', 'ES', 'ET', 'FI', + 'FJ', 'FK', 'FM', 'FO', 'FR', 'GA', 'GB', 'GD', 'GE', 'GF', 'GG', 'GH', 'GI', 'GL', + 'GM', 'GN', 'GP', 'GQ', 'GR', 'GS', 'GT', 'GU', 'GW', 'GY', 'HK', 'HM', 'HN', 'HR', + 'HT', 'HU', 'ID', 'IE', 'IL', 'IM', 'IN', 'IO', 'IQ', 'IR', 'IS', 'IT', 'JE', 'JM', + 'JO', 'JP', 'KE', 'KG', 'KH', 'KI', 'KM', 'KN', 'KP', 'KR', 'KW', 'KY', 'KZ', 'LA', + 'LB', 'LC', 'LI', 'LK', 'LR', 'LS', 'LT', 'LU', 'LV', 'LY', 'MA', 'MC', 'MD', 'ME', + 'MF', 'MG', 'MH', 'MK', 'ML', 'MM', 'MN', 'MO', 'MP', 'MQ', 'MR', 'MS', 'MT', 'MU', + 'MV', 'MW', 'MX', 'MY', 'MZ', 'NA', 'NC', 'NE', 'NF', 'NG', 'NI', 'NL', 'NO', 'NP', + 'NR', 'NU', 'NZ', 'OM', 'PA', 'PE', 'PF', 'PG', 'PH', 'PK', 'PL', 'PM', 'PN', 'PR', + 'PS', 'PT', 'PW', 'PY', 'QA', 'RE', 'RO', 'RS', 'RU', 'RW', 'SA', 'SB', 'SC', 'SD', + 'SE', 'SG', 'SH', 'SI', 'SJ', 'SK', 'SL', 'SM', 'SN', 'SO', 'SR', 'SS', 'ST', 'SV', + 'SX', 'SY', 'SZ', 'TC', 'TD', 'TF', 'TG', 'TH', 'TJ', 'TK', 'TL', 'TM', 'TN', 'TO', + 'TR', 'TT', 'TV', 'TW', 'TZ', 'UA', 'UG', 'UM', 'US', 'UY', 'UZ', 'VA', 'VC', 'VE', + 'VG', 'VI', 'VN', 'VU', 'WF', 'WS', 'YE', 'YT', 'ZA', 'ZM', 'ZW']: + CountryAlias.objects.get_or_create(alias=iso_country_code, country_id=iso_country_code) + + CountryAlias.objects.get_or_create(alias="russian federation", country_id="RU") + CountryAlias.objects.get_or_create(alias="p. r. china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r. china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r.china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r. of china", country_id="CN") + CountryAlias.objects.get_or_create(alias="PRC", country_id="CN") + CountryAlias.objects.get_or_create(alias="P.R.C", country_id="CN") + CountryAlias.objects.get_or_create(alias="P.R.C.", country_id="CN") + CountryAlias.objects.get_or_create(alias="beijing", country_id="CN") + CountryAlias.objects.get_or_create(alias="shenzhen", country_id="CN") + CountryAlias.objects.get_or_create(alias="R.O.C.", country_id="TW") + CountryAlias.objects.get_or_create(alias="usa", country_id="US") + CountryAlias.objects.get_or_create(alias="UAS", country_id="US") + CountryAlias.objects.get_or_create(alias="USA.", country_id="US") + CountryAlias.objects.get_or_create(alias="u.s.a.", country_id="US") + CountryAlias.objects.get_or_create(alias="u. s. a.", country_id="US") + CountryAlias.objects.get_or_create(alias="u.s.a", country_id="US") + CountryAlias.objects.get_or_create(alias="u.s.", country_id="US") + CountryAlias.objects.get_or_create(alias="U.S", country_id="GB") + CountryAlias.objects.get_or_create(alias="US of A", country_id="US") + CountryAlias.objects.get_or_create(alias="united sates", country_id="US") + CountryAlias.objects.get_or_create(alias="united state", country_id="US") + CountryAlias.objects.get_or_create(alias="united states", country_id="US") + CountryAlias.objects.get_or_create(alias="unites states", country_id="US") + CountryAlias.objects.get_or_create(alias="texas", country_id="US") + CountryAlias.objects.get_or_create(alias="UK", country_id="GB") + CountryAlias.objects.get_or_create(alias="united kingcom", country_id="GB") + CountryAlias.objects.get_or_create(alias="great britain", country_id="GB") + CountryAlias.objects.get_or_create(alias="england", country_id="GB") + CountryAlias.objects.get_or_create(alias="U.K.", country_id="GB") + CountryAlias.objects.get_or_create(alias="U.K", country_id="GB") + CountryAlias.objects.get_or_create(alias="Uk", country_id="GB") + CountryAlias.objects.get_or_create(alias="scotland", country_id="GB") + CountryAlias.objects.get_or_create(alias="republic of korea", country_id="KR") + CountryAlias.objects.get_or_create(alias="korea", country_id="KR") + CountryAlias.objects.get_or_create(alias="korea rep", country_id="KR") + CountryAlias.objects.get_or_create(alias="korea (the republic of)", country_id="KR") + CountryAlias.objects.get_or_create(alias="the netherlands", country_id="NL") + CountryAlias.objects.get_or_create(alias="netherland", country_id="NL") + CountryAlias.objects.get_or_create(alias="danmark", country_id="DK") + CountryAlias.objects.get_or_create(alias="sweeden", country_id="SE") + CountryAlias.objects.get_or_create(alias="swede", country_id="SE") + CountryAlias.objects.get_or_create(alias="belgique", country_id="BE") + CountryAlias.objects.get_or_create(alias="madrid", country_id="ES") + CountryAlias.objects.get_or_create(alias="espana", country_id="ES") + CountryAlias.objects.get_or_create(alias="hellas", country_id="GR") + CountryAlias.objects.get_or_create(alias="gemany", country_id="DE") + CountryAlias.objects.get_or_create(alias="deutschland", country_id="DE") + CountryAlias.objects.get_or_create(alias="italia", country_id="IT") + CountryAlias.objects.get_or_create(alias="isreal", country_id="IL") + CountryAlias.objects.get_or_create(alias="tel aviv", country_id="IL") + CountryAlias.objects.get_or_create(alias="UAE", country_id="AE") + CountryAlias.objects.get_or_create(alias="grand-duchy of luxembourg", country_id="LU") + CountryAlias.objects.get_or_create(alias="brasil", country_id="BR") + + +class Migration(migrations.Migration): + + dependencies = [ + ('stats', '0001_initial'), + ] + + operations = [ + migrations.RunPython(add_affiliation_info, migrations.RunPython.noop) + ] diff --git a/ietf/stats/migrations/0003_registration_registrationstats.py b/ietf/stats/migrations/0003_registration_registrationstats.py new file mode 100644 index 000000000..46bc6a84e --- /dev/null +++ b/ietf/stats/migrations/0003_registration_registrationstats.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.6 on 2017-03-30 14:38 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('person', '0015_clean_primary'), + ('meeting', '0047_import_shared_audio_files'), + ('stats', '0002_add_initial_aliases'), + ] + + operations = [ + migrations.CreateModel( + name='MeetingRegistration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('first_name', models.CharField(max_length=255)), + ('last_name', models.CharField(max_length=255)), + ('affiliation', models.CharField(blank=True, max_length=255)), + ('country_code', models.CharField(max_length=2)), + ('meeting', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='meeting.Meeting')), + ('person', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='person.Person')), + ], + ), + ] diff --git a/ietf/stats/migrations/__init__.py b/ietf/stats/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ietf/stats/models.py b/ietf/stats/models.py new file mode 100644 index 000000000..051af882d --- /dev/null +++ b/ietf/stats/models.py @@ -0,0 +1,55 @@ +from django.db import models +from ietf.meeting.models import Meeting +from ietf.name.models import CountryName +from ietf.person.models import Person + + +class AffiliationAlias(models.Model): + """Records that alias should be treated as name for statistical + purposes.""" + + alias = models.CharField(max_length=255, help_text="Note that aliases will be matched case-insensitive and both before and after some clean-up.", unique=True) + name = models.CharField(max_length=255) + + def __unicode__(self): + return u"{} -> {}".format(self.alias, self.name) + + def save(self, *args, **kwargs): + self.alias = self.alias.lower() + super(AffiliationAlias, self).save(*args, **kwargs) + + class Meta: + verbose_name_plural = "affiliation aliases" + +class AffiliationIgnoredEnding(models.Model): + """Records that ending should be stripped from the affiliation for statistical purposes.""" + + ending = models.CharField(max_length=255, help_text="Regexp with ending, e.g. 'Inc\\.?' - remember to escape .!") + + def __unicode__(self): + return self.ending + +class CountryAlias(models.Model): + """Records that alias should be treated as country for statistical + purposes.""" + + alias = models.CharField(max_length=255, help_text="Note that lower-case aliases are matched case-insensitive while aliases with at least one uppercase letter is matched case-sensitive. So 'United States' is best entered as 'united states' so it both matches 'United States' and 'United states' and 'UNITED STATES', whereas 'US' is best entered as 'US' so it doesn't accidentally match an ordinary word like 'us'.") + country = models.ForeignKey(CountryName, max_length=255) + + def __unicode__(self): + return u"{} -> {}".format(self.alias, self.country.name) + + class Meta: + verbose_name_plural = "country aliases" + +class MeetingRegistration(models.Model): + """Registration attendee records from the IETF registration system""" + meeting = models.ForeignKey(Meeting) + first_name = models.CharField(max_length=255) + last_name = models.CharField(max_length=255) + affiliation = models.CharField(blank=True, max_length=255) + country_code = models.CharField(max_length=2) # ISO 3166 + person = models.ForeignKey(Person, blank=True, null=True) + + def __unicode__(self): + return u"{} {}".format(self.first_name, self.last_name) diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py new file mode 100644 index 000000000..979863762 --- /dev/null +++ b/ietf/stats/resources.py @@ -0,0 +1,69 @@ +# Autogenerated by the makeresources management command 2017-02-15 10:10 PST +from tastypie.resources import ModelResource +from tastypie.fields import ToManyField # pyflakes:ignore +from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore +from tastypie.cache import SimpleCache + +from ietf import api +from ietf.api import ToOneField # pyflakes:ignore + +from ietf.stats.models import CountryAlias, AffiliationIgnoredEnding, AffiliationAlias, MeetingRegistration + + +from ietf.name.resources import CountryNameResource +class CountryAliasResource(ModelResource): + country = ToOneField(CountryNameResource, 'country') + class Meta: + queryset = CountryAlias.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'countryalias' + filtering = { + "id": ALL, + "alias": ALL, + "country": ALL_WITH_RELATIONS, + } +api.stats.register(CountryAliasResource()) + +class AffiliationIgnoredEndingResource(ModelResource): + class Meta: + queryset = AffiliationIgnoredEnding.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'affiliationignoredending' + filtering = { + "id": ALL, + "ending": ALL, + } +api.stats.register(AffiliationIgnoredEndingResource()) + +class AffiliationAliasResource(ModelResource): + class Meta: + queryset = AffiliationAlias.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'affiliationalias' + filtering = { + "id": ALL, + "alias": ALL, + "name": ALL, + } +api.stats.register(AffiliationAliasResource()) + +class MeetingRegistrationResource(ModelResource): + class Meta: + queryset = MeetingRegistration.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'meetingregistration' + filtering = { + "id": ALL, + "meeting": ALL_WITH_RELATIONS, + "first_name": ALL, + "last_name": ALL, + "affiliation": ALL, + "country_code": ALL, + "person": ALL_WITH_RELATIONS + } +api.stats.register(MeetingRegistrationResource()) + diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 57d0faa28..1fbc2c70c 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,17 +1,149 @@ +import datetime + +from mock import patch from pyquery import PyQuery +from requests import Response from django.urls import reverse as urlreverse from ietf.utils.test_data import make_test_data, make_review_data -from ietf.utils.test_utils import login_testing_unauthorized, TestCase +from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent import ietf.stats.views +from ietf.submit.models import Submission +from ietf.doc.models import Document, DocAlias, State, RelatedDocument, NewRevisionDocEvent +from ietf.meeting.factories import MeetingFactory +from ietf.person.models import Person +from ietf.name.models import FormalLanguageName, DocRelationshipName, CountryName +from ietf.stats.models import MeetingRegistration, CountryAlias +from ietf.stats.utils import get_meeting_registration_data + class StatisticsTests(TestCase): def test_stats_index(self): url = urlreverse(ietf.stats.views.stats_index) r = self.client.get(url) self.assertEqual(r.status_code, 200) + def test_document_stats(self): + draft = make_test_data() + + # create some data for the statistics + Submission.objects.create( + authors=[ { "name": "Some Body", "email": "somebody@example.com", "affiliation": "Some Inc.", "country": "US" }], + pages=30, + rev=draft.rev, + words=4000, + draft=draft, + file_types="txt", + state_id="posted", + ) + + draft.formal_languages.add(FormalLanguageName.objects.get(slug="xml")) + Document.objects.filter(pk=draft.pk).update(words=4000) + # move it back so it shows up in the yearly summaries + NewRevisionDocEvent.objects.filter(doc=draft, rev=draft.rev).update( + time=datetime.datetime.now() - datetime.timedelta(days=500)) + + referencing_draft = Document.objects.create( + name="draft-ietf-mars-referencing", + type_id="draft", + title="Referencing", + stream_id="ietf", + abstract="Test", + rev="00", + pages=2, + words=100 + ) + referencing_draft.set_state(State.objects.get(used=True, type="draft", slug="active")) + DocAlias.objects.create(document=referencing_draft, name=referencing_draft.name) + RelatedDocument.objects.create( + source=referencing_draft, + target=draft.docalias_set.first(), + relationship=DocRelationshipName.objects.get(slug="refinfo") + ) + NewRevisionDocEvent.objects.create( + type="new_revision", + by=Person.objects.get(name="(System)"), + doc=referencing_draft, + desc="New revision available", + rev=referencing_draft.rev, + time=datetime.datetime.now() - datetime.timedelta(days=1000) + ) + + + # check redirect + url = urlreverse(ietf.stats.views.document_stats) + + authors_url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": "authors" }) + + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertTrue(authors_url in r["Location"]) + + # check various stats types + for stats_type in ["authors", "pages", "words", "format", "formlang", + "author/documents", "author/affiliation", "author/country", + "author/continent", "author/citations", "author/hindex", + "yearly/affiliation", "yearly/country", "yearly/continent"]: + for document_type in ["", "rfc", "draft"]: + for time_choice in ["", "5y"]: + url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type }) + r = self.client.get(url, { + "type": document_type, + "time": time_choice, + }) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(q('#chart')) + if not stats_type.startswith("yearly"): + self.assertTrue(q('table.stats-data')) + + def test_meeting_stats(self): + # create some data for the statistics + make_test_data() + meeting = MeetingFactory(type_id='ietf', date=datetime.date.today(), number="96") + MeetingRegistration.objects.create(first_name='John', last_name='Smith', country_code='US', meeting=meeting) + CountryAlias.objects.get_or_create(alias="US", country=CountryName.objects.get(slug="US")) + MeetingRegistration.objects.create(first_name='Jaume', last_name='Guillaume', country_code='FR', meeting=meeting) + CountryAlias.objects.get_or_create(alias="FR", country=CountryName.objects.get(slug="FR")) + + # check redirect + url = urlreverse(ietf.stats.views.meeting_stats) + + authors_url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": "overview" }) + + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertTrue(authors_url in r["Location"]) + + # check various stats types + for stats_type in ["overview", "country", "continent"]: + url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": stats_type }) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(q('#chart')) + if stats_type == "overview": + self.assertTrue(q('table.stats-data')) + + for stats_type in ["country", "continent"]: + url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": stats_type, "num": meeting.number }) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(q('#chart')) + self.assertTrue(q('table.stats-data')) + + def test_known_country_list(self): + make_test_data() + + # check redirect + url = urlreverse(ietf.stats.views.known_countries_list) + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertTrue("United States" in unicontent(r)) + def test_review_stats(self): doc = make_test_data() review_req = make_review_data(doc) @@ -56,3 +188,14 @@ class StatisticsTests(TestCase): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('.review-stats td:contains("1")')) + + @patch('requests.get') + def test_get_meeting_registration_data(self, mock_get): + response = Response() + response.status_code = 200 + response._content = '[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]' + mock_get.return_value = response + meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96") + get_meeting_registration_data(meeting) + query = MeetingRegistration.objects.filter(first_name='John',last_name='Smith',country_code='US') + self.assertTrue(query.count(), 1) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index f19c313c0..94fbe6951 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -5,5 +5,9 @@ from ietf.utils.urls import url urlpatterns = [ url("^$", views.stats_index), + url("^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", views.document_stats), + url("^knowncountries/$", views.known_countries_list), + url("^meeting/(?P\d+)/(?Pcountry|continent)/$", views.meeting_stats), + url("^meeting/(?:(?Poverview|country|continent)/)?$", views.meeting_stats), url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), ] diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py new file mode 100644 index 000000000..9b0f4cd70 --- /dev/null +++ b/ietf/stats/utils.py @@ -0,0 +1,238 @@ +import re +import requests +from collections import defaultdict + +from django.conf import settings + +from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias, MeetingRegistration +from ietf.name.models import CountryName + +def compile_affiliation_ending_stripping_regexp(): + parts = [] + for ending_re in AffiliationIgnoredEnding.objects.values_list("ending", flat=True): + try: + re.compile(ending_re) + except re.error: + pass + + parts.append(ending_re) + + re_str = ",? *({}) *$".format("|".join(parts)) + + return re.compile(re_str, re.IGNORECASE) + + +def get_aliased_affiliations(affiliations): + """Given non-unique sequence of affiliations, returns dictionary with + aliases needed. + + We employ the following strategies, interleaved: + + - Stripping company endings like Inc., GmbH etc. from database + + - Looking up aliases stored directly in the database, like + "Examplar International" -> "Examplar" + + - Case-folding so Examplar and EXAMPLAR is merged with the + winner being the one with most occurrences (so input should not + be made unique) or most upper case letters in case of ties. + Case folding can be overridden by the aliases in the database.""" + + res = {} + + ending_re = compile_affiliation_ending_stripping_regexp() + + known_aliases = { alias.lower(): name for alias, name in AffiliationAlias.objects.values_list("alias", "name") } + + affiliations_with_case_spellings = defaultdict(set) + case_spelling_count = defaultdict(int) + for affiliation in affiliations: + original_affiliation = affiliation + + # check aliases from DB + name = known_aliases.get(affiliation.lower()) + if name is not None: + affiliation = name + res[original_affiliation] = affiliation + + # strip ending + name = ending_re.sub("", affiliation) + if name != affiliation: + affiliation = name + res[original_affiliation] = affiliation + + # check aliases from DB + name = known_aliases.get(affiliation.lower()) + if name is not None: + affiliation = name + res[original_affiliation] = affiliation + + affiliations_with_case_spellings[affiliation.lower()].add(original_affiliation) + case_spelling_count[affiliation] += 1 + + def affiliation_sort_key(affiliation): + count = case_spelling_count[affiliation] + uppercase_letters = sum(1 for c in affiliation if c.isupper()) + return (count, uppercase_letters) + + # now we just need to pick the most popular uppercase/lowercase + # spelling for each affiliation with more than one + for similar_affiliations in affiliations_with_case_spellings.itervalues(): + if len(similar_affiliations) > 1: + most_popular = sorted(similar_affiliations, key=affiliation_sort_key, reverse=True)[0] + for affiliation in similar_affiliations: + if affiliation != most_popular: + res[affiliation] = most_popular + + return res + + +def get_aliased_countries(countries): + known_aliases = dict(CountryAlias.objects.values_list("alias", "country__name")) + + # add aliases for known countries + for slug, name in CountryName.objects.values_list("slug", "name"): + known_aliases[name.lower()] = name + + def lookup_alias(possible_alias): + name = known_aliases.get(possible_alias) + if name is not None: + return name + + name = known_aliases.get(possible_alias.lower()) + if name is not None: + return name + + return possible_alias + + known_re_aliases = { + re.compile(u"\\b{}\\b".format(re.escape(alias))): name + for alias, name in known_aliases.iteritems() + } + + # specific hack: check for zip codes from the US since in the + # early days, the addresses often didn't include the country + us_zipcode_re = re.compile(r"\b(AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY|AS|GU|MP|PR|VI|UM|FM|MH|PW|Ca|Cal.|California|CALIFORNIA|Colorado|Georgia|Illinois|Ill|Maryland|Ma|Ma.|Mass|Massachuss?etts|Michigan|Minnesota|New Jersey|New York|Ny|N.Y.|North Carolina|NORTH CAROLINA|Ohio|Oregon|Pennsylvania|Tx|Texas|Tennessee|Utah|Vermont|Virginia|Va.|Washington)[., -]*[0-9]{5}\b") + + us_country_name = CountryName.objects.get(slug="US").name + + def last_text_part_stripped(split): + for t in reversed(split): + t = t.strip() + if t: + return t + return u"" + + known_countries = set(CountryName.objects.values_list("name", flat=True)) + + res = {} + + for country in countries: + if country in res or country in known_countries: + continue + + original_country = country + + # aliased name + country = lookup_alias(country) + if country in known_countries: + res[original_country] = country + continue + + # contains US zipcode + if us_zipcode_re.search(country): + res[original_country] = us_country_name + continue + + # do a little bit of cleanup + if len(country) > 1 and country[-1] == "." and not country[-2].isupper(): + country = country.rstrip(".") + + country = country.strip("-,").strip() + + # aliased name + country = lookup_alias(country) + if country in known_countries: + res[original_country] = country + continue + + # country name at end, separated by comma + last_part = lookup_alias(last_text_part_stripped(country.split(","))) + if last_part in known_countries: + res[original_country] = last_part + continue + + # country name at end, separated by whitespace + last_part = lookup_alias(last_text_part_stripped(country.split())) + if last_part in known_countries: + res[original_country] = last_part + continue + + # country name anywhere + country_lower = country.lower() + found = False + for alias_re, name in known_re_aliases.iteritems(): + if alias_re.search(country) or alias_re.search(country_lower): + res[original_country] = name + found = True + break + + if found: + continue + + # unknown country + res[original_country] = "" + + return res + + +def clean_country_name(country_name): + if country_name: + country_name = get_aliased_countries([country_name]).get(country_name, country_name) + if country_name and CountryName.objects.filter(name=country_name).exists(): + return country_name + + return "" + + +def compute_hirsch_index(citation_counts): + """Computes the h-index given a sequence containing the number of + citations for each document.""" + + i = 0 + + for count in sorted(citation_counts, reverse=True): + if i + 1 > count: + break + + i += 1 + + return i + + +def get_meeting_registration_data(meeting): + """"Retrieve registration attendee data and summary statistics. Returns number + of Registration records created.""" + num_created = 0 + response = requests.get(settings.REGISTRATION_ATTENDEES_BASE_URL + meeting.number) + if response.status_code == 200: + decoded = [] + try: + decoded = response.json() + except ValueError: + pass + + for registration in decoded: + object, created = MeetingRegistration.objects.get_or_create( + meeting_id=meeting.pk, + first_name=registration['FirstName'], + last_name=registration['LastName'], + affiliation=registration['Company'], + country_code=registration['Country']) + if created: + num_created += 1 + return num_created + + + + diff --git a/ietf/stats/views.py b/ietf/stats/views.py index e161a408c..71dc9896f 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,9 +1,18 @@ -import datetime, itertools, json, calendar +import datetime +import itertools +import json +import calendar +import os +from collections import defaultdict from django.shortcuts import render from django.contrib.auth.decorators import login_required from django.urls import reverse as urlreverse from django.http import HttpResponseRedirect, HttpResponseForbidden +from django.db.models import Count, Q +from django.utils.safestring import mark_safe +from django.conf import settings +from django.shortcuts import get_object_or_404 import dateutil.relativedelta @@ -12,14 +21,921 @@ from ietf.review.utils import (extract_review_request_data, ReviewRequestData, compute_review_request_stats, sum_raw_review_request_aggregations) +from ietf.submit.models import Submission from ietf.group.models import Role, Group from ietf.person.models import Person -from ietf.name.models import ReviewRequestStateName, ReviewResultName +from ietf.name.models import ReviewRequestStateName, ReviewResultName, CountryName, DocRelationshipName +from ietf.person.name import plain_name +from ietf.doc.models import DocAlias, Document, State, DocEvent +from ietf.meeting.models import Meeting +from ietf.stats.models import MeetingRegistration, CountryAlias +from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries, compute_hirsch_index from ietf.ietfauth.utils import has_role def stats_index(request): return render(request, "stats/index.html") +def generate_query_string(query_dict, overrides): + query_part = u"" + + if query_dict or overrides: + d = query_dict.copy() + for k, v in overrides.iteritems(): + if type(v) in (list, tuple): + if not v: + if k in d: + del d[k] + else: + d.setlist(k, v) + else: + if v is None or v == u"": + if k in d: + del d[k] + else: + d[k] = v + + if d: + query_part = u"?" + d.urlencode() + + return query_part + +def get_choice(request, get_parameter, possible_choices, multiple=False): + # the statistics are built with links to make navigation faster, + # so we don't really have a form in most cases, so just use this + # helper instead to select between the choices + values = request.GET.getlist(get_parameter) + found = [t[0] for t in possible_choices if t[0] in values] + + if multiple: + return found + else: + if found: + return found[0] + else: + return None + +def add_url_to_choices(choices, url_builder): + return [ (slug, label, url_builder(slug)) for slug, label in choices] + +def put_into_bin(value, bin_size): + if value is None: + return (value, value) + + v = (value // bin_size) * bin_size + return (v, "{} - {}".format(v, v + bin_size - 1)) + +def prune_unknown_bin_with_known(bins): + # remove from the unknown bin all authors within the + # named/known bins + all_known = { n for b, names in bins.iteritems() if b for n in names } + bins[""] = [name for name in bins[""] if name not in all_known] + if not bins[""]: + del bins[""] + +def count_bins(bins): + return len({ n for b, names in bins.iteritems() if b for n in names }) + +def add_labeled_top_series_from_bins(chart_data, bins, limit): + """Take bins on the form (x, label): [name1, name2, ...], figure out + how many there are per label, take the overall top ones and put + them into sorted series like [(x1, len(names1)), (x2, len(names2)), ...].""" + aggregated_bins = defaultdict(set) + xs = set() + for (x, label), names in bins.iteritems(): + xs.add(x) + aggregated_bins[label].update(names) + + xs = list(sorted(xs)) + + sorted_bins = sorted(aggregated_bins.iteritems(), key=lambda t: len(t[1]), reverse=True) + top = [ label for label, names in list(sorted_bins)[:limit]] + + for label in top: + series_data = [] + + for x in xs: + names = bins.get((x, label), set()) + + series_data.append((x, len(names))) + + chart_data.append({ + "data": series_data, + "name": label + }) + +def document_stats(request, stats_type=None): + def build_document_stats_url(stats_type_override=Ellipsis, get_overrides={}): + kwargs = { + "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, + } + + return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) + + # statistics types + possible_document_stats_types = add_url_to_choices([ + ("authors", "Number of authors"), + ("pages", "Pages"), + ("words", "Words"), + ("format", "Format"), + ("formlang", "Formal languages"), + ], lambda slug: build_document_stats_url(stats_type_override=slug)) + + possible_author_stats_types = add_url_to_choices([ + ("author/documents", "Number of documents"), + ("author/affiliation", "Affiliation"), + ("author/country", "Country"), + ("author/continent", "Continent"), + ("author/citations", "Citations"), + ("author/hindex", "h-index"), + ], lambda slug: build_document_stats_url(stats_type_override=slug)) + + possible_yearly_stats_types = add_url_to_choices([ + ("yearly/affiliation", "Affiliation"), + ("yearly/country", "Country"), + ("yearly/continent", "Continent"), + ], lambda slug: build_document_stats_url(stats_type_override=slug)) + + + if not stats_type: + return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_document_stats_types[0][0])) + + + possible_document_types = add_url_to_choices([ + ("", "All"), + ("rfc", "RFCs"), + ("draft", "Drafts"), + ], lambda slug: build_document_stats_url(get_overrides={ "type": slug })) + + document_type = get_choice(request, "type", possible_document_types) or "" + + + possible_time_choices = add_url_to_choices([ + ("", "All time"), + ("5y", "Past 5 years"), + ], lambda slug: build_document_stats_url(get_overrides={ "time": slug })) + + time_choice = request.GET.get("time") or "" + + from_time = None + if "y" in time_choice: + try: + y = int(time_choice.rstrip("y")) + from_time = datetime.datetime.today() - dateutil.relativedelta.relativedelta(years=y) + except ValueError: + pass + + chart_data = [] + table_data = [] + stats_title = "" + template_name = stats_type.replace("/", "_") + bin_size = 1 + alias_data = [] + eu_countries = None + + + if any(stats_type == t[0] for t in possible_document_stats_types): + # filter documents + docalias_filters = Q(document__type="draft") + + rfc_state = State.objects.get(type="draft", slug="rfc") + if document_type == "rfc": + docalias_filters &= Q(document__states=rfc_state) + elif document_type == "draft": + docalias_filters &= ~Q(document__states=rfc_state) + + if from_time: + # this is actually faster than joining in the database, + # despite the round-trip back and forth + docs_within_time_constraint = list(Document.objects.filter( + type="draft", + docevent__time__gte=from_time, + docevent__type__in=["published_rfc", "new_revision"], + ).values_list("pk")) + + docalias_filters &= Q(document__in=docs_within_time_constraint) + + docalias_qs = DocAlias.objects.filter(docalias_filters) + + if document_type == "rfc": + doc_label = "RFC" + elif document_type == "draft": + doc_label = "draft" + else: + doc_label = "document" + + total_docs = docalias_qs.values_list("document").distinct().count() + + def generate_canonical_names(docalias_qs): + for doc_id, ts in itertools.groupby(docalias_qs.order_by("document"), lambda t: t[0]): + chosen = None + for t in ts: + if chosen is None: + chosen = t + else: + if t[1].startswith("rfc"): + chosen = t + elif t[1].startswith("draft") and not chosen[1].startswith("rfc"): + chosen = t + + yield chosen + + if stats_type == "authors": + stats_title = "Number of authors for each {}".format(doc_label) + + bins = defaultdict(set) + + for name, canonical_name, author_count in generate_canonical_names(docalias_qs.values_list("document", "name").annotate(Count("document__documentauthor"))): + bins[author_count].add(canonical_name) + + series_data = [] + for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / (total_docs or 1) + series_data.append((author_count, percentage)) + table_data.append((author_count, percentage, names)) + + chart_data.append({ "data": series_data }) + + elif stats_type == "pages": + stats_title = "Number of pages for each {}".format(doc_label) + + bins = defaultdict(set) + + for name, canonical_name, pages in generate_canonical_names(docalias_qs.values_list("document", "name", "document__pages")): + bins[pages].add(canonical_name) + + series_data = [] + for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / (total_docs or 1) + if pages is not None: + series_data.append((pages, len(names))) + table_data.append((pages, percentage, names)) + + chart_data.append({ "data": series_data }) + + elif stats_type == "words": + stats_title = "Number of words for each {}".format(doc_label) + + bin_size = 500 + + bins = defaultdict(set) + + for name, canonical_name, words in generate_canonical_names(docalias_qs.values_list("document", "name", "document__words")): + bins[put_into_bin(words, bin_size)].add(canonical_name) + + series_data = [] + for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]): + percentage = len(names) * 100.0 / (total_docs or 1) + if words is not None: + series_data.append((value, len(names))) + + table_data.append((words, percentage, names)) + + chart_data.append({ "data": series_data }) + + elif stats_type == "format": + stats_title = "Submission formats for each {}".format(doc_label) + + bins = defaultdict(set) + + # on new documents, we should have a Submission row with the file types + submission_types = {} + + for doc_name, file_types in Submission.objects.values_list("draft", "file_types").order_by("submission_date", "id"): + submission_types[doc_name] = file_types + + doc_names_with_missing_types = {} + for doc_name, canonical_name, rev in generate_canonical_names(docalias_qs.values_list("document", "name", "document__rev")): + types = submission_types.get(doc_name) + if types: + for dot_ext in types.split(","): + bins[dot_ext.lstrip(".").upper()].add(canonical_name) + + else: + + if canonical_name.startswith("rfc"): + filename = canonical_name + else: + filename = canonical_name + "-" + rev + + doc_names_with_missing_types[filename] = canonical_name + + # look up the remaining documents on disk + for filename in itertools.chain(os.listdir(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR), os.listdir(settings.RFC_PATH)): + t = filename.split(".", 1) + if len(t) != 2: + continue + + basename, ext = t + if any(ext.lower().endswith(blacklisted_ext.lower()) for blacklisted_ext in settings.DOCUMENT_FORMAT_BLACKLIST): + continue + + canonical_name = doc_names_with_missing_types.get(basename) + + if canonical_name: + bins[ext.upper()].add(canonical_name) + + series_data = [] + for fmt, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / (total_docs or 1) + series_data.append((fmt, len(names))) + + table_data.append((fmt, percentage, names)) + + chart_data.append({ "data": series_data }) + + elif stats_type == "formlang": + stats_title = "Formal languages used for each {}".format(doc_label) + + bins = defaultdict(set) + + for name, canonical_name, formal_language_name in generate_canonical_names(docalias_qs.values_list("document", "name", "document__formal_languages__name")): + bins[formal_language_name].add(canonical_name) + + series_data = [] + for formal_language, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / (total_docs or 1) + if formal_language is not None: + series_data.append((formal_language, len(names))) + table_data.append((formal_language, percentage, names)) + + chart_data.append({ "data": series_data }) + + elif any(stats_type == t[0] for t in possible_author_stats_types): + person_filters = Q(documentauthor__document__type="draft") + + # filter persons + rfc_state = State.objects.get(type="draft", slug="rfc") + if document_type == "rfc": + person_filters &= Q(documentauthor__document__states=rfc_state) + elif document_type == "draft": + person_filters &= ~Q(documentauthor__document__states=rfc_state) + + if from_time: + # this is actually faster than joining in the database, + # despite the round-trip back and forth + docs_within_time_constraint = set(Document.objects.filter( + type="draft", + docevent__time__gte=from_time, + docevent__type__in=["published_rfc", "new_revision"], + ).values_list("pk")) + + person_filters &= Q(documentauthor__document__in=docs_within_time_constraint) + + person_qs = Person.objects.filter(person_filters) + + if document_type == "rfc": + doc_label = "RFC" + elif document_type == "draft": + doc_label = "draft" + else: + doc_label = "document" + + if stats_type == "author/documents": + stats_title = "Number of {}s per author".format(doc_label) + + bins = defaultdict(set) + + person_qs = Person.objects.filter(person_filters) + + for name, document_count in person_qs.values_list("name").annotate(Count("documentauthor")): + bins[document_count].add(name) + + total_persons = count_bins(bins) + + series_data = [] + for document_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / (total_persons or 1) + series_data.append((document_count, percentage)) + table_data.append((document_count, percentage, [plain_name(n) for n in names])) + + chart_data.append({ "data": series_data }) + + elif stats_type == "author/affiliation": + stats_title = "Number of {} authors per affiliation".format(doc_label) + + bins = defaultdict(set) + + person_qs = Person.objects.filter(person_filters) + + # Since people don't write the affiliation names in the + # same way, and we don't want to go back and edit them + # either, we transform them here. + + name_affiliation_set = { + (name, affiliation) + for name, affiliation in person_qs.values_list("name", "documentauthor__affiliation") + } + + aliases = get_aliased_affiliations(affiliation for _, affiliation in name_affiliation_set) + + for name, affiliation in name_affiliation_set: + bins[aliases.get(affiliation, affiliation)].add(name) + + prune_unknown_bin_with_known(bins) + total_persons = count_bins(bins) + + series_data = [] + for affiliation, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / (total_persons or 1) + if affiliation: + series_data.append((affiliation, len(names))) + table_data.append((affiliation, percentage, [plain_name(n) for n in names])) + + series_data.sort(key=lambda t: t[1], reverse=True) + series_data = series_data[:30] + + chart_data.append({ "data": series_data }) + + for alias, name in sorted(aliases.iteritems(), key=lambda t: t[1]): + alias_data.append((name, alias)) + + elif stats_type == "author/country": + stats_title = "Number of {} authors per country".format(doc_label) + + bins = defaultdict(set) + + person_qs = Person.objects.filter(person_filters) + + # Since people don't write the country names in the + # same way, and we don't want to go back and edit them + # either, we transform them here. + + name_country_set = { + (name, country) + for name, country in person_qs.values_list("name", "documentauthor__country") + } + + aliases = get_aliased_countries(country for _, country in name_country_set) + + countries = { c.name: c for c in CountryName.objects.all() } + eu_name = "EU" + eu_countries = { c for c in countries.itervalues() if c.in_eu } + + for name, country in name_country_set: + country_name = aliases.get(country, country) + bins[country_name].add(name) + + c = countries.get(country_name) + if c and c.in_eu: + bins[eu_name].add(name) + + prune_unknown_bin_with_known(bins) + total_persons = count_bins(bins) + + series_data = [] + for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / (total_persons or 1) + if country: + series_data.append((country, len(names))) + table_data.append((country, percentage, [plain_name(n) for n in names])) + + series_data.sort(key=lambda t: t[1], reverse=True) + series_data = series_data[:30] + + chart_data.append({ "data": series_data }) + + for alias, country_name in aliases.iteritems(): + alias_data.append((country_name, alias, countries.get(country_name))) + + alias_data.sort() + + elif stats_type == "author/continent": + stats_title = "Number of {} authors per continent".format(doc_label) + + bins = defaultdict(set) + + person_qs = Person.objects.filter(person_filters) + + name_country_set = { + (name, country) + for name, country in person_qs.values_list("name", "documentauthor__country") + } + + aliases = get_aliased_countries(country for _, country in name_country_set) + + country_to_continent = dict(CountryName.objects.values_list("name", "continent__name")) + + for name, country in name_country_set: + country_name = aliases.get(country, country) + continent_name = country_to_continent.get(country_name, "") + bins[continent_name].add(name) + + prune_unknown_bin_with_known(bins) + total_persons = count_bins(bins) + + series_data = [] + for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / (total_persons or 1) + if continent: + series_data.append((continent, len(names))) + table_data.append((continent, percentage, [plain_name(n) for n in names])) + + series_data.sort(key=lambda t: t[1], reverse=True) + + chart_data.append({ "data": series_data }) + + elif stats_type == "author/citations": + stats_title = "Number of citations of {}s written by author".format(doc_label) + + bins = defaultdict(set) + + cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold'])) + person_filters &= Q(documentauthor__document__docalias__relateddocument__relationship__in=cite_relationships) + + person_qs = Person.objects.filter(person_filters) + + for name, citations in person_qs.values_list("name").annotate(Count("documentauthor__document__docalias__relateddocument")): + bins[citations].add(name) + + total_persons = count_bins(bins) + + series_data = [] + for citations, names in sorted(bins.iteritems(), key=lambda t: t[0], reverse=True): + percentage = len(names) * 100.0 / (total_persons or 1) + series_data.append((citations, percentage)) + table_data.append((citations, percentage, [plain_name(n) for n in names])) + + chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) }) + + elif stats_type == "author/hindex": + stats_title = "h-index for {}s written by author".format(doc_label) + + bins = defaultdict(set) + + cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold'])) + person_filters &= Q(documentauthor__document__docalias__relateddocument__relationship__in=cite_relationships) + + person_qs = Person.objects.filter(person_filters) + + values = person_qs.values_list("name", "documentauthor__document").annotate(Count("documentauthor__document__docalias__relateddocument")) + for name, ts in itertools.groupby(values.order_by("name"), key=lambda t: t[0]): + h_index = compute_hirsch_index([citations for _, document, citations in ts]) + bins[h_index].add(name) + + total_persons = count_bins(bins) + + series_data = [] + for citations, names in sorted(bins.iteritems(), key=lambda t: t[0], reverse=True): + percentage = len(names) * 100.0 / (total_persons or 1) + series_data.append((citations, percentage)) + table_data.append((citations, percentage, [plain_name(n) for n in names])) + + chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) }) + + elif any(stats_type == t[0] for t in possible_yearly_stats_types): + + person_filters = Q(documentauthor__document__type="draft") + + # filter persons + rfc_state = State.objects.get(type="draft", slug="rfc") + if document_type == "rfc": + person_filters &= Q(documentauthor__document__states=rfc_state) + elif document_type == "draft": + person_filters &= ~Q(documentauthor__document__states=rfc_state) + + doc_years = defaultdict(set) + + docevent_qs = DocEvent.objects.filter( + doc__type="draft", + type__in=["published_rfc", "new_revision"], + ).values_list("doc", "time").order_by("doc") + + for doc, time in docevent_qs.iterator(): + doc_years[doc].add(time.year) + + person_qs = Person.objects.filter(person_filters) + + if document_type == "rfc": + doc_label = "RFC" + elif document_type == "draft": + doc_label = "draft" + else: + doc_label = "document" + + template_name = "yearly" + + years_from = from_time.year if from_time else 1 + years_to = datetime.date.today().year - 1 + + + if stats_type == "yearly/affiliation": + stats_title = "Number of {} authors per affiliation over the years".format(doc_label) + + person_qs = Person.objects.filter(person_filters) + + name_affiliation_doc_set = { + (name, affiliation, doc) + for name, affiliation, doc in person_qs.values_list("name", "documentauthor__affiliation", "documentauthor__document") + } + + aliases = get_aliased_affiliations(affiliation for _, affiliation, _ in name_affiliation_doc_set) + + bins = defaultdict(set) + for name, affiliation, doc in name_affiliation_doc_set: + a = aliases.get(affiliation, affiliation) + if a: + for year in doc_years.get(doc): + if years_from <= year <= years_to: + bins[(year, a)].add(name) + + add_labeled_top_series_from_bins(chart_data, bins, limit=8) + + elif stats_type == "yearly/country": + stats_title = "Number of {} authors per country over the years".format(doc_label) + + person_qs = Person.objects.filter(person_filters) + + name_country_doc_set = { + (name, country, doc) + for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document") + } + + aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set) + + countries = { c.name: c for c in CountryName.objects.all() } + eu_name = "EU" + eu_countries = { c for c in countries.itervalues() if c.in_eu } + + bins = defaultdict(set) + + for name, country, doc in name_country_doc_set: + country_name = aliases.get(country, country) + c = countries.get(country_name) + + if country_name: + for year in doc_years.get(doc): + if years_from <= year <= years_to: + bins[(year, country_name)].add(name) + + if c and c.in_eu: + bins[(year, eu_name)].add(name) + + add_labeled_top_series_from_bins(chart_data, bins, limit=8) + + + elif stats_type == "yearly/continent": + stats_title = "Number of {} authors per continent".format(doc_label) + + person_qs = Person.objects.filter(person_filters) + + name_country_doc_set = { + (name, country, doc) + for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document") + } + + aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set) + + country_to_continent = dict(CountryName.objects.values_list("name", "continent__name")) + + bins = defaultdict(set) + + for name, country, doc in name_country_doc_set: + country_name = aliases.get(country, country) + continent_name = country_to_continent.get(country_name, "") + + if continent_name: + for year in doc_years.get(doc): + if years_from <= year <= years_to: + bins[(year, continent_name)].add(name) + + add_labeled_top_series_from_bins(chart_data, bins, limit=8) + + return render(request, "stats/document_stats.html", { + "chart_data": mark_safe(json.dumps(chart_data)), + "table_data": table_data, + "stats_title": stats_title, + "possible_document_stats_types": possible_document_stats_types, + "possible_author_stats_types": possible_author_stats_types, + "possible_yearly_stats_types": possible_yearly_stats_types, + "stats_type": stats_type, + "possible_document_types": possible_document_types, + "document_type": document_type, + "possible_time_choices": possible_time_choices, + "time_choice": time_choice, + "doc_label": doc_label, + "bin_size": bin_size, + "show_aliases_url": build_document_stats_url(get_overrides={ "showaliases": "1" }), + "hide_aliases_url": build_document_stats_url(get_overrides={ "showaliases": None }), + "alias_data": alias_data, + "eu_countries": sorted(eu_countries or [], key=lambda c: c.name), + "content_template": "stats/document_stats_{}.html".format(template_name), + }) + + +def known_countries_list(request, stats_type=None, acronym=None): + countries = CountryName.objects.prefetch_related("countryalias_set") + for c in countries: + # the sorting is a bit of a hack - it puts the ISO code first + # since it was added in a migration + c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) + + return render(request, "stats/known_countries_list.html", { + "countries": countries, + "ticket_email_address": settings.SECRETARIAT_TICKET_EMAIL, + }) + +def meeting_stats(request, num=None, stats_type=None): + meeting = None + if num is not None: + meeting = get_object_or_404(Meeting, number=num, type="ietf") + + def build_meeting_stats_url(number=None, stats_type_override=Ellipsis, get_overrides={}): + kwargs = { + "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, + } + + if number is not None: + kwargs["num"] = number + + return urlreverse(meeting_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) + + # statistics types + if meeting: + possible_stats_types = add_url_to_choices([ + ("country", "Country"), + ("continent", "Continent"), + ], lambda slug: build_meeting_stats_url(number=meeting.number, stats_type_override=slug)) + else: + possible_stats_types = add_url_to_choices([ + ("overview", "Overview"), + ("country", "Country"), + ("continent", "Continent"), + ], lambda slug: build_meeting_stats_url(number=None, stats_type_override=slug)) + + if not stats_type: + return HttpResponseRedirect(build_meeting_stats_url(number=num, stats_type_override=possible_stats_types[0][0])) + + chart_data = [] + piechart_data = [] + table_data = [] + stats_title = "" + template_name = stats_type + bin_size = 1 + eu_countries = None + + def get_country_mapping(registrations): + return { + alias.alias: alias.country + for alias in CountryAlias.objects.filter(alias__in=set(r.country_code for r in registrations)).select_related("country", "country__continent") + if alias.alias.isupper() + } + + if meeting and any(stats_type == t[0] for t in possible_stats_types): + registrations = MeetingRegistration.objects.filter(meeting=meeting) + + if stats_type == "country": + stats_title = "Number of registrations for {} {} per country".format(meeting.type.name, meeting.number) + + bins = defaultdict(set) + + country_mapping = get_country_mapping(registrations) + + eu_name = "EU" + eu_countries = set(CountryName.objects.filter(in_eu=True)) + + for r in registrations: + name = (r.first_name + " " + r.last_name).strip() + c = country_mapping.get(r.country_code) + bins[c.name if c else ""].add(name) + + if c and c.in_eu: + bins[eu_name].add(name) + + prune_unknown_bin_with_known(bins) + total_registrations = count_bins(bins) + + series_data = [] + for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / (total_registrations or 1) + if country: + series_data.append((country, len(names))) + table_data.append((country, percentage, names)) + + if country and country != eu_name: + piechart_data.append({ "name": country, "y": percentage }) + + series_data.sort(key=lambda t: t[1], reverse=True) + series_data = series_data[:20] + + piechart_data.sort(key=lambda d: d["y"], reverse=True) + pie_cut_off = 8 + piechart_data = piechart_data[:pie_cut_off] + [{ "name": "Other", "y": sum(d["y"] for d in piechart_data[pie_cut_off:])}] + + chart_data.append({ "data": series_data }) + + elif stats_type == "continent": + stats_title = "Number of registrations for {} {} per continent".format(meeting.type.name, meeting.number) + + bins = defaultdict(set) + + country_mapping = get_country_mapping(registrations) + + for r in registrations: + name = (r.first_name + " " + r.last_name).strip() + c = country_mapping.get(r.country_code) + bins[c.continent.name if c else ""].add(name) + + prune_unknown_bin_with_known(bins) + total_registrations = count_bins(bins) + + series_data = [] + for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / (total_registrations or 1) + if continent: + series_data.append((continent, len(names))) + table_data.append((continent, percentage, names)) + + series_data.sort(key=lambda t: t[1], reverse=True) + + chart_data.append({ "data": series_data }) + + + elif not meeting and any(stats_type == t[0] for t in possible_stats_types): + template_name = "overview" + + registrations = MeetingRegistration.objects.filter(meeting__type="ietf") + + if stats_type == "overview": + stats_title = "Number of registrations per meeting" + + bins = defaultdict(set) + + for first_name, last_name, meeting_number in registrations.values_list("first_name", "last_name", "meeting__number"): + meeting_number = int(meeting_number) + name = (first_name + " " + last_name).strip() + + bins[meeting_number].add(name) + + meeting_cities = dict(Meeting.objects.filter(number__in=bins.iterkeys()).values_list("number", "city")) + + series_data = [] + for meeting_number, names in sorted(bins.iteritems()): + series_data.append((meeting_number, len(names))) + url = build_meeting_stats_url(number=meeting_number, stats_type_override="country") + label = "IETF {} - {}".format(meeting_number, meeting_cities.get(str(meeting_number), "")) + table_data.append((meeting_number, label, url, names)) + + series_data.sort(key=lambda t: t[0]) + table_data.sort(key=lambda t: t[0], reverse=True) + + chart_data.append({ "name": "Registrations", "data": series_data }) + + + elif stats_type == "country": + stats_title = "Number of registrations per country across meetings" + + country_mapping = get_country_mapping(registrations) + + eu_name = "EU" + eu_countries = set(CountryName.objects.filter(in_eu=True)) + + bins = defaultdict(set) + + for first_name, last_name, country_code, meeting_number in registrations.values_list("first_name", "last_name", "country_code", "meeting__number"): + meeting_number = int(meeting_number) + name = (first_name + " " + last_name).strip() + c = country_mapping.get(country_code) + + if c: + bins[(meeting_number, c.name)].add(name) + if c.in_eu: + bins[(meeting_number, eu_name)].add(name) + + add_labeled_top_series_from_bins(chart_data, bins, limit=8) + + + elif stats_type == "continent": + stats_title = "Number of registrations per country across meetings" + + country_mapping = get_country_mapping(registrations) + + bins = defaultdict(set) + + for first_name, last_name, country_code, meeting_number in registrations.values_list("first_name", "last_name", "country_code", "meeting__number"): + meeting_number = int(meeting_number) + name = (first_name + " " + last_name).strip() + c = country_mapping.get(country_code) + + if c: + bins[(meeting_number, c.continent.name)].add(name) + + add_labeled_top_series_from_bins(chart_data, bins, limit=8) + + + return render(request, "stats/meeting_stats.html", { + "chart_data": mark_safe(json.dumps(chart_data)), + "piechart_data": mark_safe(json.dumps(piechart_data)), + "table_data": table_data, + "stats_title": stats_title, + "possible_stats_types": possible_stats_types, + "stats_type": stats_type, + "bin_size": bin_size, + "meeting": meeting, + "eu_countries": sorted(eu_countries or [], key=lambda c: c.name), + "content_template": "stats/meeting_stats_{}.html".format(template_name), + }) + + @login_required def review_stats(request, stats_type=None, acronym=None): # This view is a bit complex because we want to show a bunch of @@ -39,41 +955,7 @@ def review_stats(request, stats_type=None, acronym=None): if acr: kwargs["acronym"] = acr - base_url = urlreverse(review_stats, kwargs=kwargs) - query_part = u"" - - if request.GET or get_overrides: - d = request.GET.copy() - for k, v in get_overrides.iteritems(): - if type(v) in (list, tuple): - if not v: - if k in d: - del d[k] - else: - d.setlist(k, v) - else: - if v is None or v == u"": - if k in d: - del d[k] - else: - d[k] = v - - if d: - query_part = u"?" + d.urlencode() - - return base_url + query_part - - def get_choice(get_parameter, possible_choices, multiple=False): - values = request.GET.getlist(get_parameter) - found = [t[0] for t in possible_choices if t[0] in values] - - if multiple: - return found - else: - if found: - return found[0] - else: - return None + return urlreverse(review_stats, kwargs=kwargs) + generate_query_string(request.GET, get_overrides) # which overview - team or reviewer if acronym: @@ -91,21 +973,19 @@ def review_stats(request, stats_type=None, acronym=None): if level == "team": possible_stats_types.append(("time", "Changes over time")) - possible_stats_types = [ (slug, label, build_review_stats_url(stats_type_override=slug)) - for slug, label in possible_stats_types ] + possible_stats_types = add_url_to_choices(possible_stats_types, + lambda slug: build_review_stats_url(stats_type_override=slug)) if not stats_type: return HttpResponseRedirect(build_review_stats_url(stats_type_override=possible_stats_types[0][0])) # what to count - possible_count_choices = [ + possible_count_choices = add_url_to_choices([ ("", "Review requests"), ("pages", "Reviewed pages"), - ] + ], lambda slug: build_review_stats_url(get_overrides={ "count": slug })) - possible_count_choices = [ (slug, label, build_review_stats_url(get_overrides={ "count": slug })) for slug, label in possible_count_choices ] - - count = get_choice("count", possible_count_choices) or "" + count = get_choice(request, "count", possible_count_choices) or "" # time range def parse_date(s): @@ -190,7 +1070,7 @@ def review_stats(request, stats_type=None, acronym=None): if stats_type == "time": possible_teams = [(t.acronym, t.acronym) for t in teams] - selected_teams = get_choice("team", possible_teams, multiple=True) + selected_teams = get_choice(request, "team", possible_teams, multiple=True) def add_if_exists_else_subtract(element, l): if element in l: @@ -198,9 +1078,12 @@ def review_stats(request, stats_type=None, acronym=None): else: return l + [element] - possible_teams = [(slug, label, build_review_stats_url(get_overrides={ - "team": add_if_exists_else_subtract(slug, selected_teams) - })) for slug, label in possible_teams] + possible_teams = add_url_to_choices( + possible_teams, + lambda slug: build_review_stats_url(get_overrides={ + "team": add_if_exists_else_subtract(slug, selected_teams) + }) + ) query_teams = [t for t in query_teams if t.acronym in selected_teams] extracted_data = extract_review_request_data(query_teams, query_reviewers, from_time, to_time) @@ -232,33 +1115,28 @@ def review_stats(request, stats_type=None, acronym=None): # choice - possible_completion_types = [ + possible_completion_types = add_url_to_choices([ ("completed_in_time", "Completed in time"), ("completed_late", "Completed late"), ("not_completed", "Not completed"), ("average_assignment_to_closure_days", "Avg. compl. days"), - ] + ], lambda slug: build_review_stats_url(get_overrides={ "completion": slug, "result": None, "state": None })) - possible_completion_types = [ - (slug, label, build_review_stats_url(get_overrides={ "completion": slug, "result": None, "state": None })) - for slug, label in possible_completion_types - ] + selected_completion_type = get_choice(request, "completion", possible_completion_types) - selected_completion_type = get_choice("completion", possible_completion_types) + possible_results = add_url_to_choices( + [(r.slug, r.name) for r in results], + lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": slug, "state": None }) + ) - possible_results = [ - (r.slug, r.name, build_review_stats_url(get_overrides={ "completion": None, "result": r.slug, "state": None })) - for r in results - ] - - selected_result = get_choice("result", possible_results) + selected_result = get_choice(request, "result", possible_results) - possible_states = [ - (s.slug, s.name, build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": s.slug })) - for s in states - ] + possible_states = add_url_to_choices( + [(s.slug, s.name) for s in states], + lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": slug }) + ) - selected_state = get_choice("state", possible_states) + selected_state = get_choice(request, "state", possible_states) if not selected_completion_type and not selected_result and not selected_state: selected_completion_type = "completed_in_time" diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index 790936907..5e8844a62 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -21,6 +21,7 @@ from ietf.doc.fields import SearchableDocAliasesField from ietf.ipr.mail import utc_from_string from ietf.meeting.models import Meeting from ietf.message.models import Message +from ietf.name.models import FormalLanguageName from ietf.submit.models import Submission, Preapproval from ietf.submit.utils import validate_submission_rev, validate_submission_document_date from ietf.submit.parsers.pdf_parser import PDFParser @@ -29,7 +30,6 @@ from ietf.submit.parsers.ps_parser import PSParser from ietf.submit.parsers.xml_parser import XMLParser from ietf.utils.draft import Draft - class SubmissionUploadForm(forms.Form): txt = forms.FileField(label=u'.txt format', required=False) xml = forms.FileField(label=u'.xml format', required=False) @@ -177,18 +177,14 @@ class SubmissionUploadForm(forms.Form): self.abstract = self.xmlroot.findtext('front/abstract').strip() if type(self.abstract) is unicode: self.abstract = unidecode(self.abstract) - self.author_list = [] author_info = self.xmlroot.findall('front/author') for author in author_info: - author_dict = dict( - company = author.findtext('organization').strip(), - last_name = author.attrib.get('surname').strip(), - full_name = author.attrib.get('fullname').strip(), - email = author.findtext('address/email').strip(), - ) - self.author_list.append(author_dict) - line = email.utils.formataddr((author_dict['full_name'], author_dict['email'])) - self.authors.append(line) + self.authors.append({ + "name": author.attrib.get('fullname').strip(), + "email": author.findtext('address/email').strip(), + "affiliation": author.findtext('organization').strip(), + "country": author.findtext('address/postal/country').strip(), + }) except forms.ValidationError: raise except Exception as e: @@ -324,18 +320,12 @@ class SubmissionUploadForm(forms.Form): return None class NameEmailForm(forms.Form): - """For validating supplied submitter and author information.""" name = forms.CharField(required=True) - email = forms.EmailField(label=u'Email address') - - #Fields for secretariat only - approvals_received = forms.BooleanField(label=u'Approvals received', required=False, initial=False) + email = forms.EmailField(label=u'Email address', required=True) def __init__(self, *args, **kwargs): - email_required = kwargs.pop("email_required", True) super(NameEmailForm, self).__init__(*args, **kwargs) - self.fields["email"].required = email_required self.fields["name"].widget.attrs["class"] = "name" self.fields["email"].widget.attrs["class"] = "email" @@ -345,6 +335,18 @@ class NameEmailForm(forms.Form): def clean_email(self): return self.cleaned_data["email"].replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() +class AuthorForm(NameEmailForm): + affiliation = forms.CharField(max_length=100, required=False) + country = forms.CharField(max_length=255, required=False) + + def __init__(self, *args, **kwargs): + super(AuthorForm, self).__init__(*args, **kwargs) + self.fields["email"].required = False + +class SubmitterForm(NameEmailForm): + #Fields for secretariat only + approvals_received = forms.BooleanField(label=u'Approvals received', required=False, initial=False) + def cleaned_line(self): line = self.cleaned_data["name"] email = self.cleaned_data.get("email") @@ -376,13 +378,14 @@ class EditSubmissionForm(forms.ModelForm): rev = forms.CharField(label=u'Revision', max_length=2, required=True) document_date = forms.DateField(required=True) pages = forms.IntegerField(required=True) + formal_languages = forms.ModelMultipleChoiceField(queryset=FormalLanguageName.objects.filter(used=True), widget=forms.CheckboxSelectMultiple, required=False) abstract = forms.CharField(widget=forms.Textarea, required=True, strip=False) note = forms.CharField(label=mark_safe(u'Comment to the Secretariat'), widget=forms.Textarea, required=False, strip=False) class Meta: model = Submission - fields = ['title', 'rev', 'document_date', 'pages', 'abstract', 'note'] + fields = ['title', 'rev', 'document_date', 'pages', 'formal_languages', 'abstract', 'note'] def clean_rev(self): rev = self.cleaned_data["rev"] diff --git a/ietf/submit/migrations/0019_add_formal_languages_and_words.py b/ietf/submit/migrations/0019_add_formal_languages_and_words.py new file mode 100644 index 000000000..ec346ca87 --- /dev/null +++ b/ietf/submit/migrations/0019_add_formal_languages_and_words.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0020_add_formlang_names'), + ('submit', '0018_fix_more_bad_submission_docevents'), + ] + + operations = [ + migrations.AddField( + model_name='submission', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), + migrations.AddField( + model_name='submission', + name='words', + field=models.IntegerField(null=True, blank=True), + ), + ] diff --git a/ietf/submit/migrations/0020_authors_as_jsonfield.py b/ietf/submit/migrations/0020_authors_as_jsonfield.py new file mode 100644 index 000000000..2d8b41eca --- /dev/null +++ b/ietf/submit/migrations/0020_authors_as_jsonfield.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations +import jsonfield.fields + +def parse_email_line(line): + """Split line on the form 'Some Name '""" + import re + m = re.match("([^<]+) <([^>]+)>$", line) + if m: + return dict(name=m.group(1), email=m.group(2)) + else: + return dict(name=line, email="") + +def parse_authors(author_lines): + res = [] + for line in author_lines.replace("\r", "").split("\n"): + line = line.strip() + if line: + res.append(parse_email_line(line)) + return res + +def convert_author_lines_to_json(apps, schema_editor): + import json + + Submission = apps.get_model("submit", "Submission") + for s in Submission.objects.all().iterator(): + Submission.objects.filter(pk=s.pk).update(authors=json.dumps(parse_authors(s.authors))) + +class Migration(migrations.Migration): + + dependencies = [ + ('submit', '0019_add_formal_languages_and_words'), + ] + + operations = [ + migrations.RunPython(convert_author_lines_to_json, migrations.RunPython.noop), + migrations.AlterField( + model_name='submission', + name='authors', + field=jsonfield.fields.JSONField(default=list, help_text=b'List of authors with name, email, affiliation and country.'), + ), + ] diff --git a/ietf/submit/models.py b/ietf/submit/models.py index 8788400f4..90a6a1378 100644 --- a/ietf/submit/models.py +++ b/ietf/submit/models.py @@ -10,7 +10,7 @@ from ietf.doc.models import Document from ietf.person.models import Person from ietf.group.models import Group from ietf.message.models import Message -from ietf.name.models import DraftSubmissionStateName +from ietf.name.models import DraftSubmissionStateName, FormalLanguageName from ietf.utils.accesstoken import generate_random_key, generate_access_token @@ -36,7 +36,10 @@ class Submission(models.Model): abstract = models.TextField(blank=True) rev = models.CharField(max_length=3, blank=True) pages = models.IntegerField(null=True, blank=True) - authors = models.TextField(blank=True, help_text="List of author names and emails, one author per line, e.g. \"John Doe <john@example.org>\".") + words = models.IntegerField(null=True, blank=True) + formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document") + + authors = jsonfield.JSONField(default=list, help_text="List of authors with name, email, affiliation and country.") note = models.TextField(blank=True) replaces = models.CharField(max_length=1000, blank=True) @@ -53,20 +56,6 @@ class Submission(models.Model): def __unicode__(self): return u"%s-%s" % (self.name, self.rev) - def authors_parsed(self): - if not hasattr(self, '_cached_authors_parsed'): - from ietf.submit.utils import ensure_person_email_info_exists - res = [] - for line in self.authors.replace("\r", "").split("\n"): - line = line.strip() - if line: - parsed = parse_email_line(line) - if not parsed["email"]: - parsed["email"] = ensure_person_email_info_exists(**parsed).address - res.append(parsed) - self._cached_authors_parsed = res - return self._cached_authors_parsed - def submitter_parsed(self): return parse_email_line(self.submitter) diff --git a/ietf/submit/test_submission.txt b/ietf/submit/test_submission.txt index 97c95b3c4..b6551bf2d 100644 --- a/ietf/submit/test_submission.txt +++ b/ietf/submit/test_submission.txt @@ -61,8 +61,10 @@ Internet-Draft Testing Tests %(month)s %(year)s Table of Contents 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 - 2. Security Considerations . . . . . . . . . . . . . . . . . . . 2 - 3. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 2 + 2. Yang . . . . . . . . . . . . . . . . . . . . . . . . . . . . 2 + 3. JSON example . . . . . . . . . . . . . . . . . . . . . . . . 2 + 4. Security Considerations . . . . . . . . . . . . . . . . . . . 2 + 5. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 2 Author's Address . . . . . . . . . . . . . . . . . . . . . . . . 2 1. Introduction @@ -169,11 +171,19 @@ Table of Contents -3. Security Considerations +3. JSON example + + The JSON object should look like this: + + { + "test": 1234 + } + +4. Security Considerations There are none. -4. IANA Considerations +5. IANA Considerations No new registrations for IANA. diff --git a/ietf/submit/test_submission.xml b/ietf/submit/test_submission.xml index a313e3706..a8b6d9cde 100644 --- a/ietf/submit/test_submission.xml +++ b/ietf/submit/test_submission.xml @@ -137,6 +137,15 @@ module ietf-mpls { +
+ + The JSON object should look like this: + + { + "test": 1234 + } + +
There are none. diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 2348be203..f960c97c4 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -18,7 +18,8 @@ from ietf.group.models import Group from ietf.group.utils import setup_default_community_list_for_group from ietf.meeting.models import Meeting from ietf.message.models import Message -from ietf.person.models import Person, Email +from ietf.name.models import FormalLanguageName +from ietf.person.models import Person from ietf.person.factories import UserFactory, PersonFactory from ietf.submit.models import Submission, Preapproval from ietf.submit.mail import add_submission_email, process_response_email @@ -48,7 +49,6 @@ def submission_file(name, rev, group, format, templatename, author=None): surname=author.last_name(), email=author.email().address.lower(), ) - file = StringIO(submission_text) file.name = "%s-%s.%s" % (name, rev, format) return file @@ -118,7 +118,7 @@ class SubmitTests(TestCase): q = PyQuery(r.content) print(q('div.has-error div.alert').text()) - self.assertEqual(r.status_code, 302) + self.assertNoFormPostErrors(r, ".has-error,.alert-danger") status_url = r["Location"] for format in formats: @@ -126,10 +126,12 @@ class SubmitTests(TestCase): self.assertEqual(Submission.objects.filter(name=name).count(), 1) submission = Submission.objects.get(name=name) self.assertTrue(all([ c.passed!=False for c in submission.checks.all() ])) - self.assertEqual(len(submission.authors_parsed()), 1) - author = submission.authors_parsed()[0] + self.assertEqual(len(submission.authors), 1) + author = submission.authors[0] self.assertEqual(author["name"], "Author Name") self.assertEqual(author["email"], "author@example.com") + self.assertEqual(author["affiliation"], "Test Centre Inc.") + self.assertEqual(author["country"], "UK") return status_url @@ -184,6 +186,7 @@ class SubmitTests(TestCase): abstract="Blahblahblah.", rev="01", pages=2, + words=100, intended_std_level_id="ps", ad=draft.ad, expires=datetime.datetime.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), @@ -241,9 +244,11 @@ class SubmitTests(TestCase): self.assertEqual(draft.stream_id, "ietf") self.assertTrue(draft.expires >= datetime.datetime.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE - 1)) self.assertEqual(draft.get_state("draft-stream-%s" % draft.stream_id).slug, "wg-doc") - self.assertEqual(draft.authors.count(), 1) - self.assertEqual(draft.authors.all()[0].get_name(), "Author Name") - self.assertEqual(draft.authors.all()[0].address, "author@example.com") + authors = draft.documentauthor_set.all() + self.assertEqual(len(authors), 1) + self.assertEqual(authors[0].person.plain_name(), "Author Name") + self.assertEqual(authors[0].email.address, "author@example.com") + self.assertEqual(set(draft.formal_languages.all()), set(FormalLanguageName.objects.filter(slug="json"))) self.assertEqual(draft.relations_that_doc("replaces").count(), 1) self.assertTrue(draft.relations_that_doc("replaces").first().target, replaced_alias) self.assertEqual(draft.relations_that_doc("possibly-replaces").count(), 1) @@ -281,12 +286,12 @@ class SubmitTests(TestCase): draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="added_comment", by=Person.objects.get(user__username="secretary"), desc="Test")]) if not change_authors: draft.documentauthor_set.all().delete() - ensure_person_email_info_exists('Author Name','author@example.com') - draft.documentauthor_set.create(author=Email.objects.get(address='author@example.com')) + author_person, author_email = ensure_person_email_info_exists('Author Name','author@example.com') + draft.documentauthor_set.create(person=author_person, email=author_email) else: # Make it such that one of the previous authors has an invalid email address - bogus_email = ensure_person_email_info_exists('Bogus Person',None) - DocumentAuthor.objects.create(document=draft,author=bogus_email,order=draft.documentauthor_set.latest('order').order+1) + bogus_person, bogus_email = ensure_person_email_info_exists('Bogus Person',None) + DocumentAuthor.objects.create(document=draft, person=bogus_person, email=bogus_email, order=draft.documentauthor_set.latest('order').order+1) prev_author = draft.documentauthor_set.all()[0] @@ -334,7 +339,7 @@ class SubmitTests(TestCase): confirm_email = outbox[-1] self.assertTrue("Confirm submission" in confirm_email["Subject"]) self.assertTrue(name in confirm_email["Subject"]) - self.assertTrue(prev_author.author.address in confirm_email["To"]) + self.assertTrue(prev_author.email.address in confirm_email["To"]) if change_authors: self.assertTrue("author@example.com" not in confirm_email["To"]) self.assertTrue("submitter@example.com" not in confirm_email["To"]) @@ -415,9 +420,10 @@ class SubmitTests(TestCase): self.assertEqual(draft.stream_id, "ietf") self.assertEqual(draft.get_state_slug("draft-stream-%s" % draft.stream_id), "wg-doc") self.assertEqual(draft.get_state_slug("draft-iana-review"), "changed") - self.assertEqual(draft.authors.count(), 1) - self.assertEqual(draft.authors.all()[0].get_name(), "Author Name") - self.assertEqual(draft.authors.all()[0].address, "author@example.com") + authors = draft.documentauthor_set.all() + self.assertEqual(len(authors), 1) + self.assertEqual(authors[0].person.plain_name(), "Author Name") + self.assertEqual(authors[0].email.address, "author@example.com") self.assertEqual(len(outbox), mailbox_before + 3) self.assertTrue((u"I-D Action: %s" % name) in outbox[-3]["Subject"]) self.assertTrue((u"I-D Action: %s" % name) in draft.message_set.order_by("-time")[0].subject) @@ -654,7 +660,7 @@ class SubmitTests(TestCase): "authors-prefix": ["authors-", "authors-0", "authors-1", "authors-2"], }) - self.assertEqual(r.status_code, 302) + self.assertNoFormPostErrors(r, ".has-error,.alert-danger") submission = Submission.objects.get(name=name) self.assertEqual(submission.title, "some title") @@ -666,14 +672,14 @@ class SubmitTests(TestCase): self.assertEqual(submission.replaces, draft.docalias_set.all().first().name) self.assertEqual(submission.state_id, "manual") - authors = submission.authors_parsed() + authors = submission.authors self.assertEqual(len(authors), 3) self.assertEqual(authors[0]["name"], "Person 1") self.assertEqual(authors[0]["email"], "person1@example.com") self.assertEqual(authors[1]["name"], "Person 2") self.assertEqual(authors[1]["email"], "person2@example.com") self.assertEqual(authors[2]["name"], "Person 3") - self.assertEqual(authors[2]["email"], "unknown-email-Person-3") + self.assertEqual(authors[2]["email"], "") self.assertEqual(len(outbox), mailbox_before + 1) self.assertTrue("Manual Post Requested" in outbox[-1]["Subject"]) @@ -929,7 +935,6 @@ class SubmitTests(TestCase): files = {"txt": submission_file(name, rev, group, "txt", "test_submission.nonascii", author=author) } r = self.client.post(url, files) - self.assertEqual(r.status_code, 302) status_url = r["Location"] r = self.client.get(status_url) @@ -1433,8 +1438,8 @@ Subject: test self.assertEqual(Submission.objects.filter(name=name).count(), 1) submission = Submission.objects.get(name=name) self.assertTrue(all([ c.passed!=False for c in submission.checks.all() ])) - self.assertEqual(len(submission.authors_parsed()), 1) - author = submission.authors_parsed()[0] + self.assertEqual(len(submission.authors), 1) + author = submission.authors[0] self.assertEqual(author["name"], "Author Name") self.assertEqual(author["email"], "author@example.com") diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 62758c20a..b6cbb5bbf 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -47,7 +47,7 @@ def validate_submission(submission): if not submission.abstract: errors['abstract'] = 'Abstract is empty or was not found' - if not submission.authors_parsed(): + if not submission.authors: errors['authors'] = 'No authors found' # revision @@ -130,7 +130,7 @@ def docevent_from_submission(request, submission, desc, who=None): else: submitter_parsed = submission.submitter_parsed() if submitter_parsed["name"] and submitter_parsed["email"]: - by = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]).person + by, _ = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]) else: by = system @@ -184,7 +184,7 @@ def post_submission(request, submission, approvedDesc): system = Person.objects.get(name="(System)") submitter_parsed = submission.submitter_parsed() if submitter_parsed["name"] and submitter_parsed["email"]: - submitter = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]).person + submitter, _ = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]) submitter_info = u'%s <%s>' % (submitter_parsed["name"], submitter_parsed["email"]) else: submitter = system @@ -258,6 +258,8 @@ def post_submission(request, submission, approvedDesc): update_authors(draft, submission) + draft.formal_languages = submission.formal_languages.all() + trouble = rebuild_reference_relations(draft, filename=os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (submission.name, submission.rev))) if trouble: log.log('Rebuild_reference_relations trouble: %s'%trouble) @@ -344,10 +346,9 @@ def update_replaces_from_submission(request, submission, draft): if rdoc == draft: continue - # TODO - I think the .exists() is in the wrong place below.... if (is_secretariat or (draft.group in is_chair_of and (rdoc.group.type_id == "individ" or rdoc.group in is_chair_of)) - or (submitter_email and rdoc.authors.filter(address__iexact=submitter_email)).exists()): + or (submitter_email and rdoc.documentauthor_set.filter(email__address__iexact=submitter_email).exists())): approved.append(r) else: if r not in existing_suggested: @@ -434,24 +435,27 @@ def ensure_person_email_info_exists(name, email): email.time = datetime.datetime.now() email.save() - return email + return person, email def update_authors(draft, submission): - authors = [] - for order, author in enumerate(submission.authors_parsed()): - email = ensure_person_email_info_exists(author["name"], author["email"]) + persons = [] + for order, author in enumerate(submission.authors): + person, email = ensure_person_email_info_exists(author["name"], author.get("email")) - a = DocumentAuthor.objects.filter(document=draft, author=email).first() + a = DocumentAuthor.objects.filter(document=draft, person=person).first() if not a: - a = DocumentAuthor(document=draft, author=email) + a = DocumentAuthor(document=draft, person=person) + a.email = email + a.affiliation = author.get("affiliation") or "" + a.country = author.get("country") or "" a.order = order a.save() - log.assertion('a.author_id != "none"') + log.assertion('a.email_id != "none"') - authors.append(email) + persons.append(person) - draft.documentauthor_set.exclude(author__in=authors).delete() + draft.documentauthor_set.exclude(person__in=persons).delete() def cancel_submission(submission): submission.state = DraftSubmissionStateName.objects.get(slug="cancel") diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 99f043260..ca07dfd50 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -3,6 +3,7 @@ import base64 import datetime import os import xml2rfc +import optparse from django.conf import settings from django.contrib import messages @@ -20,7 +21,8 @@ from ietf.group.models import Group from ietf.ietfauth.utils import has_role, role_required from ietf.mailtrigger.utils import gather_address_lists from ietf.message.models import Message, MessageAttachment -from ietf.submit.forms import ( SubmissionUploadForm, NameEmailForm, EditSubmissionForm, +from ietf.name.models import FormalLanguageName +from ietf.submit.forms import ( SubmissionUploadForm, AuthorForm, SubmitterForm, EditSubmissionForm, PreapprovalForm, ReplacesForm, SubmissionEmailForm, MessageModelForm ) from ietf.submit.mail import ( send_full_url, send_approval_request_to_group, send_submission_confirmation, send_manual_post_request, add_submission_email, get_reply_to ) @@ -30,6 +32,7 @@ from ietf.submit.utils import ( approvable_submissions_for_user, preapprovals_fo recently_approved_by_user, validate_submission, create_submission_event, docevent_from_submission, post_submission, cancel_submission, rename_submission_files, get_person_from_name_email ) +from ietf.stats.utils import clean_country_name from ietf.utils.accesstoken import generate_random_key, generate_access_token from ietf.utils.draft import Draft from ietf.utils.log import log @@ -60,7 +63,7 @@ def upload_submission(request): if not form.cleaned_data['txt']: file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (form.filename, form.revision)) try: - pagedwriter = xml2rfc.PaginatedTextRfcWriter(form.xmltree, quiet=True) + pagedwriter = xml2rfc.PaginatedTextRfcWriter(form.xmltree, options=optparse.Values(defaults=dict(quiet=True, verbose=False, utf8=False))) pagedwriter.write(file_name['txt']) except Exception as e: raise ValidationError("Error from xml2rfc: %s" % e) @@ -80,10 +83,9 @@ def upload_submission(request): # If we don't have an xml file, try to extract the # relevant information from the text file for author in form.parsed_draft.get_author_list(): - full_name, first_name, middle_initial, last_name, name_suffix, email, company = author + full_name, first_name, middle_initial, last_name, name_suffix, email, country, company = author - line = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() - email = (email or "").strip() + name = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() if email: try: @@ -91,29 +93,31 @@ def upload_submission(request): except ValidationError: email = "" - if email: - # Try various ways of handling name and email, in order to avoid - # triggering a 500 error here. If the document contains non-ascii - # characters, it will be flagged later by the idnits check. - try: - line += u" <%s>" % email - except UnicodeDecodeError: + def turn_into_unicode(s): + if s is None: + return u"" + + if isinstance(s, unicode): + return s + else: try: - line = line.decode('utf-8') - email = email.decode('utf-8') - line += u" <%s>" % email + return s.decode("utf-8") except UnicodeDecodeError: try: - line = line.decode('latin-1') - email = email.decode('latin-1') - line += u" <%s>" % email + return s.decode("latin-1") except UnicodeDecodeError: - try: - line += " <%s>" % email - except UnicodeDecodeError: - pass + return "" - authors.append(line) + name = turn_into_unicode(name) + email = turn_into_unicode(email) + company = turn_into_unicode(company) + + authors.append({ + "name": name, + "email": email, + "affiliation": company, + "country": country + }) if form.abstract: abstract = form.abstract @@ -124,55 +128,39 @@ def upload_submission(request): # for this revision. # If so - we're going to update it otherwise we create a new object - submission = Submission.objects.filter(name=form.filename, - rev=form.revision, - state_id = "waiting-for-draft").distinct() - if (len(submission) == 0): - submission = None - elif (len(submission) == 1): - submission = submission[0] - - submission.state = DraftSubmissionStateName.objects.get(slug="uploaded") - submission.remote_ip=form.remote_ip - submission.title=form.title - submission.abstract=abstract - submission.rev=form.revision - submission.pages=form.parsed_draft.get_pagecount() - submission.authors="\n".join(authors) - submission.first_two_pages=''.join(form.parsed_draft.pages[:2]) - submission.file_size=file_size - submission.file_types=','.join(form.file_types) - submission.submission_date=datetime.date.today() - submission.document_date=form.parsed_draft.get_creation_date() - submission.replaces="" - - submission.save() + submissions = Submission.objects.filter(name=form.filename, + rev=form.revision, + state_id = "waiting-for-draft").distinct() + + if not submissions: + submission = Submission(name=form.filename, rev=form.revision, group=form.group) + elif len(submissions) == 1: + submission = submissions[0] else: raise Exception("Multiple submissions found waiting for upload") - if (submission == None): - try: - submission = Submission.objects.create( - state=DraftSubmissionStateName.objects.get(slug="uploaded"), - remote_ip=form.remote_ip, - name=form.filename, - group=form.group, - title=form.title, - abstract=abstract, - rev=form.revision, - pages=form.parsed_draft.get_pagecount(), - authors="\n".join(authors), - note="", - first_two_pages=''.join(form.parsed_draft.pages[:2]), - file_size=file_size, - file_types=','.join(form.file_types), - submission_date=datetime.date.today(), - document_date=form.parsed_draft.get_creation_date(), - replaces="", - ) - except Exception as e: - log("Exception: %s\n" % e) - raise + try: + submission.state = DraftSubmissionStateName.objects.get(slug="uploaded") + submission.remote_ip = form.remote_ip + submission.title = form.title + submission.abstract = abstract + submission.pages = form.parsed_draft.get_pagecount() + submission.words = form.parsed_draft.get_wordcount() + submission.authors = authors + submission.first_two_pages = ''.join(form.parsed_draft.pages[:2]) + submission.file_size = file_size + submission.file_types = ','.join(form.file_types) + submission.submission_date = datetime.date.today() + submission.document_date = form.parsed_draft.get_creation_date() + submission.replaces = "" + + submission.save() + + submission.formal_languages = FormalLanguageName.objects.filter(slug__in=form.parsed_draft.get_formal_languages()) + + except Exception as e: + log("Exception: %s\n" % e) + raise # run submission checkers def apply_check(submission, checker, method, fn): @@ -274,8 +262,8 @@ def submission_status(request, submission_id, access_token=None): group_authors_changed = False doc = submission.existing_document() if doc and doc.group: - old_authors = [ i.author.person for i in doc.documentauthor_set.all() ] - new_authors = [ get_person_from_name_email(**p) for p in submission.authors_parsed() ] + old_authors = [ author.person for author in doc.documentauthor_set.all() ] + new_authors = [ get_person_from_name_email(author["name"], author.get("email")) for author in submission.authors ] group_authors_changed = set(old_authors)!=set(new_authors) message = None @@ -290,7 +278,7 @@ def submission_status(request, submission_id, access_token=None): message = ('success', 'The submission is pending approval by the authors of the previous version. An email has been sent to: %s' % ", ".join(confirmation_list)) - submitter_form = NameEmailForm(initial=submission.submitter_parsed(), prefix="submitter") + submitter_form = SubmitterForm(initial=submission.submitter_parsed(), prefix="submitter") replaces_form = ReplacesForm(name=submission.name,initial=DocAlias.objects.filter(name__in=submission.replaces.split(","))) if request.method == 'POST': @@ -299,7 +287,7 @@ def submission_status(request, submission_id, access_token=None): if not can_edit: return HttpResponseForbidden("You do not have permission to perform this action") - submitter_form = NameEmailForm(request.POST, prefix="submitter") + submitter_form = SubmitterForm(request.POST, prefix="submitter") replaces_form = ReplacesForm(request.POST, name=submission.name) validations = [submitter_form.is_valid(), replaces_form.is_valid()] if all(validations): @@ -415,6 +403,9 @@ def submission_status(request, submission_id, access_token=None): # something went wrong, turn this into a GET and let the user deal with it return HttpResponseRedirect("") + for author in submission.authors: + author["cleaned_country"] = clean_country_name(author.get("country")) + return render(request, 'submit/submission_status.html', { 'selected': 'status', 'submission': submission, @@ -447,7 +438,7 @@ def edit_submission(request, submission_id, access_token=None): # submission itself, one for the submitter, and a list of forms # for the authors - empty_author_form = NameEmailForm(email_required=False) + empty_author_form = AuthorForm() if request.method == 'POST': # get a backup submission now, the model form may change some @@ -455,21 +446,23 @@ def edit_submission(request, submission_id, access_token=None): prev_submission = Submission.objects.get(pk=submission.pk) edit_form = EditSubmissionForm(request.POST, instance=submission, prefix="edit") - submitter_form = NameEmailForm(request.POST, prefix="submitter") + submitter_form = SubmitterForm(request.POST, prefix="submitter") replaces_form = ReplacesForm(request.POST,name=submission.name) - author_forms = [ NameEmailForm(request.POST, email_required=False, prefix=prefix) + author_forms = [ AuthorForm(request.POST, prefix=prefix) for prefix in request.POST.getlist("authors-prefix") if prefix != "authors-" ] # trigger validation of all forms validations = [edit_form.is_valid(), submitter_form.is_valid(), replaces_form.is_valid()] + [ f.is_valid() for f in author_forms ] if all(validations): + changed_fields = [] + submission.submitter = submitter_form.cleaned_line() replaces = replaces_form.cleaned_data.get("replaces", []) submission.replaces = ",".join(o.name for o in replaces) - submission.authors = "\n".join(f.cleaned_line() for f in author_forms) - if hasattr(submission, '_cached_authors_parsed'): - del submission._cached_authors_parsed + submission.authors = [ { attr: f.cleaned_data.get(attr) or "" + for attr in ["name", "email", "affiliation", "country"] } + for f in author_forms ] edit_form.save(commit=False) # transfer changes if submission.rev != prev_submission.rev: @@ -478,12 +471,18 @@ def edit_submission(request, submission_id, access_token=None): submission.state = DraftSubmissionStateName.objects.get(slug="manual") submission.save() + formal_languages_changed = False + if set(submission.formal_languages.all()) != set(edit_form.cleaned_data["formal_languages"]): + submission.formal_languages = edit_form.cleaned_data["formal_languages"] + formal_languages_changed = True + send_manual_post_request(request, submission, errors) - changed_fields = [ + changed_fields += [ submission._meta.get_field(f).verbose_name for f in list(edit_form.fields.keys()) + ["submitter", "authors"] - if getattr(submission, f) != getattr(prev_submission, f) + if (f == "formal_languages" and formal_languages_changed) + or getattr(submission, f) != getattr(prev_submission, f) ] if changed_fields: @@ -498,10 +497,10 @@ def edit_submission(request, submission_id, access_token=None): form_errors = True else: edit_form = EditSubmissionForm(instance=submission, prefix="edit") - submitter_form = NameEmailForm(initial=submission.submitter_parsed(), prefix="submitter") + submitter_form = SubmitterForm(initial=submission.submitter_parsed(), prefix="submitter") replaces_form = ReplacesForm(name=submission.name,initial=DocAlias.objects.filter(name__in=submission.replaces.split(","))) - author_forms = [ NameEmailForm(initial=author, email_required=False, prefix="authors-%s" % i) - for i, author in enumerate(submission.authors_parsed()) ] + author_forms = [ AuthorForm(initial=author, prefix="authors-%s" % i) + for i, author in enumerate(submission.authors) ] return render(request, 'submit/edit_submission.html', {'selected': 'status', diff --git a/ietf/templates/doc/bibxml.xml b/ietf/templates/doc/bibxml.xml index 9813c3a00..fdb57c132 100644 --- a/ietf/templates/doc/bibxml.xml +++ b/ietf/templates/doc/bibxml.xml @@ -2,11 +2,11 @@ {{doc.title}} - {% for entry in doc.authors.all %}{% with entry.address as email %}{% with entry.person as author %} - - {{author.affiliation}} + {% for author in doc.documentauthor_set.all %} + + {{ author.affiliation }} - {% endwith %}{% endwith %}{% endfor %} + {% endfor %} {{doc.abstract}} diff --git a/ietf/templates/doc/document_bibtex.bib b/ietf/templates/doc/document_bibtex.bib index d316b9ef9..0aeb06522 100644 --- a/ietf/templates/doc/document_bibtex.bib +++ b/ietf/templates/doc/document_bibtex.bib @@ -24,7 +24,7 @@ publisher = {% templatetag openbrace %}Internet Engineering Task Force{% templatetag closebrace %}, note = {% templatetag openbrace %}Work in Progress{% templatetag closebrace %}, url = {% templatetag openbrace %}https://datatracker.ietf.org/doc/html/{{doc.name}}-{{doc.rev}}{% templatetag closebrace %},{% endif %} - author = {% templatetag openbrace %}{% for entry in doc.authors.all %}{% with entry.person as author %}{{author.name}}{% endwith %}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, + author = {% templatetag openbrace %}{% for author in doc.documentauthor_set.all %}{{ author.person.name}}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, title = {% templatetag openbrace %}{% templatetag openbrace %}{{doc.title}}{% templatetag closebrace %}{% templatetag closebrace %}, pagetotal = {{ doc.pages }}, year = {{ doc.pub_date.year }}, diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index 7a8caba78..870e188a5 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -572,13 +572,13 @@

Authors

{% for author in doc.documentauthor_set.all %} - {% if not author.author.invalid_address %} + {% if author.email %} - + {% endif %} - {{ author.author.person }} - {% if not author.author.invalid_address %} - ({{ author.author.address }}) + {{ author.person }} + {% if author.email %} + ({{ author.email.address }}) {% endif %} {% if not forloop.last %}
{% endif %} {% endfor %} diff --git a/ietf/templates/doc/document_referenced_by.html b/ietf/templates/doc/document_referenced_by.html index 1ce3bdc89..740d68f7b 100644 --- a/ietf/templates/doc/document_referenced_by.html +++ b/ietf/templates/doc/document_referenced_by.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin staticfiles %} +{% load origin staticfiles ietf_filters %} {% block pagehead %} @@ -44,8 +44,8 @@ {% with ref.source.canonical_name as name %} - {{name}} - {% if ref.target.name != alias_name %} + {{ name|prettystdname }} + {% if ref.target.name != alias_name %}
As {{ref.target.name}} {% endif %} diff --git a/ietf/templates/doc/document_references.html b/ietf/templates/doc/document_references.html index b63801d9a..e8c9a7c03 100644 --- a/ietf/templates/doc/document_references.html +++ b/ietf/templates/doc/document_references.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin staticfiles %} +{% load origin staticfiles ietf_filters %} {% block pagehead %} @@ -35,7 +35,7 @@ {% for ref in refs %} {% with ref.target.name as name %} - {{name}} + {{ name|prettystdname }} {{ref.target.document.title}}
Refs diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html new file mode 100644 index 000000000..a3819d9e5 --- /dev/null +++ b/ietf/templates/stats/document_stats.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} + +{% load origin %} + +{% load ietf_filters staticfiles bootstrap3 %} + +{% block title %}{{ stats_title }}{% endblock %} + +{% block pagehead %} + +{% endblock %} + +{% block content %} + {% origin %} + +

Draft/RFC statistics

+ +
+
+ Documents: + +
+ {% for slug, label, url in possible_document_stats_types %} + {{ label }} + {% endfor %} +
+
+ +
+ Authors: + +
+ {% for slug, label, url in possible_author_stats_types %} + {{ label }} + {% endfor %} +
+
+ +
+ Yearly: + +
+ {% for slug, label, url in possible_yearly_stats_types %} + {{ label }} + {% endfor %} +
+
+ +
Options
+ +
+ Document type: +
+ {% for slug, label, url in possible_document_types %} + {{ label }} + {% endfor %} +
+ + Time: +
+ {% for slug, label, url in possible_time_choices %} + {{ label }} + {% endfor %} +
+
+
+ +
+ {% include content_template %} +
+{% endblock %} + +{% block js %} + + + + +{% endblock %} diff --git a/ietf/templates/stats/document_stats_author_affiliation.html b/ietf/templates/stats/document_stats_author_affiliation.html new file mode 100644 index 000000000..8c6df8cce --- /dev/null +++ b/ietf/templates/stats/document_stats_author_affiliation.html @@ -0,0 +1,107 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for affiliation, percentage, names in table_data %} + + + + + + {% endfor %} + +
AffiliationPercentage of authorsAuthors
{{ affiliation|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
+ +

The statistics are based entirely on the author affiliation + provided with each draft. Since this may vary across documents, an + author may be counted with more than one affiliation, making the + total sum more than 100%.

+ + + +

Affiliation Aliases

+ +

In generating the above statistics, some heuristics have been + applied to determine the affiliations of each author.

+ +{% if request.GET.showaliases %} +

Hide generated aliases

+ + {% if request.user.is_staff %} +

Note: since you're an admin, you can add an extra known alias or see the existing known aliases and generally ignored endings.

+ {% endif %} + + {% if alias_data %} + + + + + + + {% for name, alias in alias_data %} + + + + + {% endfor %} +
AffiliationAlias
+ {% ifchanged %} + {{ name|default:"(unknown)" }} + {% endifchanged %} + {{ alias }}
+ {% endif %} + +{% else %} +

Show generated aliases

+{% endif %} diff --git a/ietf/templates/stats/document_stats_author_citations.html b/ietf/templates/stats/document_stats_author_citations.html new file mode 100644 index 000000000..25c41a978 --- /dev/null +++ b/ietf/templates/stats/document_stats_author_citations.html @@ -0,0 +1,71 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for citations, percentage, names in table_data %} + + + + + + {% endfor %} + +
CitationsPercentage of authorsAuthors
{{ citations }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}
+ +

Note that the citation counts do not exclude self-references.

diff --git a/ietf/templates/stats/document_stats_author_continent.html b/ietf/templates/stats/document_stats_author_continent.html new file mode 100644 index 000000000..b35334f95 --- /dev/null +++ b/ietf/templates/stats/document_stats_author_continent.html @@ -0,0 +1,69 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for continent, percentage, names in table_data %} + + + + + + {% endfor %} + +
ContinentPercentage of authorsAuthors
{{ continent|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
+ +

The statistics are based entirely on the author addresses provided + with each draft. Since this varies across documents, a travelling + author may be counted in more than country, making the total sum + more than 100%.

diff --git a/ietf/templates/stats/document_stats_author_country.html b/ietf/templates/stats/document_stats_author_country.html new file mode 100644 index 000000000..ca01636ad --- /dev/null +++ b/ietf/templates/stats/document_stats_author_country.html @@ -0,0 +1,130 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for country, percentage, names in table_data %} + + + + + + {% endfor %} + +
CountryPercentage of authorsAuthors
{{ country|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
+ +

The statistics are based entirely on the author addresses provided + with each draft. Since this varies across documents, a travelling + author may be counted in more than country, making the total sum + more than 100%.

+ +

In case no country information is found for an author in the time + period, the author is counted as (unknown).

+ +

EU (European Union) is not a country, but has been added for reference, as the sum of + all current EU member countries: + {% for c in eu_countries %}{{ c.name }}{% if not forloop.last %}, {% endif %}{% endfor %}.

+ +

Country Aliases

+ +

In generating the above statistics, some heuristics have been + applied to figure out which country each author is from.

+ +{% if request.GET.showaliases %} +

Hide generated aliases

+ + {% if request.user.is_staff %} +

Note: since you're an admin, some extra links are visible. You + can either correct a document author entry directly in case the + information is obviously missing or add an alias if an unknown + country name + is being used. +

+ + {% endif %} + + {% if alias_data %} + + + + + + + + {% for name, alias, country in alias_data %} + + + + + + {% endfor %} +
CountryAlias
+ {% ifchanged %} + {% if country and request.user.is_staff %} + + {% endif %} + {{ name|default:"(unknown)" }} + {% if country and request.user.is_staff %} + + {% endif %} + {% endifchanged %} + {{ alias }} + {% if request.user.is_staff and name != "EU" %} + Matching authors + {% endif %} +
+ {% endif %} + +{% else %} +

Show generated aliases

+{% endif %} diff --git a/ietf/templates/stats/document_stats_author_documents.html b/ietf/templates/stats/document_stats_author_documents.html new file mode 100644 index 000000000..30d8882a4 --- /dev/null +++ b/ietf/templates/stats/document_stats_author_documents.html @@ -0,0 +1,70 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for document_count, percentage, names in table_data %} + + + + + + {% endfor %} + +
DocumentsPercentage of authorsAuthors
{{ document_count }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}
diff --git a/ietf/templates/stats/document_stats_author_hindex.html b/ietf/templates/stats/document_stats_author_hindex.html new file mode 100644 index 000000000..e8ba232e6 --- /dev/null +++ b/ietf/templates/stats/document_stats_author_hindex.html @@ -0,0 +1,79 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for h_index, percentage, names in table_data %} + + + + + + {% endfor %} + +
h-indexPercentage of authorsAuthors
{{ h_index }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=25 %}
+ +

Hirsch index or h-index is a + measure of the + productivity and impact of the publications of an author. An + author with an h-index of 5 has had 5 publications each cited at + least 5 times - to increase the index to 6, the 5 publications plus + 1 more would have to have been cited at least 6 times, each. Thus a + high h-index requires many highly-cited publications.

+ +

Note that the h-index calculations do not exclude self-references.

diff --git a/ietf/templates/stats/document_stats_authors.html b/ietf/templates/stats/document_stats_authors.html new file mode 100644 index 000000000..abfc5ff96 --- /dev/null +++ b/ietf/templates/stats/document_stats_authors.html @@ -0,0 +1,69 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for author_count, percentage, names in table_data %} + + + + + + {% endfor %} + +
AuthorsPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ author_count }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_format.html b/ietf/templates/stats/document_stats_format.html new file mode 100644 index 000000000..c7f42f7ae --- /dev/null +++ b/ietf/templates/stats/document_stats_format.html @@ -0,0 +1,65 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for pages, percentage, names in table_data %} + + + + + + {% endfor %} + +
FormatPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_formlang.html b/ietf/templates/stats/document_stats_formlang.html new file mode 100644 index 000000000..7c9470f8c --- /dev/null +++ b/ietf/templates/stats/document_stats_formlang.html @@ -0,0 +1,65 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for formal_language, percentage, names in table_data %} + + + + + + {% endfor %} + +
Formal languagePercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ formal_language }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_pages.html b/ietf/templates/stats/document_stats_pages.html new file mode 100644 index 000000000..40cc55fa9 --- /dev/null +++ b/ietf/templates/stats/document_stats_pages.html @@ -0,0 +1,63 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for pages, percentage, names in table_data %} + + + + + + {% endfor %} + +
PagesPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_words.html b/ietf/templates/stats/document_stats_words.html new file mode 100644 index 000000000..96bcb0e75 --- /dev/null +++ b/ietf/templates/stats/document_stats_words.html @@ -0,0 +1,63 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for pages, percentage, names in table_data %} + + + + + + {% endfor %} + +
WordsPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_yearly.html b/ietf/templates/stats/document_stats_yearly.html new file mode 100644 index 000000000..b77988448 --- /dev/null +++ b/ietf/templates/stats/document_stats_yearly.html @@ -0,0 +1,53 @@ +

{{ stats_title }}

+ +
+ + diff --git a/ietf/templates/stats/includes/number_with_details_cell.html b/ietf/templates/stats/includes/number_with_details_cell.html new file mode 100644 index 000000000..2d4f0d0fb --- /dev/null +++ b/ietf/templates/stats/includes/number_with_details_cell.html @@ -0,0 +1,8 @@ +{% if content_limit and names|length <= content_limit %} + {% for n in names %} + {{ n }}
+ {% endfor %} +{% else %} + {# {{ names|length }} #} + {{ names|length }} +{% endif %} diff --git a/ietf/templates/stats/index.html b/ietf/templates/stats/index.html index 11b9bb8e5..a0259f7ee 100644 --- a/ietf/templates/stats/index.html +++ b/ietf/templates/stats/index.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% load origin %}{% origin %} +{% load origin %} {% load ietf_filters staticfiles bootstrap3 %} @@ -9,10 +9,12 @@

{% block title %}Statistics{% endblock %}

-

Currently, there are statistics for:

+

Statistics on...

{% endblock %} diff --git a/ietf/templates/stats/known_countries_list.html b/ietf/templates/stats/known_countries_list.html new file mode 100644 index 000000000..37f4fa8cb --- /dev/null +++ b/ietf/templates/stats/known_countries_list.html @@ -0,0 +1,47 @@ +{% extends "base.html" %} + +{% load origin %} + +{% load ietf_filters staticfiles bootstrap3 %} + +{% block content %} + {% origin %} + +

{% block title %}Countries known to the Datatracker{% endblock %}

+ +

In case you think a country or an alias is missing from the list, you can file a ticket.

+ + {% if request.user.is_staff %} +

Note: since you're an admin, the country names are linked to their corresponding admin page.

+ {% endif %} + + + + + + + + + + {% for c in countries %} + + + + + {% endfor %} + +
NameAliases (lowercase aliases are matched case-insensitive)
+ {% if request.user.is_staff %} + + {% endif %} + {{ c.name }} + {% if request.user.is_staff %} + + {% endif %} + + {% for a in c.aliases %} + {{ a.alias }}{% if not forloop.last %},{% endif %} + {% endfor %} +
+ +{% endblock %} diff --git a/ietf/templates/stats/meeting_stats.html b/ietf/templates/stats/meeting_stats.html new file mode 100644 index 000000000..1c359215f --- /dev/null +++ b/ietf/templates/stats/meeting_stats.html @@ -0,0 +1,46 @@ +{% extends "base.html" %} + +{% load origin %} + +{% load ietf_filters staticfiles bootstrap3 %} + +{% block title %}{{ stats_title }}{% endblock %} + +{% block pagehead %} + +{% endblock %} + +{% block content %} + {% origin %} + +

Meeting Statistics

+ + {% if meeting %} +

+ « Back to overview +

+ {% endif %} + +
+
+ Registrations: + +
+ {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
+
+
+ +
+ {% include content_template %} +
+{% endblock %} + +{% block js %} + + + + +{% endblock %} diff --git a/ietf/templates/stats/meeting_stats_continent.html b/ietf/templates/stats/meeting_stats_continent.html new file mode 100644 index 000000000..436507f8b --- /dev/null +++ b/ietf/templates/stats/meeting_stats_continent.html @@ -0,0 +1,64 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for continent, percentage, names in table_data %} + + + + + + {% endfor %} + +
ContinentPercentage of registrationsRegistrations
{{ continent|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/meeting_stats_country.html b/ietf/templates/stats/meeting_stats_country.html new file mode 100644 index 000000000..dc42e0086 --- /dev/null +++ b/ietf/templates/stats/meeting_stats_country.html @@ -0,0 +1,99 @@ +

{{ stats_title }}

+ +
+ + + +
+ + + +

Data

+ + + + + + + + + + + {% for country, percentage, names in table_data %} + + + + + + {% endfor %} + +
CountryPercentage of registrationsRegistrations
{{ country|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
+ +

EU (European Union) is not a country, but has been added for reference, as the sum of + all current EU member countries: + {% for c in eu_countries %}{{ c.name }}{% if not forloop.last %}, {% endif %}{% endfor %}. +

diff --git a/ietf/templates/stats/meeting_stats_overview.html b/ietf/templates/stats/meeting_stats_overview.html new file mode 100644 index 000000000..0a44da90b --- /dev/null +++ b/ietf/templates/stats/meeting_stats_overview.html @@ -0,0 +1,77 @@ +

{{ stats_title }}

+ +
+ + + +{% if table_data %} +

Data

+ + + + + + + + + + {% for meeting_number, label, url, names in table_data %} + + + + + {% endfor %} + +
MeetingRegistrations
{{ label }}{% include "stats/includes/number_with_details_cell.html" %}
+ +{% endif %} diff --git a/ietf/templates/submit/announce_to_lists.txt b/ietf/templates/submit/announce_to_lists.txt index 7ddcad55f..59c00c32f 100644 --- a/ietf/templates/submit/announce_to_lists.txt +++ b/ietf/templates/submit/announce_to_lists.txt @@ -3,7 +3,7 @@ A New Internet-Draft is available from the on-line Internet-Drafts directories. {% if submission.group %}This draft is a work item of the {{ submission.group.name }}{% if group.type.name %} {{ group.type.name }}{% endif %} of the {% if group.type_id == "rg" %}IRTF{% else %}IETF{% endif %}.{% endif %} Title : {{ submission.title }} - Author{{ submission.authors_parsed|pluralize:" ,s" }} : {% for author in submission.authors_parsed %}{{ author.name }}{% if not forloop.last %} + Author{{ submission.authors|pluralize:" ,s" }} : {% for author in submission.authors %}{{ author.name }}{% if not forloop.last %} {% endif %}{% endfor %} Filename : {{ submission.name }}-{{ submission.rev }}.txt Pages : {{ submission.pages }} diff --git a/ietf/templates/submit/approval_request.txt b/ietf/templates/submit/approval_request.txt index 83677d166..828b14bc4 100644 --- a/ietf/templates/submit/approval_request.txt +++ b/ietf/templates/submit/approval_request.txt @@ -22,7 +22,7 @@ To approve the draft, go to this URL (note: you need to login to be able to appr Authors: -{% for author in submission.authors_parsed %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} +{% for author in submission.authors %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} {% endfor %} {% endautoescape %} diff --git a/ietf/templates/submit/manual_post_request.txt b/ietf/templates/submit/manual_post_request.txt index b4e595ab2..78fd563dc 100644 --- a/ietf/templates/submit/manual_post_request.txt +++ b/ietf/templates/submit/manual_post_request.txt @@ -31,7 +31,7 @@ I-D Submission Tool URL: Authors: -{% for author in submission.authors_parsed %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} +{% for author in submission.authors %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} {% endfor %} Comment to the secretariat: diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index 147947c9a..473829130 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -48,9 +48,7 @@ {% if check.errors %}

The {{check.checker}} returned {{ check.errors }} error{{ check.errors|pluralize }} - and {{ check.warnings }} warning - - {{ check.warnings|pluralize }} ; click the button + and {{ check.warnings }} warning{{ check.warnings|pluralize }}; click the button below to see details. Please fix those, and resubmit.

{% elif check.warnings %} @@ -197,17 +195,38 @@ Authors - {% with submission.authors_parsed as authors %} - {{ authors|length }} author{{ authors|pluralize }} - {% endwith %} + {{ submission.authors|length }} author{{ submission.authors|pluralize }} {% if errors.authors %}

{{ errors.authors|safe }}

{% endif %} - {% for author in submission.authors_parsed %} + {% for author in submission.authors %} Author {{ forloop.counter }} - {{ author.name }} {% if author.email %}<{{ author.email }}>{% endif %} + + {{ author.name }} {% if author.email %}<{{ author.email }}>{% endif %} + - + {% if author.affiliation %} + {{ author.affiliation }} + {% else %} + unknown affiliation + {% endif %} + - + {% if author.country %} + {{ author.country }} + {% if author.cleaned_country and author.country != author.cleaned_country %} + (understood to be {{ author.cleaned_country }}) + {% endif %} + {% else %} + unknown country + {% endif %} + + {% if author.country and not author.cleaned_country %} +
+ Unrecognized country: "{{ author.country }}": See recognized country names. + {% endif %} + + {% endfor %} @@ -231,6 +250,14 @@ File size {{ submission.file_size|filesizeformat }} + + + Formal languages used + + {% for l in submission.formal_languages.all %}{{ l.name }}{% if not forloop.last %}, {% endif %}{% empty %}None recognized{% endfor %} + {% if errors.formal_languages %}

{{ errors.formal_languages }}

{% endif %} + + {% if can_edit %} @@ -311,7 +338,7 @@ {% if user|has_role:"Secretariat" %}

- Send Email + Send Email {% endif %} {% if show_send_full_url %} diff --git a/ietf/templates/submit/submitter_form.html b/ietf/templates/submit/submitter_form.html index 93994c10c..1564dd3e5 100644 --- a/ietf/templates/submit/submitter_form.html +++ b/ietf/templates/submit/submitter_form.html @@ -11,8 +11,8 @@ {% load ietf_filters %} {% buttons %} - {% for author in submission.authors_parsed %} - + {% for author in submission.authors %} + {% endfor %} {% endbuttons %} diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index 14a67f8dc..e605c7018 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -263,8 +263,8 @@ class Draft(): sentence = True if re.search("[^ \t]", line): if newpage: - # 33 is a somewhat arbitrary count for a 'short' line - shortthis = len(line.strip()) < 33 # 33 is a somewhat arbitrary count for a 'short' line + # 36 is a somewhat arbitrary count for a 'short' line + shortthis = len(line.strip()) < 36 # 36 is a somewhat arbitrary count for a 'short' line if sentence or (shortprev and not shortthis): stripped += [""] else: @@ -273,7 +273,7 @@ class Draft(): blankcount = 0 sentence = False newpage = False - shortprev = len(line.strip()) < 33 # 33 is a somewhat arbitrary count for a 'short' line + shortprev = len(line.strip()) < 36 # 36 is a somewhat arbitrary count for a 'short' line if re.search("[.:]$", line): sentence = True if re.search("^[ \t]*$", line): @@ -296,6 +296,37 @@ class Draft(): self._pagecount = count_pages return self._pagecount + # ------------------------------------------------------------------ + def get_wordcount(self): + count = 0 + # match any sequence of non-white-space characters like the Unix command "wc" + word_re = re.compile(r'\S+', re.UNICODE) + for l in self.lines: + count += sum(1 for _ in word_re.finditer(l)) + return count + + # ------------------------------------------------------------------ + def get_formal_languages(self): + language_regexps = [ + ("abnf", [re.compile(r"\bABNF"), re.compile(r" +[a-zA-Z][a-zA-Z0-9_-]* +=[/ ]")]), + ("asn1", [re.compile(r'DEFINITIONS +::= +BEGIN')]), + ("cbor", [re.compile(r'\b(?:CBOR|CDDL)\b'), re.compile(r" +[a-zA-Z][a-zA-Z0-9_-]* += +[\{\[\(]")]), + ("ccode", [re.compile(r"(?:\+\+\))|(?:for \(i)|(?: [!=]= 0\) \{)|(?: struct [a-zA-Z_0-9]+ \{)")]), + ("json", [re.compile(r'\bJSON\b'), re.compile(r" \"[^\"]+\" ?: [a-zA-Z0-9\.\"\{\[]")]), + ("xml", [re.compile(r"<\?xml")]), + ] + already_matched = set() + for l in self.lines: + for lang_name, patterns in language_regexps: + for p in patterns: + if p not in already_matched and p.search(l): + already_matched.add(p) + return [ + lang_name + for lang_name, patterns in language_regexps + if all(p in already_matched for p in patterns) + ] + # ---------------------------------------------------------------------- def get_status(self): if self._status == None: @@ -825,7 +856,8 @@ class Draft(): nonblank_count = 0 blanklines = 0 email = None - for line in self.lines[start+1:]: + country = None + for line_offset, line in enumerate(self.lines[start+1:]): _debug( " " + line.strip()) # Break on the second blank line if not line: @@ -866,15 +898,18 @@ class Draft(): else: pass - try: - column = line[beg:end].strip() - except: - column = line - column = re.sub(" *\(at\) *", "@", column) - column = re.sub(" *\(dot\) *", ".", column) - column = re.sub(" +at +", "@", column) - column = re.sub(" +dot +", ".", column) - column = re.sub("&cisco.com", "@cisco.com", column) + def columnify(l): + try: + column = l.replace('\t', 8 * ' ')[max(0, beg - 1):end].strip() + except: + column = l + column = re.sub(" *(?:\(at\)| | at ) *", "@", column) + column = re.sub(" *(?:\(dot\)| | dot ) *", ".", column) + column = re.sub("&cisco.com", "@cisco.com", column) + column = column.replace("\xa0", " ") + return column + + column = columnify(line) # if re.search("^\w+: \w+", column): # keyword = True @@ -885,13 +920,42 @@ class Draft(): # break #_debug( " Column text :: " + column) + if nonblank_count >= 2 and blanklines == 0: + # Usually, the contact info lines will look + # like this: "Email: someone@example.com" or + # "Tel: +1 (412)-2390 23123", but sometimes + # the : is left out. That's okay for things we + # can't misinterpret, but "tel" may match "Tel + # Aviv 69710, Israel" so match + # - misc contact info + # - tel/fax [number] + # - [phone number] + # - [email] + + other_contact_info_regex = re.compile(r'^(((contact )?e|\(e|e-|m|electronic )?mail|email_id|mailto|e-main|(tele)?phone|voice|mobile|work|uri|url|tel:)\b|^((ph|tel\.?|telefax|fax) *[:.]? *\(?( ?\+ ?)?[0-9]+)|^(\++[0-9]+|\(\+*[0-9]+\)|\(dsn\)|[0-9]+)([ -.]*\b|\b[ -.]*)(([0-9]{2,}|\([0-9]{2,}\)|(\([0-9]\)|[0-9])[ -][0-9]{2,}|\([0-9]\)[0-9]+)([ -.]+([0-9]+|\([0-9]+\)))+|([0-9]{7,}|\([0-9]{7,}\)))|^(?|^https?://|^www\.') + next_line_index = start + 1 + line_offset + 1 + + if (not country + and not other_contact_info_regex.search(column.lower()) + and next_line_index < len(self.lines)): + + next_line_lower = columnify(self.lines[next_line_index]).lower().strip() + + if not next_line_lower or other_contact_info_regex.search(next_line_lower): + # country should be here, as the last + # part of the address, right before an + # empty line or other contact info + country = column.strip() or None + _debug(" Country: %s" % country) + _debug("3: authors[%s]: %s" % (i, authors[i])) emailmatch = re.search("[-A-Za-z0-9_.+]+@[-A-Za-z0-9_.]+", column) if emailmatch and not "@" in author: email = emailmatch.group(0).lower() break - authors[i] = authors[i] + ( email, ) + + authors[i] = authors[i] + ( email, country) else: if not author in ignore: companies[i] = authors[i] @@ -917,8 +981,8 @@ class Draft(): _debug(" * Final company list: %s" % (companies,)) _debug(" * Final companies_seen: %s" % (companies_seen,)) self._author_info = authors - self._authors_with_firm = [ "%s <%s> (%s)"%(full,email,company) for full,first,middle,last,suffix,email,company in authors ] # pyflakes:ignore - self._authors = [ "%s <%s>"%(full,email) if email else full for full,first,middle,last,suffix,email,company in authors ] + self._authors_with_firm = [ "%s <%s> (%s)"%(full,email,company) for full,first,middle,last,suffix,email,country,company in authors ] # pyflakes:ignore + self._authors = [ "%s <%s>"%(full,email) if email else full for full,first,middle,last,suffix,email,country,company in authors ] self._authors.sort() _debug(" * Final author list: " + ", ".join(self._authors)) _debug("-"*72) @@ -1138,10 +1202,10 @@ def getmeta(fn): def _output(docname, fields, outfile=sys.stdout): global company_domain if opt_getauthors: - # Output an (incomplete!) getauthors-compatible format. Country - # information is always UNKNOWN, and information about security and - # iana sections presence is missing. - for full,first,middle,last,suffix,email,company in fields["_authorlist"]: + # Output an (incomplete!) getauthors-compatible format. + # Information about security and iana sections presence is + # missing. + for full,first,middle,last,suffix,email,country,company in fields["_authorlist"]: if company in company_domain: company = company_domain[company] else: @@ -1152,7 +1216,7 @@ def _output(docname, fields, outfile=sys.stdout): fields["name"] = full fields["email"] = email fields["company"] = company - fields["country"] = "UNKNOWN" + fields["country"] = country or "UNKNOWN" try: year, month, day = fields["doccreationdate"].split("-") except ValueError: diff --git a/ietf/utils/test_data.py b/ietf/utils/test_data.py index 836231e85..282478a52 100644 --- a/ietf/utils/test_data.py +++ b/ietf/utils/test_data.py @@ -95,7 +95,7 @@ def make_immutable_base_data(): # one area area = create_group(name="Far Future", acronym="farfut", type_id="area", parent=ietf) - create_person(area, "ad", name="Areað Irector", username="ad", email_address="aread@ietf.org") + create_person(area, "ad", name=u"Areað Irector", username="ad", email_address="aread@ietf.org") # second area opsarea = create_group(name="Operations", acronym="ops", type_id="area", parent=ietf) @@ -297,7 +297,10 @@ def make_test_data(): DocumentAuthor.objects.create( document=draft, - author=Email.objects.get(address="aread@ietf.org"), + person=Person.objects.get(email__address="aread@ietf.org"), + email=Email.objects.get(address="aread@ietf.org"), + country="Germany", + affiliation="IETF", order=1 ) @@ -444,8 +447,11 @@ def make_downref_test_data(): DocumentAuthor.objects.create( document=draft, - author=Email.objects.get(address="aread@ietf.org"), - order=1, + person=Person.objects.get(email__address="aread@ietf.org"), + email=Email.objects.get(address="aread@ietf.org"), + country="US", + affiliation="", + order=1 ) rfc_doc_alias = DocAlias.objects.get(name='rfc9998') diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 2970d0a4c..0afa4ad09 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -265,11 +265,11 @@ class CoverageTest(TestCase): if self.runner.run_full_test_suite: # Permit 0.02% variation in results -- otherwise small code changes become a pain fudge_factor = 0.0002 + self.assertLessEqual(len(test_missing), len(master_missing), + msg = "New %s without test coverage since %s: %s" % (test, latest_coverage_version, list(set(test_missing) - set(master_missing)))) self.assertGreaterEqual(test_coverage, master_coverage - fudge_factor, msg = "The %s coverage percentage is now lower (%.2f%%) than for version %s (%.2f%%)" % ( test, test_coverage*100, latest_coverage_version, master_coverage*100, )) - self.assertLessEqual(len(test_missing), len(master_missing), - msg = "New %s without test coverage since %s: %s" % (test, latest_coverage_version, list(set(test_missing) - set(master_missing)))) def template_coverage_test(self): global loaded_templates diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index 0e7b1973d..6e240f763 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -69,7 +69,7 @@ def unicontent(r): def reload_db_objects(*objects): """Rerequest the given arguments from the database so they're refreshed, to be used like - foo, bar = reload_objects(foo, bar)""" + foo, bar = reload_db_objects(foo, bar)""" t = tuple(o.__class__.objects.get(pk=o.pk) for o in objects) if len(objects) == 1: @@ -108,6 +108,28 @@ class TestCase(django.test.TestCase): os.mkdir(path) return path + def assertNoFormPostErrors(self, response, error_css_selector=".has-error"): + """Try to fish out form errors, if none found at least check the + status code to be a redirect. + + Assumptions: + - a POST is followed by a 302 redirect + - form errors can be found with a simple CSS selector + + """ + + if response.status_code == 200: + from pyquery import PyQuery + from lxml import html + self.maxDiff = None + + errors = [html.tostring(n).decode() for n in PyQuery(response.content)(error_css_selector)] + if errors: + explanation = u"{} != {}\nGot form back with errors:\n----\n".format(response.status_code, 302) + u"----\n".join(errors) + self.assertEqual(response.status_code, 302, explanation) + + self.assertEqual(response.status_code, 302) + def __str__(self): return "%s (%s.%s)" % (self._testMethodName, strclass(self.__class__),self._testMethodName) diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 66fe27787..c66ea849d 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -168,7 +168,7 @@ class TemplateChecksTestCase(TestCase): Check that an URLNode's callback is in callbacks. """ cb = node.view_name.token.strip("\"'") - if cb in callbacks: + if cb in callbacks or cb.startswith("admin:"): return [] else: return [ (origin, cb), ] diff --git a/requirements.txt b/requirements.txt index 391cc9e9b..73af8cd1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,7 +47,7 @@ tqdm>=3.7.0 Trac>=1.0.10,<1.2 Unidecode>=0.4.18 #wsgiref>=0.1.2 -xml2rfc>=2.5. +xml2rfc>=2.6. xym>=0.1.2,!=0.3 #zxcvbn-python>=4.4.14 # Not needed until we do back-end password entropy validation