diff --git a/bin/daily b/bin/daily index c60c1357b..f868e933d 100755 --- a/bin/daily +++ b/bin/daily @@ -34,3 +34,7 @@ $DTDIR/ietf/bin/expire-last-calls # Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner: $DTDIR/ietf/bin/rfc-editor-index-updates -d 1969-01-01 +# Fetch meeting attendance data from ietf.org/registration/attendees +$DTDIR/ietf/manage.py fetch_meeting_attendance --latest + + diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 8ddad4e9f..ee0d27bb8 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -212,9 +212,9 @@ class ToOneField(tastypie.fields.ToOneField): if not foreign_obj: if not self.null: if callable(self.attribute): - raise ApiFieldError("The related resource for resource %s could not be found." % (previous_obj)) + raise ApiFieldError(u"The related resource for resource %s could not be found." % (previous_obj)) else: - raise ApiFieldError("The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr)) + raise ApiFieldError(u"The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr)) return None fk_resource = self.get_related_resource(foreign_obj) diff --git a/ietf/bin/find-submission-confirmation-email-in-postfix-log b/ietf/bin/find-submission-confirmation-email-in-postfix-log index 3df3c78be..69bdcc331 100755 --- a/ietf/bin/find-submission-confirmation-email-in-postfix-log +++ b/ietf/bin/find-submission-confirmation-email-in-postfix-log @@ -66,7 +66,7 @@ if "<" in from_email: submission = Submission.objects.filter(name=draft).latest('submission_date') document = Document.objects.get(name=draft) -emails = [ author.address for author in document.authors.all() ] +emails = [ author.email.address for author in document.documentauthor_set.all() if author.email ] timestrings = [] for file in [ Path(settings.INTERNET_DRAFT_PATH) / ("%s-%s.txt"%(draft, submission.rev)), diff --git a/ietf/bin/generate-draft-aliases b/ietf/bin/generate-draft-aliases index 7216aa2bc..5852e341b 100755 --- a/ietf/bin/generate-draft-aliases +++ b/ietf/bin/generate-draft-aliases @@ -65,7 +65,7 @@ def get_draft_authors_emails(draft): " Get list of authors for the given draft." # This feels 'correct'; however, it creates fairly large delta - return [email.email_address() for email in draft.authors.all()] + return [author.email.email_address() for author in draft.documentauthor_set.all() if author.email.email_address()] # This gives fairly small delta compared to current state, # however, it seems to be wrong (doesn't check for emails being diff --git a/ietf/community/migrations/0004_cleanup_data.py b/ietf/community/migrations/0004_cleanup_data.py index 6ee14c951..ab1c24f4e 100644 --- a/ietf/community/migrations/0004_cleanup_data.py +++ b/ietf/community/migrations/0004_cleanup_data.py @@ -57,7 +57,7 @@ def port_rules_to_typed_system(apps, schema_editor): elif rule.rule_type in ["author", "author_rfc"]: - found_persons = list(try_to_uniquify_person(rule, Person.objects.filter(email__documentauthor__id__gte=1).filter(name__icontains=rule.value).distinct())) + found_persons = list(try_to_uniquify_person(rule, Person.objects.filter(documentauthor__id__gte=1).filter(name__icontains=rule.value).distinct())) if found_persons: rule.person = found_persons[0] diff --git a/ietf/community/tests.py b/ietf/community/tests.py index 9efba6d17..77899544e 100644 --- a/ietf/community/tests.py +++ b/ietf/community/tests.py @@ -35,7 +35,7 @@ class CommunityListTests(TestCase): rule_state_iesg = SearchRule.objects.create(rule_type="state_iesg", state=State.objects.get(type="draft-iesg", slug="lc"), community_list=clist) - rule_author = SearchRule.objects.create(rule_type="author", state=State.objects.get(type="draft", slug="active"), person=Person.objects.filter(email__documentauthor__document=draft).first(), community_list=clist) + rule_author = SearchRule.objects.create(rule_type="author", state=State.objects.get(type="draft", slug="active"), person=Person.objects.filter(documentauthor__document=draft).first(), community_list=clist) rule_ad = SearchRule.objects.create(rule_type="ad", state=State.objects.get(type="draft", slug="active"), person=draft.ad, community_list=clist) @@ -117,7 +117,7 @@ class CommunityListTests(TestCase): r = self.client.post(url, { "action": "add_rule", "rule_type": "author_rfc", - "author_rfc-person": Person.objects.filter(email__documentauthor__document=draft).first().pk, + "author_rfc-person": Person.objects.filter(documentauthor__document=draft).first().pk, "author_rfc-state": State.objects.get(type="draft", slug="rfc").pk, }) self.assertEqual(r.status_code, 302) diff --git a/ietf/community/utils.py b/ietf/community/utils.py index ddd62d9c4..0a573c195 100644 --- a/ietf/community/utils.py +++ b/ietf/community/utils.py @@ -90,7 +90,7 @@ def docs_matching_community_list_rule(rule): elif rule.rule_type.startswith("state_"): return docs.filter(states=rule.state) elif rule.rule_type in ["author", "author_rfc"]: - return docs.filter(states=rule.state, documentauthor__author__person=rule.person) + return docs.filter(states=rule.state, documentauthor__person=rule.person) elif rule.rule_type == "ad": return docs.filter(states=rule.state, ad=rule.person) elif rule.rule_type == "shepherd": @@ -123,7 +123,7 @@ def community_list_rules_matching_doc(doc): rules |= SearchRule.objects.filter( rule_type__in=["author", "author_rfc"], state__in=states, - person__in=list(Person.objects.filter(email__documentauthor__document=doc)), + person__in=list(Person.objects.filter(documentauthor__document=doc)), ) if doc.ad_id: diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index db753e070..58222b24e 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -26,7 +26,7 @@ class DocAliasInline(admin.TabularInline): class DocAuthorInline(admin.TabularInline): model = DocumentAuthor - raw_id_fields = ['author', ] + raw_id_fields = ['person', 'email'] extra = 1 class RelatedDocumentInline(admin.TabularInline): @@ -100,7 +100,7 @@ class DocumentAdmin(admin.ModelAdmin): list_display = ['name', 'rev', 'group', 'pages', 'intended_std_level', 'author_list', 'time'] search_fields = ['name'] list_filter = ['type'] - raw_id_fields = ['authors', 'group', 'shepherd', 'ad'] + raw_id_fields = ['group', 'shepherd', 'ad'] inlines = [DocAliasInline, DocAuthorInline, RelatedDocumentInline, ] form = DocumentForm @@ -123,7 +123,7 @@ class DocHistoryAdmin(admin.ModelAdmin): list_display = ['doc', 'rev', 'state', 'group', 'pages', 'intended_std_level', 'author_list', 'time'] search_fields = ['doc__name'] ordering = ['time', 'doc', 'rev'] - raw_id_fields = ['doc', 'authors', 'group', 'shepherd', 'ad'] + raw_id_fields = ['doc', 'group', 'shepherd', 'ad'] def state(self, instance): return instance.get_state() @@ -175,8 +175,8 @@ class BallotPositionDocEventAdmin(DocEventAdmin): admin.site.register(BallotPositionDocEvent, BallotPositionDocEventAdmin) class DocumentAuthorAdmin(admin.ModelAdmin): - list_display = ['id', 'document', 'author', 'order'] - search_fields = [ 'document__name', 'author__address', ] - raw_id_fields = ['document', 'author', ] + list_display = ['id', 'document', 'person', 'email', 'affiliation', 'country', 'order'] + search_fields = ['document__docalias__name', 'person__name', 'email__address', 'affiliation', 'country'] + raw_id_fields = ["document", "person", "email"] admin.site.register(DocumentAuthor, DocumentAuthorAdmin) diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index f0802fa8a..569e3e5d1 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -46,8 +46,8 @@ class DocumentFactory(factory.DjangoModelFactory): def authors(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument if create and extracted: order = 0 - for email in extracted: - DocumentAuthor.objects.create(document=obj, author=email, order=order) + for person in extracted: + DocumentAuthor.objects.create(document=obj, person=person, order=order) order += 1 @classmethod diff --git a/ietf/doc/migrations/0030_author_revamp_and_extra_attributes.py b/ietf/doc/migrations/0030_author_revamp_and_extra_attributes.py new file mode 100644 index 000000000..465d2242e --- /dev/null +++ b/ietf/doc/migrations/0030_author_revamp_and_extra_attributes.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0020_formallanguagename'), + ('doc', '0029_update_rfc_authors'), + ] + + operations = [ + migrations.AddField( + model_name='dochistory', + name='words', + field=models.IntegerField(null=True, blank=True), + ), + migrations.AddField( + model_name='document', + name='words', + field=models.IntegerField(null=True, blank=True), + ), + migrations.AddField( + model_name='dochistory', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), + migrations.AddField( + model_name='document', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), + migrations.RemoveField( + model_name='dochistory', + name='authors', + ), + migrations.RemoveField( + model_name='document', + name='authors', + ), + migrations.AddField( + model_name='dochistoryauthor', + name='affiliation', + field=models.CharField(help_text=b'Organization/company used by author for submission', max_length=100, blank=True), + ), + migrations.AddField( + model_name='dochistoryauthor', + name='country', + field=models.CharField(blank=True, help_text=b'Country used by author for submission', max_length=255), + ), + migrations.RenameField( + model_name='dochistoryauthor', + old_name='author', + new_name='email', + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='email', + field=models.ForeignKey(blank=True, to='person.Email', help_text=b'Email address used by author for submission', null=True), + ), + migrations.AddField( + model_name='dochistoryauthor', + name='person', + field=models.ForeignKey(blank=True, to='person.Person', null=True), + ), + migrations.AddField( + model_name='documentauthor', + name='affiliation', + field=models.CharField(help_text=b'Organization/company used by author for submission', max_length=100, blank=True), + ), + migrations.AddField( + model_name='documentauthor', + name='country', + field=models.CharField(blank=True, help_text=b'Country used by author for submission', max_length=255), + ), + migrations.RenameField( + model_name='documentauthor', + old_name='author', + new_name='email', + ), + migrations.AlterField( + model_name='documentauthor', + name='email', + field=models.ForeignKey(blank=True, to='person.Email', help_text=b'Email address used by author for submission', null=True), + ), + migrations.AddField( + model_name='documentauthor', + name='person', + field=models.ForeignKey(blank=True, to='person.Person', null=True), + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='document', + field=models.ForeignKey(related_name='documentauthor_set', to='doc.DocHistory'), + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='order', + field=models.IntegerField(default=1), + ), + migrations.RunSQL("update doc_documentauthor a inner join person_email e on a.email_id = e.address set a.person_id = e.person_id;", migrations.RunSQL.noop), + migrations.RunSQL("update doc_dochistoryauthor a inner join person_email e on a.email_id = e.address set a.person_id = e.person_id;", migrations.RunSQL.noop), + migrations.AlterField( + model_name='documentauthor', + name='person', + field=models.ForeignKey(to='person.Person'), + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='person', + field=models.ForeignKey(to='person.Person'), + ), + ] diff --git a/ietf/doc/migrations/0031_remove_fake_email_adresses.py b/ietf/doc/migrations/0031_remove_fake_email_adresses.py new file mode 100644 index 000000000..d6888a517 --- /dev/null +++ b/ietf/doc/migrations/0031_remove_fake_email_adresses.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def fix_invalid_emails(apps, schema_editor): + Email = apps.get_model("person", "Email") + Role = apps.get_model("group", "Role") + RoleHistory = apps.get_model("group", "RoleHistory") + DocumentAuthor = apps.get_model("doc", "DocumentAuthor") + DocHistoryAuthor = apps.get_model("doc", "DocHistoryAuthor") + + e = Email.objects.filter(address="unknown-email-Gigi-Karmous-Edwards").first() + if e: + # according to ftp://ietf.org/ietf/97dec/adsl-minutes-97dec.txt + new_e, _ = Email.objects.get_or_create( + address="GiGi.Karmous-Edwards@pulse.com", + primary=e.primary, + active=e.active, + person=e.person, + ) + Role.objects.filter(email=e).update(email=new_e) + RoleHistory.objects.filter(email=e).update(email=new_e) + e.delete() + + e = Email.objects.filter(address="unknown-email-Pat-Thaler").first() + if e: + # current chair email + new_e = Email.objects.get(address="pat.thaler@broadcom.com") + Role.objects.filter(email=e).update(email=new_e) + RoleHistory.objects.filter(email=e).update(email=new_e) + e.delete() + + e = Email.objects.filter(address="unknown-email-Greg->").first() + if e: + # current email + new_e = Email.objects.get(address="gregimirsky@gmail.com") + DocumentAuthor.objects.filter(email=e).update(email=new_e) + DocHistoryAuthor.objects.filter(email=e).update(email=new_e) + e.delete() + + DocumentAuthor.objects.filter(email__address__startswith="unknown-email-").exclude(email__address__contains="@").update(email=None) + DocHistoryAuthor.objects.filter(email__address__startswith="unknown-email-").exclude(email__address__contains="@").update(email=None) + Email.objects.exclude(address__contains="@").filter(address__startswith="unknown-email-").delete() + + assert not Email.objects.filter(address__startswith="unknown-email-") + +class Migration(migrations.Migration): + + dependencies = [ + ('doc', '0030_author_revamp_and_extra_attributes'), + ('person', '0014_auto_20160613_0751'), + ('group', '0009_auto_20150930_0758'), + ] + + operations = [ + migrations.RunPython(fix_invalid_emails, migrations.RunPython.noop), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 6604e263d..c5c9dc49d 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -18,12 +18,13 @@ import debug # pyflakes:ignore from ietf.group.models import Group from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName, - DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName ) + DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, FormalLanguageName ) from ietf.person.models import Email, Person from ietf.utils import log from ietf.utils.admin import admin_link from ietf.utils.rfcmarkup import markup from ietf.utils.validators import validate_no_control_chars +from ietf.utils.mail import formataddr logger = logging.getLogger('django') @@ -82,6 +83,8 @@ class DocumentInfo(models.Model): abstract = models.TextField(blank=True) rev = models.CharField(verbose_name="revision", max_length=16, blank=True) pages = models.IntegerField(blank=True, null=True) + words = models.IntegerField(blank=True, null=True) + formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document") order = models.IntegerField(default=1, blank=True) # This is probably obviated by SessionPresentaion.order intended_std_level = models.ForeignKey(IntendedStdLevelName, verbose_name="Intended standardization level", blank=True, null=True) std_level = models.ForeignKey(StdLevelName, verbose_name="Standardization level", blank=True, null=True) @@ -319,7 +322,7 @@ class DocumentInfo(models.Model): return self._cached_is_rfc def author_list(self): - return ", ".join(email.address for email in self.authors.all()) + return u", ".join(author.email_id for author in self.documentauthor_set.all() if author.email_id) # This, and several other ballot related functions here, assume that there is only one active ballot for a document at any point in time. # If that assumption is violated, they will only expose the most recently created ballot @@ -509,20 +512,33 @@ class RelatedDocument(models.Model): return False -class DocumentAuthor(models.Model): - document = models.ForeignKey('Document') - author = models.ForeignKey(Email, help_text="Email address used by author for submission") +class DocumentAuthorInfo(models.Model): + person = models.ForeignKey(Person) + # email should only be null for some historic documents + email = models.ForeignKey(Email, help_text="Email address used by author for submission", blank=True, null=True) + affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission") + country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission") order = models.IntegerField(default=1) - def __unicode__(self): - return u"%s %s (%s)" % (self.document.name, self.author.get_name(), self.order) + def formatted_email(self): + + if self.email: + return formataddr((self.person.plain_ascii(), self.email.address)) + else: + return "" class Meta: + abstract = True ordering = ["document", "order"] - + +class DocumentAuthor(DocumentAuthorInfo): + document = models.ForeignKey('Document') + + def __unicode__(self): + return u"%s %s (%s)" % (self.document.name, self.person, self.order) + class Document(DocumentInfo): name = models.CharField(max_length=255, primary_key=True) # immutable - authors = models.ManyToManyField(Email, through=DocumentAuthor, blank=True) def __unicode__(self): return self.name @@ -757,16 +773,13 @@ class RelatedDocHistory(models.Model): def __unicode__(self): return u"%s %s %s" % (self.source.doc.name, self.relationship.name.lower(), self.target.name) -class DocHistoryAuthor(models.Model): - document = models.ForeignKey('DocHistory') - author = models.ForeignKey(Email) - order = models.IntegerField() +class DocHistoryAuthor(DocumentAuthorInfo): + # use same naming convention as non-history version to make it a bit + # easier to write generic code + document = models.ForeignKey('DocHistory', related_name="documentauthor_set") def __unicode__(self): - return u"%s %s (%s)" % (self.document.doc.name, self.author.get_name(), self.order) - - class Meta: - ordering = ["document", "order"] + return u"%s %s (%s)" % (self.document.doc.name, self.person, self.order) class DocHistory(DocumentInfo): doc = models.ForeignKey(Document, related_name="history_set") @@ -775,7 +788,7 @@ class DocHistory(DocumentInfo): # canonical_name and replace the function on Document with a # property name = models.CharField(max_length=255) - authors = models.ManyToManyField(Email, through=DocHistoryAuthor, blank=True) + def __unicode__(self): return unicode(self.doc.name) diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index f91d03af8..ac6efd312 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -99,7 +99,6 @@ class DocumentResource(ModelResource): shepherd = ToOneField(EmailResource, 'shepherd', null=True) states = ToManyField(StateResource, 'states', null=True) tags = ToManyField(DocTagNameResource, 'tags', null=True) - authors = ToManyField(EmailResource, 'authors', null=True) rfc = CharField(attribute='rfc_number', null=True) class Meta: cache = SimpleCache() @@ -128,14 +127,14 @@ class DocumentResource(ModelResource): "shepherd": ALL_WITH_RELATIONS, "states": ALL_WITH_RELATIONS, "tags": ALL_WITH_RELATIONS, - "authors": ALL_WITH_RELATIONS, } api.doc.register(DocumentResource()) -from ietf.person.resources import EmailResource +from ietf.person.resources import PersonResource, EmailResource class DocumentAuthorResource(ModelResource): + person = ToOneField(PersonResource, 'person') + email = ToOneField(EmailResource, 'email', null=True) document = ToOneField(DocumentResource, 'document') - author = ToOneField(EmailResource, 'author') class Meta: cache = SimpleCache() queryset = DocumentAuthor.objects.all() @@ -143,9 +142,12 @@ class DocumentAuthorResource(ModelResource): #resource_name = 'documentauthor' filtering = { "id": ALL, + "affiliation": ALL, + "country": ALL, "order": ALL, + "person": ALL_WITH_RELATIONS, + "email": ALL_WITH_RELATIONS, "document": ALL_WITH_RELATIONS, - "author": ALL_WITH_RELATIONS, } api.doc.register(DocumentAuthorResource()) @@ -209,7 +211,6 @@ class DocHistoryResource(ModelResource): doc = ToOneField(DocumentResource, 'doc') states = ToManyField(StateResource, 'states', null=True) tags = ToManyField(DocTagNameResource, 'tags', null=True) - authors = ToManyField(EmailResource, 'authors', null=True) class Meta: cache = SimpleCache() queryset = DocHistory.objects.all() @@ -239,7 +240,6 @@ class DocHistoryResource(ModelResource): "doc": ALL_WITH_RELATIONS, "states": ALL_WITH_RELATIONS, "tags": ALL_WITH_RELATIONS, - "authors": ALL_WITH_RELATIONS, } api.doc.register(DocHistoryResource()) @@ -412,10 +412,11 @@ class InitialReviewDocEventResource(ModelResource): } api.doc.register(InitialReviewDocEventResource()) -from ietf.person.resources import EmailResource +from ietf.person.resources import PersonResource, EmailResource class DocHistoryAuthorResource(ModelResource): + person = ToOneField(PersonResource, 'person') + email = ToOneField(EmailResource, 'email', null=True) document = ToOneField(DocHistoryResource, 'document') - author = ToOneField(EmailResource, 'author') class Meta: cache = SimpleCache() queryset = DocHistoryAuthor.objects.all() @@ -423,9 +424,12 @@ class DocHistoryAuthorResource(ModelResource): #resource_name = 'dochistoryauthor' filtering = { "id": ALL, + "affiliation": ALL, + "country": ALL, "order": ALL, + "person": ALL_WITH_RELATIONS, + "email": ALL_WITH_RELATIONS, "document": ALL_WITH_RELATIONS, - "author": ALL_WITH_RELATIONS, } api.doc.register(DocHistoryAuthorResource()) diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 260ed1940..b71be5aa5 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -192,6 +192,11 @@ def rfcnospace(string): else: return string +@register.filter +def prettystdname(string): + from ietf.doc.utils import prettify_std_name + return prettify_std_name(unicode(string or "")) + @register.filter(name='rfcurl') def rfclink(string): """ diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 351512b01..c014386a4 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -87,7 +87,7 @@ class SearchTests(TestCase): self.assertTrue(draft.title in unicontent(r)) # find by author - r = self.client.get(base_url + "?activedrafts=on&by=author&author=%s" % draft.authors.all()[0].person.name_parts()[1]) + r = self.client.get(base_url + "?activedrafts=on&by=author&author=%s" % draft.documentauthor_set.first().person.name_parts()[1]) self.assertEqual(r.status_code, 200) self.assertTrue(draft.title in unicontent(r)) @@ -1237,7 +1237,7 @@ class ChartTests(ResourceTestCaseMixin, TestCase): person = PersonFactory.create() DocumentFactory.create( states=[('draft','active')], - authors=[person.email(), ], + authors=[person, ], ) conf_url = urlreverse('ietf.doc.views_stats.chart_conf_person_drafts', kwargs=dict(id=person.id)) diff --git a/ietf/doc/tests_downref.py b/ietf/doc/tests_downref.py index 5a7ff5931..bce1d7c3a 100644 --- a/ietf/doc/tests_downref.py +++ b/ietf/doc/tests_downref.py @@ -13,6 +13,11 @@ from ietf.utils.test_data import make_test_data, make_downref_test_data from ietf.utils.test_utils import login_testing_unauthorized, unicontent class Downref(TestCase): + + def setUp(self): + make_test_data() + make_downref_test_data() + def test_downref_registry(self): url = urlreverse('ietf.doc.views_downref.downref_registry') @@ -103,7 +108,3 @@ class Downref(TestCase): self.assertTrue(RelatedDocument.objects.filter(source=draft, target=rfc, relationship_id='downref-approval')) self.assertEqual(draft.docevent_set.count(), draft_de_count_before + 1) self.assertEqual(rfc.document.docevent_set.count(), rfc_de_count_before + 1) - - def setUp(self): - make_test_data() - make_downref_test_data() diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index e1df40ed8..e91cb8aaa 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -377,7 +377,10 @@ class EditInfoTests(TestCase): DocumentAuthor.objects.create( document=draft, - author=Email.objects.get(address="aread@ietf.org"), + person=Person.objects.get(email__address="aread@ietf.org"), + email=Email.objects.get(address="aread@ietf.org"), + country="US", + affiliation="", order=1 ) @@ -1348,7 +1351,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.basea.documentauthor_set.create(author=Email.objects.create(address="basea_author@example.com"),order=1) + p = Person.objects.create(address="basea_author") + e = Email.objects.create(address="basea_author@example.com", person=p) + self.basea.documentauthor_set.create(person=p, email=e, order=1) self.baseb = Document.objects.create( name="draft-test-base-b", @@ -1359,7 +1364,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() - datetime.timedelta(days = 365 - settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.baseb.documentauthor_set.create(author=Email.objects.create(address="baseb_author@example.com"),order=1) + p = Person.objects.create(name="baseb_author") + e = Email.objects.create(address="baseb_author@example.com", person=p) + self.baseb.documentauthor_set.create(person=p, email=e, order=1) self.replacea = Document.objects.create( name="draft-test-replace-a", @@ -1370,7 +1377,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() + datetime.timedelta(days = settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.replacea.documentauthor_set.create(author=Email.objects.create(address="replacea_author@example.com"),order=1) + p = Person.objects.create(name="replacea_author") + e = Email.objects.create(address="replacea_author@example.com", person=p) + self.replacea.documentauthor_set.create(person=p, email=e, order=1) self.replaceboth = Document.objects.create( name="draft-test-replace-both", @@ -1381,7 +1390,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() + datetime.timedelta(days = settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.replaceboth.documentauthor_set.create(author=Email.objects.create(address="replaceboth_author@example.com"),order=1) + p = Person.objects.create(name="replaceboth_author") + e = Email.objects.create(address="replaceboth_author@example.com", person=p) + self.replaceboth.documentauthor_set.create(person=p, email=e, order=1) self.basea.set_state(State.objects.get(used=True, type="draft", slug="active")) self.baseb.set_state(State.objects.get(used=True, type="draft", slug="expired")) diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index 0d77703ab..23825bcd4 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -260,10 +260,7 @@ class ReviewTests(TestCase): # set up some reviewer-suitability factors reviewer_email = Email.objects.get(person__user__username="reviewer") - DocumentAuthor.objects.create( - author=reviewer_email, - document=doc, - ) + DocumentAuthor.objects.create(person=reviewer_email.person, email=reviewer_email, document=doc) doc.rev = "10" doc.save_with_history([DocEvent.objects.create(doc=doc, rev=doc.rev, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")]) @@ -572,7 +569,7 @@ class ReviewTests(TestCase): self.assertTrue(settings.MAILING_LIST_ARCHIVE_URL in review_req.review.external_url) # Check that the review has the reviewer as author - self.assertEqual(review_req.reviewer, review_req.review.authors.first()) + self.assertEqual(review_req.reviewer, review_req.review.documentauthor_set.first().email) # Check that we have a copy of the outgoing message msgid = outbox[0]["Message-ID"] diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 9210ab11a..4107bf41b 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -56,7 +56,6 @@ from ietf.group.models import Role from ietf.group.utils import can_manage_group_type, can_manage_materials from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, user_is_person, role_required from ietf.name.models import StreamName, BallotPositionName -from ietf.person.models import Email from ietf.utils.history import find_history_active_at from ietf.doc.forms import TelechatForm, NotifyForm from ietf.doc.mails import email_comment @@ -167,7 +166,7 @@ def document_main(request, name, rev=None): can_edit_replaces = has_role(request.user, ("Area Director", "Secretariat", "IRTF Chair", "WG Chair", "RG Chair", "WG Secretary", "RG Secretary")) - is_author = unicode(request.user) in set([email.address for email in doc.authors.all()]) + is_author = request.user.is_authenticated() and doc.documentauthor_set.filter(person__user=request.user).exists() can_view_possibly_replaces = can_edit_replaces or is_author rfc_number = name[3:] if name.startswith("") else None @@ -994,11 +993,11 @@ def document_json(request, name, rev=None): data["intended_std_level"] = extract_name(doc.intended_std_level) data["std_level"] = extract_name(doc.std_level) data["authors"] = [ - dict(name=e.person.name, - email=e.address, - affiliation=e.person.affiliation) - for e in Email.objects.filter(documentauthor__document=doc).select_related("person").order_by("documentauthor__order") - ] + dict(name=author.person.name, + email=author.email.address, + affiliation=author.affiliation) + for author in doc.documentauthor_set.all().select_related("person", "email").order_by("order") + ] data["shepherd"] = doc.shepherd.formatted_email() if doc.shepherd else None data["ad"] = doc.ad.role_email("ad").formatted_email() if doc.ad else None diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index 9e010fba0..1db51ec9c 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -567,7 +567,7 @@ def complete_review(request, name, request_id): else: author_email = request.user.person.email() frm = request.user.person.formatted_email() - author, created = DocumentAuthor.objects.get_or_create(document=review, author=author_email) + author, created = DocumentAuthor.objects.get_or_create(document=review, email=author_email, person=request.user.person) if need_to_email_review: # email the review diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index b46b7a373..ed81b5682 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -167,7 +167,7 @@ def retrieve_search_results(form, all_types=False): # radio choices by = query["by"] if by == "author": - docs = docs.filter(authors__person__alias__name__icontains=query["author"]) + docs = docs.filter(documentauthor__person__alias__name__icontains=query["author"]) elif by == "group": docs = docs.filter(group__acronym=query["group"]) elif by == "area": diff --git a/ietf/doc/views_stats.py b/ietf/doc/views_stats.py index 639a9bb93..54249774b 100644 --- a/ietf/doc/views_stats.py +++ b/ietf/doc/views_stats.py @@ -171,7 +171,7 @@ def chart_data_person_drafts(request, id): if not person: data = [] else: - data = model_to_timeline_data(DocEvent, doc__authors__person=person, type='new_revision') + data = model_to_timeline_data(DocEvent, doc__documentauthor__person=person, type='new_revision') return JsonResponse(data, safe=False) diff --git a/ietf/group/models.py b/ietf/group/models.py index c49abc37a..93277c13f 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -14,6 +14,7 @@ from ietf.group.colors import fg_group_colors, bg_group_colors from ietf.name.models import GroupStateName, GroupTypeName, DocTagName, GroupMilestoneStateName, RoleName from ietf.person.models import Email, Person from ietf.utils.mail import formataddr +from ietf.utils.log import unreachable class GroupInfo(models.Model): @@ -278,7 +279,12 @@ class Role(models.Model): return u"%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym or self.group.name) def name_and_email(self): - "Returns name and email, e.g.: u'Ano Nymous ' " + """ + Returns name and email, e.g.: u'Ano Nymous ' + Is intended for display use, not in email context. + Use self.formatted_email() for that. + """ + unreachable() if self.person: return u"%s <%s>" % (self.person.plain_name(), self.email.address) else: diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py index 2601f9888..22a2845ef 100644 --- a/ietf/idindex/index.py +++ b/ietf/idindex/index.py @@ -117,15 +117,15 @@ def all_id2_txt(): file_types = file_types_for_drafts() authors = {} - for a in DocumentAuthor.objects.filter(document__name__startswith="draft-").order_by("order").select_related("author", "author__person").iterator(): + for a in DocumentAuthor.objects.filter(document__name__startswith="draft-").order_by("order").select_related("email", "person").iterator(): if a.document_id not in authors: l = authors[a.document_id] = [] else: l = authors[a.document_id] - if "@" in a.author.address: - l.append(u'%s <%s>' % (a.author.person.plain_name().replace("@", ""), a.author.address.replace(",", ""))) + if a.email: + l.append(u'%s <%s>' % (a.person.plain_name().replace("@", ""), a.email.address.replace(",", ""))) else: - l.append(a.author.person.plain_name()) + l.append(a.person.plain_name()) shepherds = dict((e.pk, e.formatted_ascii_email().replace('"', '')) for e in Email.objects.filter(shepherd_document_set__type="draft").select_related("person").distinct()) @@ -236,12 +236,12 @@ def active_drafts_index_by_group(extra_values=()): d["initial_rev_time"] = time # add authors - for a in DocumentAuthor.objects.filter(document__states=active_state).order_by("order").select_related("author__person"): + for a in DocumentAuthor.objects.filter(document__states=active_state).order_by("order").select_related("person"): d = docs_dict.get(a.document_id) if d: if "authors" not in d: d["authors"] = [] - d["authors"].append(a.author.person.plain_ascii()) # This should probably change to .plain_name() when non-ascii names are permitted + d["authors"].append(a.person.plain_ascii()) # This should probably change to .plain_name() when non-ascii names are permitted # put docs into groups for d in docs_dict.itervalues(): diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py index e06739237..28408cfff 100644 --- a/ietf/idindex/tests.py +++ b/ietf/idindex/tests.py @@ -97,7 +97,7 @@ class IndexTests(TestCase): self.assertEqual(t[12], ".pdf,.txt") self.assertEqual(t[13], draft.title) author = draft.documentauthor_set.order_by("order").get() - self.assertEqual(t[14], u"%s <%s>" % (author.author.person.name, author.author.address)) + self.assertEqual(t[14], u"%s <%s>" % (author.person.name, author.email.address)) self.assertEqual(t[15], u"%s <%s>" % (draft.shepherd.person.name, draft.shepherd.address)) self.assertEqual(t[16], u"%s <%s>" % (draft.ad.plain_ascii(), draft.ad.email_address())) diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index f1709c4c3..2abf1ba72 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -64,15 +64,15 @@ def get_document_emails(ipr): messages = [] for rel in ipr.iprdocrel_set.all(): doc = rel.document.document - authors = doc.authors.all() - + if is_draft(doc): doc_info = 'Internet-Draft entitled "{}" ({})'.format(doc.title,doc.name) else: doc_info = 'RFC entitled "{}" (RFC{})'.format(doc.title,get_rfc_num(doc)) addrs = gather_address_lists('ipr_posted_on_doc',doc=doc).as_strings(compact=False) - author_names = ', '.join([a.person.name for a in authors]) + + author_names = ', '.join(a.person.name for a in doc.documentauthor_set.select_related("person")) context = dict( doc_info=doc_info, diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py index 24e721463..b5de7e005 100644 --- a/ietf/mailtrigger/models.py +++ b/ietf/mailtrigger/models.py @@ -6,7 +6,6 @@ from django.template import Template, Context from email.utils import parseaddr from ietf.utils.mail import formataddr - import debug # pyflakes:ignore from ietf.group.models import Role @@ -178,10 +177,14 @@ class Recipient(models.Model): return addrs def gather_submission_authors(self, **kwargs): + """ + Returns a list of name and email, e.g.: [ 'Ano Nymous ' ] + Is intended for display use, not in email context. + """ addrs = [] if 'submission' in kwargs: submission = kwargs['submission'] - addrs.extend(["%s <%s>" % (author["name"], author["email"]) for author in submission.authors_parsed() if author["email"]]) + addrs.extend(["%s <%s>" % (author["name"], author["email"]) for author in submission.authors if author.get("email")]) return addrs def gather_submission_group_chairs(self, **kwargs): @@ -203,10 +206,14 @@ class Recipient(models.Model): submission = kwargs['submission'] doc=submission.existing_document() if doc: - old_authors = [i.author.formatted_email() for i in doc.documentauthor_set.all() if not i.author.invalid_address()] - new_authors = [ formataddr((author["name"], author["email"])) for author in submission.authors_parsed() if author["email"]] - addrs.extend(old_authors) - if doc.group and set(old_authors)!=set(new_authors): + old_authors = [ author for author in doc.documentauthor_set.all() if author.email ] + + addrs.extend([ author.formatted_email() for author in old_authors]) + + old_author_email_set = set(author.email.address for author in old_authors) + new_author_email_set = set(author["email"] for author in submission.authors if author.get("email")) + + if doc.group and old_author_email_set != new_author_email_set: if doc.group.type_id in ['wg','rg','ag']: addrs.extend(Recipient.objects.get(slug='group_chairs').gather(**{'group':doc.group})) elif doc.group.type_id in ['area']: @@ -216,8 +223,8 @@ class Recipient(models.Model): if doc.stream_id and doc.stream_id not in ['ietf']: addrs.extend(Recipient.objects.get(slug='stream_managers').gather(**{'streams':[doc.stream_id]})) else: - addrs.extend([formataddr((author["name"], author["email"])) for author in submission.authors_parsed() if author["email"]]) - if submission.submitter_parsed()["email"]: + addrs.extend([formataddr((author["name"], author["email"])) for author in submission.authors if author.get("email")]) + if submission.submitter_parsed()["email"]: addrs.append(submission.submitter) return addrs diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index 8618e8898..97e5c72f0 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -6,6 +6,8 @@ from django import forms from django.db.models import Q from django.forms import BaseInlineFormSet +import debug # pyflakes:ignore + from ietf.doc.models import Document, DocAlias, State, NewRevisionDocEvent from ietf.doc.utils import get_document_content from ietf.group.models import Group diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 5f8ef177a..914e907f7 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -297,7 +297,7 @@ class MeetingTests(TestCase): r = self.client.get(urlreverse("ietf.meeting.views.materials", kwargs=dict(num=meeting.number))) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - row = q('#content td div:contains("%s")' % str(session.group.acronym)).closest("tr") + row = q('#content #%s' % str(session.group.acronym)).closest("tr") self.assertTrue(row.find('a:contains("Agenda")')) self.assertTrue(row.find('a:contains("Minutes")')) self.assertTrue(row.find('a:contains("Slideshow")')) @@ -987,6 +987,7 @@ class InterimTests(TestCase): 'session_set-INITIAL_FORMS':0} r = self.client.post(urlreverse("ietf.meeting.views.interim_request"),data) + self.assertRedirects(r,urlreverse('ietf.meeting.views.upcoming')) meeting = Meeting.objects.order_by('id').last() self.assertEqual(meeting.type_id,'interim') diff --git a/ietf/name/admin.py b/ietf/name/admin.py index 684e930ce..c4d2caa35 100644 --- a/ietf/name/admin.py +++ b/ietf/name/admin.py @@ -1,17 +1,21 @@ from django.contrib import admin from ietf.name.models import ( - BallotPositionName, ConstraintName, DBTemplateTypeName, DocRelationshipName, + BallotPositionName, ConstraintName, ContinentName, CountryName, + DBTemplateTypeName, DocRelationshipName, DocReminderTypeName, DocTagName, DocTypeName, DraftSubmissionStateName, - FeedbackTypeName, GroupMilestoneStateName, GroupStateName, GroupTypeName, + FeedbackTypeName, FormalLanguageName, GroupMilestoneStateName, GroupStateName, GroupTypeName, IntendedStdLevelName, IprDisclosureStateName, IprEventTypeName, IprLicenseTypeName, LiaisonStatementEventTypeName, LiaisonStatementPurposeName, LiaisonStatementState, LiaisonStatementTagName, MeetingTypeName, NomineePositionStateName, ReviewRequestStateName, ReviewResultName, ReviewTypeName, RoleName, RoomResourceName, SessionStatusName, StdLevelName, StreamName, TimeSlotTypeName, ) +from ietf.stats.models import CountryAlias + class NameAdmin(admin.ModelAdmin): list_display = ["slug", "name", "desc", "used"] + search_fields = ["slug", "name"] prepopulate_from = { "slug": ("name",) } class DocRelationshipNameAdmin(NameAdmin): @@ -26,12 +30,24 @@ class GroupTypeNameAdmin(NameAdmin): list_display = ["slug", "name", "verbose_name", "desc", "used"] admin.site.register(GroupTypeName, GroupTypeNameAdmin) +class CountryAliasInline(admin.TabularInline): + model = CountryAlias + extra = 1 + +class CountryNameAdmin(NameAdmin): + list_display = ["slug", "name", "continent", "in_eu"] + list_filter = ["continent", "in_eu"] + inlines = [CountryAliasInline] +admin.site.register(CountryName, CountryNameAdmin) + admin.site.register(BallotPositionName, NameAdmin) admin.site.register(ConstraintName, NameAdmin) +admin.site.register(ContinentName, NameAdmin) admin.site.register(DBTemplateTypeName, NameAdmin) admin.site.register(DocReminderTypeName, NameAdmin) admin.site.register(DocTagName, NameAdmin) admin.site.register(DraftSubmissionStateName, NameAdmin) +admin.site.register(FormalLanguageName, NameAdmin) admin.site.register(FeedbackTypeName, NameAdmin) admin.site.register(GroupMilestoneStateName, NameAdmin) admin.site.register(GroupStateName, NameAdmin) diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 9f29ed30e..6a6cd71e2 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -4018,6 +4018,3064 @@ "model": "name.constraintname", "pk": "conflict" }, + { + "fields": { + "desc": "", + "name": "Africa", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "africa" + }, + { + "fields": { + "desc": "", + "name": "Antarctica", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "antarctica" + }, + { + "fields": { + "desc": "", + "name": "Asia", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "asia" + }, + { + "fields": { + "desc": "", + "name": "Europe", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "europe" + }, + { + "fields": { + "desc": "", + "name": "North America", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "north-america" + }, + { + "fields": { + "desc": "", + "name": "Oceania", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "oceania" + }, + { + "fields": { + "desc": "", + "name": "South America", + "order": 0, + "used": true + }, + "model": "name.continentname", + "pk": "south-america" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Andorra", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AD" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "United Arab Emirates", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AE" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Afghanistan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AF" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Antigua and Barbuda", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AG" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Anguilla", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AI" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Albania", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AL" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Armenia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AM" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Angola", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AO" + }, + { + "fields": { + "continent": "antarctica", + "desc": "", + "in_eu": false, + "name": "Antarctica", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AQ" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Argentina", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AR" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "American Samoa", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Austria", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AT" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Australia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AU" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Aruba", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AW" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Åland Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AX" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Azerbaijan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "AZ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Bosnia and Herzegovina", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BA" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Barbados", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BB" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Bangladesh", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BD" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Belgium", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BE" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Burkina Faso", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BF" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Bulgaria", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BG" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Bahrain", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BH" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Burundi", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BI" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Benin", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BJ" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Barthélemy", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BL" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Bermuda", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BM" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Brunei", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BN" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Bolivia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BO" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Bonaire, Sint Eustatius and Saba", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BQ" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Brazil", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BR" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Bahamas", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BS" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Bhutan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BT" + }, + { + "fields": { + "continent": "antarctica", + "desc": "", + "in_eu": false, + "name": "Bouvet Island", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BV" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Botswana", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BW" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Belarus", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BY" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Belize", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "BZ" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Canada", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CA" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Cocos (Keeling) Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CC" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Congo (the Democratic Republic of the)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CD" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Central African Republic", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CF" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Congo", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CG" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Switzerland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CH" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Côte d'Ivoire", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CI" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Cook Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CK" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Chile", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CL" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Cameroon", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CM" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "China", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CN" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Colombia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CO" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Costa Rica", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CR" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Cuba", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CU" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Cabo Verde", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CV" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Curaçao", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CW" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Christmas Island", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CX" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": true, + "name": "Cyprus", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CY" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Czech Republic", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "CZ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Germany", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DE" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Djibouti", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DJ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Denmark", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DK" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Dominica", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DM" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Dominican Republic", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DO" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Algeria", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "DZ" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Ecuador", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "EC" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Estonia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "EE" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Egypt", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "EG" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Western Sahara", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "EH" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Eritrea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ER" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Spain", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ES" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Ethiopia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ET" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Finland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FI" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Fiji", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FJ" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Falkland Islands [Malvinas]", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FK" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Micronesia (Federated States of)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FM" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Faroe Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FO" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "France", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "FR" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Gabon", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GA" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "United Kingdom", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GB" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Grenada", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GD" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Georgia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GE" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "French Guiana", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GF" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Guernsey", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GG" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Ghana", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GH" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Gibraltar", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GI" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Greenland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GL" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Gambia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GM" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Guinea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GN" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Guadeloupe", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GP" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Equatorial Guinea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GQ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Greece", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GR" + }, + { + "fields": { + "continent": "antarctica", + "desc": "", + "in_eu": false, + "name": "South Georgia and the South Sandwich Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GS" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Guatemala", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GT" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Guam", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GU" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Guinea-Bissau", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GW" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Guyana", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "GY" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Hong Kong", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HK" + }, + { + "fields": { + "continent": "antarctica", + "desc": "", + "in_eu": false, + "name": "Heard Island and McDonald Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HM" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Honduras", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HN" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Croatia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HR" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Haiti", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HT" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Hungary", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "HU" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Indonesia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ID" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Ireland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IE" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Israel", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IL" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Isle of Man", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IM" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "India", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IN" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "British Indian Ocean Territory", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IO" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Iraq", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IQ" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Iran", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IR" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Iceland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Italy", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "IT" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Jersey", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "JE" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Jamaica", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "JM" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Jordan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "JO" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Japan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "JP" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Kenya", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KE" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Kyrgyzstan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KG" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Cambodia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KH" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Kiribati", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KI" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Comoros", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KM" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Kitts and Nevis", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KN" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "North Korea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KP" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "South Korea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KR" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Kuwait", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KW" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Cayman Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KY" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Kazakhstan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "KZ" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Laos", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LA" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Lebanon", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LB" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Lucia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LC" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Liechtenstein", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LI" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Sri Lanka", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LK" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Liberia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LR" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Lesotho", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Lithuania", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LT" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Luxembourg", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LU" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Latvia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LV" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Libya", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "LY" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Morocco", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MA" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Monaco", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MC" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Moldova", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MD" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Montenegro", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ME" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Martin (French part)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MF" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Madagascar", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MG" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Marshall Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MH" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Macedonia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MK" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Mali", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ML" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Myanmar", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MM" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Mongolia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MN" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Macao", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MO" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Northern Mariana Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MP" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Martinique", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MQ" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Mauritania", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MR" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Montserrat", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Malta", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MT" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Mauritius", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MU" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Maldives", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MV" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Malawi", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MW" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Mexico", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MX" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Malaysia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MY" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Mozambique", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "MZ" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Namibia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NA" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "New Caledonia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NC" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Niger", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NE" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Norfolk Island", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NF" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Nigeria", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NG" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Nicaragua", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NI" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Netherlands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NL" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Norway", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NO" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Nepal", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NP" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Nauru", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NR" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Niue", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NU" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "New Zealand", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "NZ" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Oman", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "OM" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Panama", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PA" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Peru", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PE" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "French Polynesia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PF" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Papua New Guinea", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PG" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Philippines", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PH" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Pakistan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PK" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Poland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PL" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Pierre and Miquelon", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PM" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Pitcairn", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PN" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Puerto Rico", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PR" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Palestine, State of", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Portugal", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PT" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Palau", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PW" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Paraguay", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "PY" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Qatar", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "QA" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Réunion", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "RE" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Romania", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "RO" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Serbia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "RS" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Russia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "RU" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Rwanda", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "RW" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Saudi Arabia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SA" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Solomon Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SB" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Seychelles", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SC" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Sudan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SD" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Sweden", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SE" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Singapore", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SG" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Saint Helena, Ascension and Tristan da Cunha", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SH" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Slovenia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SI" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Svalbard and Jan Mayen", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SJ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": true, + "name": "Slovakia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SK" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Sierra Leone", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SL" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "San Marino", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SM" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Senegal", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SN" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Somalia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SO" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Suriname", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SR" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "South Sudan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SS" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Sao Tome and Principe", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ST" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "El Salvador", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SV" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Sint Maarten (Dutch part)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SX" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Syria", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SY" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Swaziland", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "SZ" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Turks and Caicos Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TC" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Chad", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TD" + }, + { + "fields": { + "continent": "antarctica", + "desc": "", + "in_eu": false, + "name": "French Southern Territories", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TF" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Togo", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TG" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Thailand", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TH" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Tajikistan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TJ" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Tokelau", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TK" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Timor-Leste", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TL" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Turkmenistan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TM" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Tunisia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TN" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Tonga", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TO" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Turkey", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TR" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Trinidad and Tobago", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TT" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Tuvalu", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TV" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Taiwan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TW" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Tanzania", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "TZ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Ukraine", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "UA" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Uganda", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "UG" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "United States Minor Outlying Islands", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "UM" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "United States of America", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "US" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Uruguay", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "UY" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Uzbekistan", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "UZ" + }, + { + "fields": { + "continent": "europe", + "desc": "", + "in_eu": false, + "name": "Holy See", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VA" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Saint Vincent and the Grenadines", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VC" + }, + { + "fields": { + "continent": "south-america", + "desc": "", + "in_eu": false, + "name": "Venezuela", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VE" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Virgin Islands (British)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VG" + }, + { + "fields": { + "continent": "north-america", + "desc": "", + "in_eu": false, + "name": "Virgin Islands (U.S.)", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VI" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Vietnam", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VN" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Vanuatu", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "VU" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Wallis and Futuna", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "WF" + }, + { + "fields": { + "continent": "oceania", + "desc": "", + "in_eu": false, + "name": "Samoa", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "WS" + }, + { + "fields": { + "continent": "asia", + "desc": "", + "in_eu": false, + "name": "Yemen", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "YE" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Mayotte", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "YT" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "South Africa", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ZA" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Zambia", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ZM" + }, + { + "fields": { + "continent": "africa", + "desc": "", + "in_eu": false, + "name": "Zimbabwe", + "order": 0, + "used": true + }, + "model": "name.countryname", + "pk": "ZW" + }, { "fields": { "desc": "", @@ -4846,6 +7904,66 @@ "model": "name.feedbacktypename", "pk": "questio" }, + { + "fields": { + "desc": "Augmented Backus-Naur Form", + "name": "ABNF", + "order": 1, + "used": true + }, + "model": "name.formallanguagename", + "pk": "abnf" + }, + { + "fields": { + "desc": "Abstract Syntax Notation One", + "name": "ASN.1", + "order": 2, + "used": true + }, + "model": "name.formallanguagename", + "pk": "asn1" + }, + { + "fields": { + "desc": "Concise Binary Object Representation", + "name": "CBOR", + "order": 3, + "used": true + }, + "model": "name.formallanguagename", + "pk": "cbor" + }, + { + "fields": { + "desc": "Code in the C Programming Language", + "name": "C Code", + "order": 4, + "used": true + }, + "model": "name.formallanguagename", + "pk": "ccode" + }, + { + "fields": { + "desc": "Javascript Object Notation", + "name": "JSON", + "order": 5, + "used": true + }, + "model": "name.formallanguagename", + "pk": "json" + }, + { + "fields": { + "desc": "Extensible Markup Language", + "name": "XML", + "order": 6, + "used": true + }, + "model": "name.formallanguagename", + "pk": "xml" + }, { "fields": { "desc": "", diff --git a/ietf/name/migrations/0020_formallanguagename.py b/ietf/name/migrations/0020_formallanguagename.py new file mode 100644 index 000000000..4742516f1 --- /dev/null +++ b/ietf/name/migrations/0020_formallanguagename.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0019_add_docrelationshipname_downref_approval'), + ] + + operations = [ + migrations.CreateModel( + name='FormalLanguageName', + fields=[ + ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('desc', models.TextField(blank=True)), + ('used', models.BooleanField(default=True)), + ('order', models.IntegerField(default=0)), + ], + options={ + 'ordering': ['order', 'name'], + 'abstract': False, + }, + ), + ] diff --git a/ietf/name/migrations/0021_add_formlang_names.py b/ietf/name/migrations/0021_add_formlang_names.py new file mode 100644 index 000000000..3375e2118 --- /dev/null +++ b/ietf/name/migrations/0021_add_formlang_names.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def insert_initial_formal_language_names(apps, schema_editor): + FormalLanguageName = apps.get_model("name", "FormalLanguageName") + FormalLanguageName.objects.get_or_create(slug="abnf", name="ABNF", desc="Augmented Backus-Naur Form", order=1) + FormalLanguageName.objects.get_or_create(slug="asn1", name="ASN.1", desc="Abstract Syntax Notation One", order=2) + FormalLanguageName.objects.get_or_create(slug="cbor", name="CBOR", desc="Concise Binary Object Representation", order=3) + FormalLanguageName.objects.get_or_create(slug="ccode", name="C Code", desc="Code in the C Programming Language", order=4) + FormalLanguageName.objects.get_or_create(slug="json", name="JSON", desc="Javascript Object Notation", order=5) + FormalLanguageName.objects.get_or_create(slug="xml", name="XML", desc="Extensible Markup Language", order=6) + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0020_formallanguagename'), + ] + + operations = [ + migrations.RunPython(insert_initial_formal_language_names, migrations.RunPython.noop) + ] diff --git a/ietf/name/migrations/0022_continentname_countryname.py b/ietf/name/migrations/0022_continentname_countryname.py new file mode 100644 index 000000000..b0a1e91d7 --- /dev/null +++ b/ietf/name/migrations/0022_continentname_countryname.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0021_add_formlang_names'), + ] + + operations = [ + migrations.CreateModel( + name='ContinentName', + fields=[ + ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('desc', models.TextField(blank=True)), + ('used', models.BooleanField(default=True)), + ('order', models.IntegerField(default=0)), + ], + options={ + 'ordering': ['order', 'name'], + 'abstract': False, + }, + ), + migrations.CreateModel( + name='CountryName', + fields=[ + ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('desc', models.TextField(blank=True)), + ('used', models.BooleanField(default=True)), + ('order', models.IntegerField(default=0)), + ('in_eu', models.BooleanField(default=False, verbose_name='In EU')), + ('continent', models.ForeignKey(to='name.ContinentName')), + ], + options={ + 'ordering': ['order', 'name'], + 'abstract': False, + }, + ), + ] diff --git a/ietf/name/migrations/0023_add_country_continent_names.py b/ietf/name/migrations/0023_add_country_continent_names.py new file mode 100644 index 000000000..2ee423513 --- /dev/null +++ b/ietf/name/migrations/0023_add_country_continent_names.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def insert_initial_country_continent_names(apps, schema_editor): + ContinentName = apps.get_model("name", "ContinentName") + africa, _ = ContinentName.objects.get_or_create(slug="africa", name="Africa") + antarctica, _ = ContinentName.objects.get_or_create(slug="antarctica", name="Antarctica") + asia, _ = ContinentName.objects.get_or_create(slug="asia", name="Asia") + europe, _ = ContinentName.objects.get_or_create(slug="europe", name="Europe") + north_america, _ = ContinentName.objects.get_or_create(slug="north-america", name="North America") + oceania, _ = ContinentName.objects.get_or_create(slug="oceania", name="Oceania") + south_america, _ = ContinentName.objects.get_or_create(slug="south-america", name="South America") + + CountryName = apps.get_model("name", "CountryName") + CountryName.objects.get_or_create(slug="AD", name=u"Andorra", continent=europe) + CountryName.objects.get_or_create(slug="AE", name=u"United Arab Emirates", continent=asia) + CountryName.objects.get_or_create(slug="AF", name=u"Afghanistan", continent=asia) + CountryName.objects.get_or_create(slug="AG", name=u"Antigua and Barbuda", continent=north_america) + CountryName.objects.get_or_create(slug="AI", name=u"Anguilla", continent=north_america) + CountryName.objects.get_or_create(slug="AL", name=u"Albania", continent=europe) + CountryName.objects.get_or_create(slug="AM", name=u"Armenia", continent=asia) + CountryName.objects.get_or_create(slug="AO", name=u"Angola", continent=africa) + CountryName.objects.get_or_create(slug="AQ", name=u"Antarctica", continent=antarctica) + CountryName.objects.get_or_create(slug="AR", name=u"Argentina", continent=south_america) + CountryName.objects.get_or_create(slug="AS", name=u"American Samoa", continent=oceania) + CountryName.objects.get_or_create(slug="AT", name=u"Austria", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="AU", name=u"Australia", continent=oceania) + CountryName.objects.get_or_create(slug="AW", name=u"Aruba", continent=north_america) + CountryName.objects.get_or_create(slug="AX", name=u"Åland Islands", continent=europe) + CountryName.objects.get_or_create(slug="AZ", name=u"Azerbaijan", continent=asia) + CountryName.objects.get_or_create(slug="BA", name=u"Bosnia and Herzegovina", continent=europe) + CountryName.objects.get_or_create(slug="BB", name=u"Barbados", continent=north_america) + CountryName.objects.get_or_create(slug="BD", name=u"Bangladesh", continent=asia) + CountryName.objects.get_or_create(slug="BE", name=u"Belgium", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="BF", name=u"Burkina Faso", continent=africa) + CountryName.objects.get_or_create(slug="BG", name=u"Bulgaria", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="BH", name=u"Bahrain", continent=asia) + CountryName.objects.get_or_create(slug="BI", name=u"Burundi", continent=africa) + CountryName.objects.get_or_create(slug="BJ", name=u"Benin", continent=africa) + CountryName.objects.get_or_create(slug="BL", name=u"Saint Barthélemy", continent=north_america) + CountryName.objects.get_or_create(slug="BM", name=u"Bermuda", continent=north_america) + CountryName.objects.get_or_create(slug="BN", name=u"Brunei", continent=asia) + CountryName.objects.get_or_create(slug="BO", name=u"Bolivia", continent=south_america) + CountryName.objects.get_or_create(slug="BQ", name=u"Bonaire, Sint Eustatius and Saba", continent=north_america) + CountryName.objects.get_or_create(slug="BR", name=u"Brazil", continent=south_america) + CountryName.objects.get_or_create(slug="BS", name=u"Bahamas", continent=north_america) + CountryName.objects.get_or_create(slug="BT", name=u"Bhutan", continent=asia) + CountryName.objects.get_or_create(slug="BV", name=u"Bouvet Island", continent=antarctica) + CountryName.objects.get_or_create(slug="BW", name=u"Botswana", continent=africa) + CountryName.objects.get_or_create(slug="BY", name=u"Belarus", continent=europe) + CountryName.objects.get_or_create(slug="BZ", name=u"Belize", continent=north_america) + CountryName.objects.get_or_create(slug="CA", name=u"Canada", continent=north_america) + CountryName.objects.get_or_create(slug="CC", name=u"Cocos (Keeling) Islands", continent=asia) + CountryName.objects.get_or_create(slug="CD", name=u"Congo (the Democratic Republic of the)", continent=africa) + CountryName.objects.get_or_create(slug="CF", name=u"Central African Republic", continent=africa) + CountryName.objects.get_or_create(slug="CG", name=u"Congo", continent=africa) + CountryName.objects.get_or_create(slug="CH", name=u"Switzerland", continent=europe) + CountryName.objects.get_or_create(slug="CI", name=u"Côte d'Ivoire", continent=africa) + CountryName.objects.get_or_create(slug="CK", name=u"Cook Islands", continent=oceania) + CountryName.objects.get_or_create(slug="CL", name=u"Chile", continent=south_america) + CountryName.objects.get_or_create(slug="CM", name=u"Cameroon", continent=africa) + CountryName.objects.get_or_create(slug="CN", name=u"China", continent=asia) + CountryName.objects.get_or_create(slug="CO", name=u"Colombia", continent=south_america) + CountryName.objects.get_or_create(slug="CR", name=u"Costa Rica", continent=north_america) + CountryName.objects.get_or_create(slug="CU", name=u"Cuba", continent=north_america) + CountryName.objects.get_or_create(slug="CV", name=u"Cabo Verde", continent=africa) + CountryName.objects.get_or_create(slug="CW", name=u"Curaçao", continent=north_america) + CountryName.objects.get_or_create(slug="CX", name=u"Christmas Island", continent=asia) + CountryName.objects.get_or_create(slug="CY", name=u"Cyprus", continent=asia, in_eu=True) + CountryName.objects.get_or_create(slug="CZ", name=u"Czech Republic", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="DE", name=u"Germany", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="DJ", name=u"Djibouti", continent=africa) + CountryName.objects.get_or_create(slug="DK", name=u"Denmark", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="DM", name=u"Dominica", continent=north_america) + CountryName.objects.get_or_create(slug="DO", name=u"Dominican Republic", continent=north_america) + CountryName.objects.get_or_create(slug="DZ", name=u"Algeria", continent=africa) + CountryName.objects.get_or_create(slug="EC", name=u"Ecuador", continent=south_america) + CountryName.objects.get_or_create(slug="EE", name=u"Estonia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="EG", name=u"Egypt", continent=africa) + CountryName.objects.get_or_create(slug="EH", name=u"Western Sahara", continent=africa) + CountryName.objects.get_or_create(slug="ER", name=u"Eritrea", continent=africa) + CountryName.objects.get_or_create(slug="ES", name=u"Spain", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="ET", name=u"Ethiopia", continent=africa) + CountryName.objects.get_or_create(slug="FI", name=u"Finland", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="FJ", name=u"Fiji", continent=oceania) + CountryName.objects.get_or_create(slug="FK", name=u"Falkland Islands [Malvinas]", continent=south_america) + CountryName.objects.get_or_create(slug="FM", name=u"Micronesia (Federated States of)", continent=oceania) + CountryName.objects.get_or_create(slug="FO", name=u"Faroe Islands", continent=europe) + CountryName.objects.get_or_create(slug="FR", name=u"France", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="GA", name=u"Gabon", continent=africa) + CountryName.objects.get_or_create(slug="GB", name=u"United Kingdom", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="GD", name=u"Grenada", continent=north_america) + CountryName.objects.get_or_create(slug="GE", name=u"Georgia", continent=asia) + CountryName.objects.get_or_create(slug="GF", name=u"French Guiana", continent=south_america) + CountryName.objects.get_or_create(slug="GG", name=u"Guernsey", continent=europe) + CountryName.objects.get_or_create(slug="GH", name=u"Ghana", continent=africa) + CountryName.objects.get_or_create(slug="GI", name=u"Gibraltar", continent=europe) + CountryName.objects.get_or_create(slug="GL", name=u"Greenland", continent=north_america) + CountryName.objects.get_or_create(slug="GM", name=u"Gambia", continent=africa) + CountryName.objects.get_or_create(slug="GN", name=u"Guinea", continent=africa) + CountryName.objects.get_or_create(slug="GP", name=u"Guadeloupe", continent=north_america) + CountryName.objects.get_or_create(slug="GQ", name=u"Equatorial Guinea", continent=africa) + CountryName.objects.get_or_create(slug="GR", name=u"Greece", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="GS", name=u"South Georgia and the South Sandwich Islands", continent=antarctica) + CountryName.objects.get_or_create(slug="GT", name=u"Guatemala", continent=north_america) + CountryName.objects.get_or_create(slug="GU", name=u"Guam", continent=oceania) + CountryName.objects.get_or_create(slug="GW", name=u"Guinea-Bissau", continent=africa) + CountryName.objects.get_or_create(slug="GY", name=u"Guyana", continent=south_america) + CountryName.objects.get_or_create(slug="HK", name=u"Hong Kong", continent=asia) + CountryName.objects.get_or_create(slug="HM", name=u"Heard Island and McDonald Islands", continent=antarctica) + CountryName.objects.get_or_create(slug="HN", name=u"Honduras", continent=north_america) + CountryName.objects.get_or_create(slug="HR", name=u"Croatia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="HT", name=u"Haiti", continent=north_america) + CountryName.objects.get_or_create(slug="HU", name=u"Hungary", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="ID", name=u"Indonesia", continent=asia) + CountryName.objects.get_or_create(slug="IE", name=u"Ireland", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="IL", name=u"Israel", continent=asia) + CountryName.objects.get_or_create(slug="IM", name=u"Isle of Man", continent=europe) + CountryName.objects.get_or_create(slug="IN", name=u"India", continent=asia) + CountryName.objects.get_or_create(slug="IO", name=u"British Indian Ocean Territory", continent=asia) + CountryName.objects.get_or_create(slug="IQ", name=u"Iraq", continent=asia) + CountryName.objects.get_or_create(slug="IR", name=u"Iran", continent=asia) + CountryName.objects.get_or_create(slug="IS", name=u"Iceland", continent=europe) + CountryName.objects.get_or_create(slug="IT", name=u"Italy", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="JE", name=u"Jersey", continent=europe) + CountryName.objects.get_or_create(slug="JM", name=u"Jamaica", continent=north_america) + CountryName.objects.get_or_create(slug="JO", name=u"Jordan", continent=asia) + CountryName.objects.get_or_create(slug="JP", name=u"Japan", continent=asia) + CountryName.objects.get_or_create(slug="KE", name=u"Kenya", continent=africa) + CountryName.objects.get_or_create(slug="KG", name=u"Kyrgyzstan", continent=asia) + CountryName.objects.get_or_create(slug="KH", name=u"Cambodia", continent=asia) + CountryName.objects.get_or_create(slug="KI", name=u"Kiribati", continent=oceania) + CountryName.objects.get_or_create(slug="KM", name=u"Comoros", continent=africa) + CountryName.objects.get_or_create(slug="KN", name=u"Saint Kitts and Nevis", continent=north_america) + CountryName.objects.get_or_create(slug="KP", name=u"North Korea", continent=asia) + CountryName.objects.get_or_create(slug="KR", name=u"South Korea", continent=asia) + CountryName.objects.get_or_create(slug="KW", name=u"Kuwait", continent=asia) + CountryName.objects.get_or_create(slug="KY", name=u"Cayman Islands", continent=north_america) + CountryName.objects.get_or_create(slug="KZ", name=u"Kazakhstan", continent=asia) + CountryName.objects.get_or_create(slug="LA", name=u"Laos", continent=asia) + CountryName.objects.get_or_create(slug="LB", name=u"Lebanon", continent=asia) + CountryName.objects.get_or_create(slug="LC", name=u"Saint Lucia", continent=north_america) + CountryName.objects.get_or_create(slug="LI", name=u"Liechtenstein", continent=europe) + CountryName.objects.get_or_create(slug="LK", name=u"Sri Lanka", continent=asia) + CountryName.objects.get_or_create(slug="LR", name=u"Liberia", continent=africa) + CountryName.objects.get_or_create(slug="LS", name=u"Lesotho", continent=africa) + CountryName.objects.get_or_create(slug="LT", name=u"Lithuania", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="LU", name=u"Luxembourg", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="LV", name=u"Latvia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="LY", name=u"Libya", continent=africa) + CountryName.objects.get_or_create(slug="MA", name=u"Morocco", continent=africa) + CountryName.objects.get_or_create(slug="MC", name=u"Monaco", continent=europe) + CountryName.objects.get_or_create(slug="MD", name=u"Moldova", continent=europe) + CountryName.objects.get_or_create(slug="ME", name=u"Montenegro", continent=europe) + CountryName.objects.get_or_create(slug="MF", name=u"Saint Martin (French part)", continent=north_america) + CountryName.objects.get_or_create(slug="MG", name=u"Madagascar", continent=africa) + CountryName.objects.get_or_create(slug="MH", name=u"Marshall Islands", continent=oceania) + CountryName.objects.get_or_create(slug="MK", name=u"Macedonia", continent=europe) + CountryName.objects.get_or_create(slug="ML", name=u"Mali", continent=africa) + CountryName.objects.get_or_create(slug="MM", name=u"Myanmar", continent=asia) + CountryName.objects.get_or_create(slug="MN", name=u"Mongolia", continent=asia) + CountryName.objects.get_or_create(slug="MO", name=u"Macao", continent=asia) + CountryName.objects.get_or_create(slug="MP", name=u"Northern Mariana Islands", continent=oceania) + CountryName.objects.get_or_create(slug="MQ", name=u"Martinique", continent=north_america) + CountryName.objects.get_or_create(slug="MR", name=u"Mauritania", continent=africa) + CountryName.objects.get_or_create(slug="MS", name=u"Montserrat", continent=north_america) + CountryName.objects.get_or_create(slug="MT", name=u"Malta", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="MU", name=u"Mauritius", continent=africa) + CountryName.objects.get_or_create(slug="MV", name=u"Maldives", continent=asia) + CountryName.objects.get_or_create(slug="MW", name=u"Malawi", continent=africa) + CountryName.objects.get_or_create(slug="MX", name=u"Mexico", continent=north_america) + CountryName.objects.get_or_create(slug="MY", name=u"Malaysia", continent=asia) + CountryName.objects.get_or_create(slug="MZ", name=u"Mozambique", continent=africa) + CountryName.objects.get_or_create(slug="NA", name=u"Namibia", continent=africa) + CountryName.objects.get_or_create(slug="NC", name=u"New Caledonia", continent=oceania) + CountryName.objects.get_or_create(slug="NE", name=u"Niger", continent=africa) + CountryName.objects.get_or_create(slug="NF", name=u"Norfolk Island", continent=oceania) + CountryName.objects.get_or_create(slug="NG", name=u"Nigeria", continent=africa) + CountryName.objects.get_or_create(slug="NI", name=u"Nicaragua", continent=north_america) + CountryName.objects.get_or_create(slug="NL", name=u"Netherlands", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="NO", name=u"Norway", continent=europe) + CountryName.objects.get_or_create(slug="NP", name=u"Nepal", continent=asia) + CountryName.objects.get_or_create(slug="NR", name=u"Nauru", continent=oceania) + CountryName.objects.get_or_create(slug="NU", name=u"Niue", continent=oceania) + CountryName.objects.get_or_create(slug="NZ", name=u"New Zealand", continent=oceania) + CountryName.objects.get_or_create(slug="OM", name=u"Oman", continent=asia) + CountryName.objects.get_or_create(slug="PA", name=u"Panama", continent=north_america) + CountryName.objects.get_or_create(slug="PE", name=u"Peru", continent=south_america) + CountryName.objects.get_or_create(slug="PF", name=u"French Polynesia", continent=oceania) + CountryName.objects.get_or_create(slug="PG", name=u"Papua New Guinea", continent=oceania) + CountryName.objects.get_or_create(slug="PH", name=u"Philippines", continent=asia) + CountryName.objects.get_or_create(slug="PK", name=u"Pakistan", continent=asia) + CountryName.objects.get_or_create(slug="PL", name=u"Poland", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="PM", name=u"Saint Pierre and Miquelon", continent=north_america) + CountryName.objects.get_or_create(slug="PN", name=u"Pitcairn", continent=oceania) + CountryName.objects.get_or_create(slug="PR", name=u"Puerto Rico", continent=north_america) + CountryName.objects.get_or_create(slug="PS", name=u"Palestine, State of", continent=asia) + CountryName.objects.get_or_create(slug="PT", name=u"Portugal", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="PW", name=u"Palau", continent=oceania) + CountryName.objects.get_or_create(slug="PY", name=u"Paraguay", continent=south_america) + CountryName.objects.get_or_create(slug="QA", name=u"Qatar", continent=asia) + CountryName.objects.get_or_create(slug="RE", name=u"Réunion", continent=africa) + CountryName.objects.get_or_create(slug="RO", name=u"Romania", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="RS", name=u"Serbia", continent=europe) + CountryName.objects.get_or_create(slug="RU", name=u"Russia", continent=europe) + CountryName.objects.get_or_create(slug="RW", name=u"Rwanda", continent=africa) + CountryName.objects.get_or_create(slug="SA", name=u"Saudi Arabia", continent=asia) + CountryName.objects.get_or_create(slug="SB", name=u"Solomon Islands", continent=oceania) + CountryName.objects.get_or_create(slug="SC", name=u"Seychelles", continent=africa) + CountryName.objects.get_or_create(slug="SD", name=u"Sudan", continent=africa) + CountryName.objects.get_or_create(slug="SE", name=u"Sweden", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="SG", name=u"Singapore", continent=asia) + CountryName.objects.get_or_create(slug="SH", name=u"Saint Helena, Ascension and Tristan da Cunha", continent=africa) + CountryName.objects.get_or_create(slug="SI", name=u"Slovenia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="SJ", name=u"Svalbard and Jan Mayen", continent=europe) + CountryName.objects.get_or_create(slug="SK", name=u"Slovakia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="SL", name=u"Sierra Leone", continent=africa) + CountryName.objects.get_or_create(slug="SM", name=u"San Marino", continent=europe) + CountryName.objects.get_or_create(slug="SN", name=u"Senegal", continent=africa) + CountryName.objects.get_or_create(slug="SO", name=u"Somalia", continent=africa) + CountryName.objects.get_or_create(slug="SR", name=u"Suriname", continent=south_america) + CountryName.objects.get_or_create(slug="SS", name=u"South Sudan", continent=africa) + CountryName.objects.get_or_create(slug="ST", name=u"Sao Tome and Principe", continent=africa) + CountryName.objects.get_or_create(slug="SV", name=u"El Salvador", continent=north_america) + CountryName.objects.get_or_create(slug="SX", name=u"Sint Maarten (Dutch part)", continent=north_america) + CountryName.objects.get_or_create(slug="SY", name=u"Syria", continent=asia) + CountryName.objects.get_or_create(slug="SZ", name=u"Swaziland", continent=africa) + CountryName.objects.get_or_create(slug="TC", name=u"Turks and Caicos Islands", continent=north_america) + CountryName.objects.get_or_create(slug="TD", name=u"Chad", continent=africa) + CountryName.objects.get_or_create(slug="TF", name=u"French Southern Territories", continent=antarctica) + CountryName.objects.get_or_create(slug="TG", name=u"Togo", continent=africa) + CountryName.objects.get_or_create(slug="TH", name=u"Thailand", continent=asia) + CountryName.objects.get_or_create(slug="TJ", name=u"Tajikistan", continent=asia) + CountryName.objects.get_or_create(slug="TK", name=u"Tokelau", continent=oceania) + CountryName.objects.get_or_create(slug="TL", name=u"Timor-Leste", continent=asia) + CountryName.objects.get_or_create(slug="TM", name=u"Turkmenistan", continent=asia) + CountryName.objects.get_or_create(slug="TN", name=u"Tunisia", continent=africa) + CountryName.objects.get_or_create(slug="TO", name=u"Tonga", continent=oceania) + CountryName.objects.get_or_create(slug="TR", name=u"Turkey", continent=europe) + CountryName.objects.get_or_create(slug="TT", name=u"Trinidad and Tobago", continent=north_america) + CountryName.objects.get_or_create(slug="TV", name=u"Tuvalu", continent=oceania) + CountryName.objects.get_or_create(slug="TW", name=u"Taiwan", continent=asia) + CountryName.objects.get_or_create(slug="TZ", name=u"Tanzania", continent=africa) + CountryName.objects.get_or_create(slug="UA", name=u"Ukraine", continent=europe) + CountryName.objects.get_or_create(slug="UG", name=u"Uganda", continent=africa) + CountryName.objects.get_or_create(slug="UM", name=u"United States Minor Outlying Islands", continent=oceania) + CountryName.objects.get_or_create(slug="US", name=u"United States of America", continent=north_america) + CountryName.objects.get_or_create(slug="UY", name=u"Uruguay", continent=south_america) + CountryName.objects.get_or_create(slug="UZ", name=u"Uzbekistan", continent=asia) + CountryName.objects.get_or_create(slug="VA", name=u"Holy See", continent=europe) + CountryName.objects.get_or_create(slug="VC", name=u"Saint Vincent and the Grenadines", continent=north_america) + CountryName.objects.get_or_create(slug="VE", name=u"Venezuela", continent=south_america) + CountryName.objects.get_or_create(slug="VG", name=u"Virgin Islands (British)", continent=north_america) + CountryName.objects.get_or_create(slug="VI", name=u"Virgin Islands (U.S.)", continent=north_america) + CountryName.objects.get_or_create(slug="VN", name=u"Vietnam", continent=asia) + CountryName.objects.get_or_create(slug="VU", name=u"Vanuatu", continent=oceania) + CountryName.objects.get_or_create(slug="WF", name=u"Wallis and Futuna", continent=oceania) + CountryName.objects.get_or_create(slug="WS", name=u"Samoa", continent=oceania) + CountryName.objects.get_or_create(slug="YE", name=u"Yemen", continent=asia) + CountryName.objects.get_or_create(slug="YT", name=u"Mayotte", continent=africa) + CountryName.objects.get_or_create(slug="ZA", name=u"South Africa", continent=africa) + CountryName.objects.get_or_create(slug="ZM", name=u"Zambia", continent=africa) + CountryName.objects.get_or_create(slug="ZW", name=u"Zimbabwe", continent=africa) + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0022_continentname_countryname'), + ] + + operations = [ + migrations.RunPython(insert_initial_country_continent_names, migrations.RunPython.noop) + ] diff --git a/ietf/name/models.py b/ietf/name/models.py index ec0b6905c..75146ed96 100644 --- a/ietf/name/models.py +++ b/ietf/name/models.py @@ -46,6 +46,8 @@ class StdLevelName(NameModel): class IntendedStdLevelName(NameModel): """Proposed Standard, (Draft Standard), Internet Standard, Experimental, Informational, Best Current Practice, Historic, ...""" +class FormalLanguageName(NameModel): + """ABNF, ASN.1, C code, CBOR, JSON, XML, ...""" class DocReminderTypeName(NameModel): "Stream state" class BallotPositionName(NameModel): @@ -99,3 +101,10 @@ class ReviewResultName(NameModel): Ready with nits, Serious Issues""" class TopicAudienceName(NameModel): """General, Nominee, Nomcom Member""" +class ContinentName(NameModel): + "Africa, Antarctica, Asia, ..." +class CountryName(NameModel): + "Afghanistan, Aaland Islands, Albania, ..." + continent = models.ForeignKey(ContinentName) + in_eu = models.BooleanField(verbose_name="In EU", default=False) + diff --git a/ietf/name/resources.py b/ietf/name/resources.py index fc4253685..3bfe4557c 100644 --- a/ietf/name/resources.py +++ b/ietf/name/resources.py @@ -15,7 +15,7 @@ from ietf.name.models import (TimeSlotTypeName, GroupStateName, DocTagName, Inte LiaisonStatementTagName, FeedbackTypeName, LiaisonStatementState, StreamName, BallotPositionName, DBTemplateTypeName, NomineePositionStateName, ReviewRequestStateName, ReviewTypeName, ReviewResultName, - TopicAudienceName, ) + TopicAudienceName, FormalLanguageName, ContinentName, CountryName) class TimeSlotTypeNameResource(ModelResource): @@ -471,3 +471,51 @@ class TopicAudienceNameResource(ModelResource): } api.name.register(TopicAudienceNameResource()) +class FormalLanguageNameResource(ModelResource): + class Meta: + queryset = FormalLanguageName.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'formallanguagename' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(FormalLanguageNameResource()) + +class ContinentNameResource(ModelResource): + class Meta: + queryset = ContinentName.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'continentname' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(ContinentNameResource()) + +class CountryNameResource(ModelResource): + continent = ToOneField(ContinentNameResource, 'continent') + class Meta: + queryset = CountryName.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'countryname' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + "in_eu": ALL, + "continent": ALL_WITH_RELATIONS, + } +api.name.register(CountryNameResource()) + diff --git a/ietf/person/admin.py b/ietf/person/admin.py index 8c5ce62c0..e51427afa 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -32,4 +32,3 @@ class PersonAdmin(admin.ModelAdmin): inlines = [ EmailInline, AliasInline, ] # actions = None admin.site.register(Person, PersonAdmin) - diff --git a/ietf/person/models.py b/ietf/person/models.py index 6539e7365..e29e07826 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -16,7 +16,7 @@ from django.utils.text import slugify import debug # pyflakes:ignore -from ietf.person.name import name_parts, initials +from ietf.person.name import name_parts, initials, plain_name from ietf.utils.mail import send_mail_preformatted from ietf.utils.storage import NoLocationMigrationFileSystemStorage from ietf.utils.mail import formataddr @@ -51,8 +51,7 @@ class PersonInfo(models.Model): return (first and first[0]+"." or "")+(middle or "")+" "+last+(suffix and " "+suffix or "") def plain_name(self): if not hasattr(self, '_cached_plain_name'): - prefix, first, middle, last, suffix = name_parts(self.name) - self._cached_plain_name = u" ".join([first, last]) + self._cached_plain_name = plain_name(self.name) return self._cached_plain_name def ascii_name(self): if not hasattr(self, '_cached_ascii_name'): @@ -137,18 +136,18 @@ class PersonInfo(models.Model): def has_drafts(self): from ietf.doc.models import Document - return Document.objects.filter(authors__person=self, type='draft').exists() + return Document.objects.filter(documentauthor__person=self, type='draft').exists() def rfcs(self): from ietf.doc.models import Document - rfcs = list(Document.objects.filter(authors__person=self, type='draft', states__slug='rfc')) + rfcs = list(Document.objects.filter(documentauthor__person=self, type='draft', states__slug='rfc')) rfcs.sort(key=lambda d: d.canonical_name() ) return rfcs def active_drafts(self): from ietf.doc.models import Document - return Document.objects.filter(authors__person=self, type='draft', states__slug='active').order_by('-time') + return Document.objects.filter(documentauthor__person=self, type='draft', states__slug='active').order_by('-time') def expired_drafts(self): from ietf.doc.models import Document - return Document.objects.filter(authors__person=self, type='draft', states__slug__in=['repl', 'expired', 'auth-rm', 'ietf-rm']).order_by('-time') + return Document.objects.filter(documentauthor__person=self, type='draft', states__slug__in=['repl', 'expired', 'auth-rm', 'ietf-rm']).order_by('-time') class Meta: abstract = True @@ -244,7 +243,11 @@ class Email(models.Model): return self.address def name_and_email(self): - "Returns name and email, e.g.: u'Ano Nymous ' " + """ + Returns name and email, e.g.: u'Ano Nymous ' + Is intended for display use, not in email context. + Use self.formatted_email() for that. + """ if self.person: return u"%s <%s>" % (self.person.plain_name(), self.address) else: @@ -260,15 +263,11 @@ class Email(models.Model): else: return self.address - def invalid_address(self): - # we have some legacy authors with unknown email addresses - return self.address.startswith("unknown-email") and "@" not in self.address - def email_address(self): """Get valid, current email address; in practise, for active, non-invalid addresses it is just the address field. In other cases, we default to person's email address.""" - if self.invalid_address() or not self.active: + if not self.active: if self.person: return self.person.email_address() return diff --git a/ietf/person/name.py b/ietf/person/name.py index a6401a2dc..14fba0a56 100644 --- a/ietf/person/name.py +++ b/ietf/person/name.py @@ -54,6 +54,10 @@ def initials(name): initials = u" ".join([ n[0]+'.' for n in given.split() ]) return initials +def plain_name(name): + prefix, first, middle, last, suffix = name_parts(name) + return u" ".join([first, last]) + if __name__ == "__main__": import sys name = u" ".join(sys.argv[1:]) diff --git a/ietf/person/resources.py b/ietf/person/resources.py index e0c8608fc..11ef747bf 100644 --- a/ietf/person/resources.py +++ b/ietf/person/resources.py @@ -6,7 +6,7 @@ from tastypie.cache import SimpleCache from ietf import api -from ietf.person.models import Person, Email, Alias, PersonHistory +from ietf.person.models import (Person, Email, Alias, PersonHistory) from ietf.utils.resources import UserResource @@ -82,4 +82,3 @@ class PersonHistoryResource(ModelResource): "user": ALL_WITH_RELATIONS, } api.person.register(PersonHistoryResource()) - diff --git a/ietf/review/utils.py b/ietf/review/utils.py index 848892609..43d3d474b 100644 --- a/ietf/review/utils.py +++ b/ietf/review/utils.py @@ -754,7 +754,7 @@ def make_assignment_choices(email_queryset, review_req): connections[r.person_id] = "is group {}".format(r.name) if doc.shepherd: connections[doc.shepherd.person_id] = "is shepherd of document" - for author in DocumentAuthor.objects.filter(document=doc, author__person__in=possible_person_ids).values_list("author__person", flat=True): + for author in DocumentAuthor.objects.filter(document=doc, person__in=possible_person_ids).values_list("person", flat=True): connections[author] = "is author of document" # unavailable periods diff --git a/ietf/secr/drafts/email.py b/ietf/secr/drafts/email.py index 07b706888..49583d990 100644 --- a/ietf/secr/drafts/email.py +++ b/ietf/secr/drafts/email.py @@ -48,12 +48,12 @@ def get_authors(draft): Takes a draft object and returns a list of authors suitable for a tombstone document """ authors = [] - for a in draft.authors.all(): + for a in draft.documentauthor_set.all(): initial = '' prefix, first, middle, last, suffix = a.person.name_parts() if first: initial = first + '. ' - entry = '%s%s <%s>' % (initial,last,a.address) + entry = '%s%s <%s>' % (initial,last,a.email.address) authors.append(entry) return authors @@ -64,10 +64,10 @@ def get_abbr_authors(draft): """ initial = '' result = '' - authors = DocumentAuthor.objects.filter(document=draft) + authors = DocumentAuthor.objects.filter(document=draft).order_by("order") if authors: - prefix, first, middle, last, suffix = authors[0].author.person.name_parts() + prefix, first, middle, last, suffix = authors[0].person.name_parts() if first: initial = first[0] + '. ' result = '%s%s' % (initial,last) @@ -140,9 +140,9 @@ def get_fullcc_list(draft): """ emails = {} # get authors - for author in draft.authors.all(): - if author.address not in emails: - emails[author.address] = '"%s"' % (author.person.name) + for author in draft.documentauthor_set.all(): + if author.email and author.email.address not in emails: + emails[author.email.address] = '"%s"' % (author.person.name) if draft.group.acronym != 'none': # add chairs diff --git a/ietf/secr/drafts/forms.py b/ietf/secr/drafts/forms.py index 801355748..f05cebb6a 100644 --- a/ietf/secr/drafts/forms.py +++ b/ietf/secr/drafts/forms.py @@ -104,6 +104,8 @@ class AuthorForm(forms.Form): ''' person = forms.CharField(max_length=50,widget=forms.TextInput(attrs={'class':'name-autocomplete'}),help_text="To see a list of people type the first name, or last name, or both.") email = forms.CharField(widget=forms.Select(),help_text="Select an email.") + affiliation = forms.CharField(max_length=100, required=False, help_text="Affiliation") + country = forms.CharField(max_length=255, required=False, help_text="Country") # check for id within parenthesis to ensure name was selected from the list def clean_person(self): diff --git a/ietf/secr/drafts/views.py b/ietf/secr/drafts/views.py index 281227cb8..3fa8ced44 100644 --- a/ietf/secr/drafts/views.py +++ b/ietf/secr/drafts/views.py @@ -540,7 +540,7 @@ def approvals(request): @role_required('Secretariat') def author_delete(request, id, oid): ''' - This view deletes the specified author(email) from the draft + This view deletes the specified author from the draft ''' DocumentAuthor.objects.get(id=oid).delete() messages.success(request, 'The author was deleted successfully') @@ -574,13 +574,17 @@ def authors(request, id): return redirect('ietf.secr.drafts.views.view', id=id) if form.is_valid(): - author = form.cleaned_data['email'] + person = form.cleaned_data['person'] + email = form.cleaned_data['email'] + affiliation = form.cleaned_data.get('affiliation') or "" + country = form.cleaned_data.get('country') or "" + authors = draft.documentauthor_set.all() if authors: order = authors.aggregate(Max('order')).values()[0] + 1 else: order = 1 - DocumentAuthor.objects.create(document=draft,author=author,order=order) + DocumentAuthor.objects.create(document=draft, person=person, email=email, affiliation=affiliation, country=country, order=order) messages.success(request, 'Author added successfully!') return redirect('ietf.secr.drafts.views.authors', id=id) diff --git a/ietf/secr/templates/drafts/authors.html b/ietf/secr/templates/drafts/authors.html index c5a4cb9ce..c47cb3983 100644 --- a/ietf/secr/templates/drafts/authors.html +++ b/ietf/secr/templates/drafts/authors.html @@ -24,6 +24,8 @@ Name Email + Affiliation + Country Order Action @@ -31,8 +33,10 @@ {% for author in draft.documentauthor_set.all %} - {{ author.author.person }} - {{ author.author }} + {{ author.person }} + {{ author.email }} + {{ author.affiliation }} + {{ author.country.name }} {{ author.order }} Delete @@ -49,6 +53,10 @@ {{ form.person.errors }}{{ form.person }}{% if form.person.help_text %}
{{ form.person.help_text }}{% endif %} {{ form.email.errors }}{{ form.email }}{% if form.email.help_text %}
{{ form.email.help_text }}{% endif %} + + + {{ form.affiliation.errors }}{{ form.affiliation }}{% if form.affiliation.help_text %}
{{ form.affiliation.help_text }}{% endif %} + {{ form.country.errors }}{{ form.country }}{% if form.country.help_text %}
{{ form.country.help_text }}{% endif %} diff --git a/ietf/secr/templates/drafts/makerfc.html b/ietf/secr/templates/drafts/makerfc.html index 962f76093..f5945d558 100644 --- a/ietf/secr/templates/drafts/makerfc.html +++ b/ietf/secr/templates/drafts/makerfc.html @@ -52,7 +52,7 @@

Author(s)

- {% for author in draft.authors.all %} + {% for author in draft.documentauthor_set.all %} {% endfor %}
{{ author.person.name }}
diff --git a/ietf/secr/templates/drafts/view.html b/ietf/secr/templates/drafts/view.html index 627832271..2f90838eb 100644 --- a/ietf/secr/templates/drafts/view.html +++ b/ietf/secr/templates/drafts/view.html @@ -73,7 +73,7 @@

Author(s)

{% for author in draft.documentauthor_set.all %} - + {% endfor %}
{{ author.author.person.name }}
{{ author.person.name }}
diff --git a/ietf/settings.py b/ietf/settings.py index 39eca7565..5493189fc 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -398,6 +398,7 @@ INSTALLED_APPS = ( 'ietf.redirects', 'ietf.release', 'ietf.review', + 'ietf.stats', 'ietf.submit', 'ietf.sync', 'ietf.utils', @@ -487,8 +488,9 @@ TEST_CODE_COVERAGE_EXCLUDE = [ "ietf/settings*", "ietf/utils/templatetags/debug_filters.py", "ietf/utils/test_runner.py", - "name/generate_fixtures.py", - "review/import_from_review_tool.py", + "ietf/name/generate_fixtures.py", + "ietf/review/import_from_review_tool.py", + "ietf/stats/backfill_data.py", ] # These are filename globs. They are used by test_parse_templates() and @@ -557,6 +559,8 @@ INTERNET_DRAFT_ARCHIVE_DIR = '/a/www/www6s/draft-archive' INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/a/www/www6s/archive/id' MEETING_RECORDINGS_DIR = '/a/www/audio' +DOCUMENT_FORMAT_WHITELIST = ["txt", "ps", "pdf", "xml", "html", ] + # Mailing list info URL for lists hosted on the IETF servers MAILING_LIST_INFO_URL = "https://www.ietf.org/mailman/listinfo/%(list_addr)s" MAILING_LIST_ARCHIVE_URL = "https://mailarchive.ietf.org" @@ -619,6 +623,8 @@ AUDIO_IMPORT_EMAIL = ['agenda@ietf.org','ietf@meetecho.com'] IANA_EVAL_EMAIL = "drafts-eval@icann.org" SESSION_REQUEST_FROM_EMAIL = 'IETF Meeting Session Request Tool ' +SECRETARIAT_TICKET_EMAIL = "ietf-action@ietf.org" + # Put real password in settings_local.py IANA_SYNC_PASSWORD = "secret" IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" @@ -883,6 +889,8 @@ SILENCED_SYSTEM_CHECKS = [ "fields.W342", # Setting unique=True on a ForeignKey has the same effect as using a OneToOneField. ] +STATS_NAMES_LIMIT = 25 + # Put the production SECRET_KEY in settings_local.py, and also any other # sensitive or site-specific changes. DO NOT commit settings_local.py to svn. from settings_local import * # pyflakes:ignore pylint: disable=wildcard-import diff --git a/ietf/static/ietf/css/ietf.css b/ietf/static/ietf/css/ietf.css index 0cda09bad..8d014938f 100644 --- a/ietf/static/ietf/css/ietf.css +++ b/ietf/static/ietf/css/ietf.css @@ -580,6 +580,15 @@ table.simple-table td:last-child { width: 7em; } +.document-stats .popover .element { + padding-left: 1em; + text-indent: -1em; +} + +.document-stats #chart { + height: 25em; +} + .stats-time-graph { height: 15em; } diff --git a/ietf/static/ietf/js/stats.js b/ietf/static/ietf/js/stats.js new file mode 100644 index 000000000..931e9ebf8 --- /dev/null +++ b/ietf/static/ietf/js/stats.js @@ -0,0 +1,60 @@ +$(document).ready(function () { + if (window.chartConf) { + window.chartConf.credits = { + enabled: false + }; + window.chartConf.exporting = { + fallbackToExportServer: false + }; + + if (!window.chartConf.legend) + window.chartConf.legend = { + enabled: false + }; + + var chart = Highcharts.chart('chart', window.chartConf); + } + + if (window.pieChartConf) { + window.pieChartConf.credits = { + enabled: false + }; + var pieChart = Highcharts.chart('pie-chart', window.pieChartConf); + } + /* + $(".popover-details").each(function () { + var stdNameRegExp = new RegExp("^(rfc|bcp|fyi|std)[0-9]+$", 'i'); + var draftRegExp = new RegExp("^draft-", 'i'); + + var html = []; + $.each(($(this).data("elements") || "").split("|"), function (i, element) { + if (!$.trim(element)) + return; + + if (draftRegExp.test(element) || stdNameRegExp.test(element)) { + var displayName = element; + + if (stdNameRegExp.test(element)) + displayName = element.slice(0, 3).toUpperCase() + " " + element.slice(3); + + html.push('
' + displayName + '
'); + } + else { + html.push('
' + element + '
'); + } + }); + + if ($(this).data("sliced")) + html.push('
…
'); + + $(this).popover({ + trigger: "focus", + template: '', + content: html.join(""), + placement: "top", + html: true + }).on("click", function (e) { + e.preventDefault(); + }); + });*/ +}); diff --git a/ietf/stats/admin.py b/ietf/stats/admin.py new file mode 100644 index 000000000..57f489746 --- /dev/null +++ b/ietf/stats/admin.py @@ -0,0 +1,22 @@ +from django.contrib import admin + +from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias + + +class AffiliationAliasAdmin(admin.ModelAdmin): + list_filter = ["name"] + list_display = ["alias", "name"] + search_fields = ["alias", "name"] +admin.site.register(AffiliationAlias, AffiliationAliasAdmin) + +class AffiliationIgnoredEndingAdmin(admin.ModelAdmin): + list_display = ["ending"] + search_fields = ["ending"] +admin.site.register(AffiliationIgnoredEnding, AffiliationIgnoredEndingAdmin) + +class CountryAliasAdmin(admin.ModelAdmin): + list_filter = ["country"] + list_display = ["alias", "country"] + search_fields = ["alias", "country__name"] +admin.site.register(CountryAlias, CountryAliasAdmin) + diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py new file mode 100755 index 000000000..28ad76017 --- /dev/null +++ b/ietf/stats/backfill_data.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python + +from __future__ import print_function, unicode_literals + +import sys +import os +import os.path +import argparse +import time + +basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +sys.path = [ basedir ] + sys.path +os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" + +virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py") +if os.path.exists(virtualenv_activation): + execfile(virtualenv_activation, dict(__file__=virtualenv_activation)) + +import django +django.setup() + +from django.conf import settings + +import debug # pyflakes:ignore + +from ietf.doc.models import Document +from ietf.name.models import FormalLanguageName +from ietf.utils.draft import Draft + +parser = argparse.ArgumentParser() +parser.add_argument("--document", help="specific document name") +parser.add_argument("--words", action="store_true", help="fill in word count") +parser.add_argument("--formlang", action="store_true", help="fill in formal languages") +parser.add_argument("--authors", action="store_true", help="fill in author info") +args = parser.parse_args() + +formal_language_dict = { l.pk: l for l in FormalLanguageName.objects.all() } + +docs_qs = Document.objects.filter(type="draft") + +if args.document: + docs_qs = docs_qs.filter(docalias__name=args.document) + +ts = time.strftime("%Y-%m-%d_%H:%M%z") +logfile = open('backfill-authorstats-%s.log'%ts, 'w') +print("Writing log to %s" % os.path.abspath(logfile.name)) + +def say(msg): + msg = msg.encode('utf8') + sys.stderr.write(msg) + sys.stderr.write('\n') + logfile.write(msg) + logfile.write('\n') + +def unicode(text): + if text is None: + return text + # order matters here: + for encoding in ['ascii', 'utf8', 'latin1', ]: + try: + utext = text.decode(encoding) + if encoding == 'latin1': + say("Warning: falling back to latin1 decoding for %s" % utext) + return utext + except UnicodeDecodeError: + pass + +start = time.time() +for doc in docs_qs.prefetch_related("docalias_set", "formal_languages", "documentauthor_set", "documentauthor_set__person", "documentauthor_set__person__alias_set"): + canonical_name = doc.name + for n in doc.docalias_set.all(): + if n.name.startswith("rfc"): + canonical_name = n.name + + if canonical_name.startswith("rfc"): + path = os.path.join(settings.RFC_PATH, canonical_name + ".txt") + else: + path = os.path.join(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR, canonical_name + "-" + doc.rev + ".txt") + + if not os.path.exists(path): + say("Skipping %s, no txt file found at %s" % (doc.name, path)) + continue + + with open(path, 'r') as f: + say("\nProcessing %s" % doc.name) + sys.stdout.flush() + d = Draft(f.read(), path) + + updated = False + + updates = {} + + if args.words: + words = d.get_wordcount() + if words != doc.words: + updates["words"] = words + + if args.formlang: + langs = d.get_formal_languages() + + new_formal_languages = set(formal_language_dict[l] for l in langs) + old_formal_languages = set(doc.formal_languages.all()) + + if new_formal_languages != old_formal_languages: + for l in new_formal_languages - old_formal_languages: + doc.formal_languages.add(l) + updated = True + for l in old_formal_languages - new_formal_languages: + doc.formal_languages.remove(l) + updated = True + + if args.authors: + old_authors = doc.documentauthor_set.all() + old_authors_by_name = {} + old_authors_by_email = {} + for author in old_authors: + for alias in author.person.alias_set.all(): + old_authors_by_name[alias.name] = author + old_authors_by_name[author.person.plain_name()] = author + + if author.email_id: + old_authors_by_email[author.email_id] = author + + # the draft parser sometimes has a problem when + # affiliation isn't in the second line and it then thinks + # it's an extra author - skip those extra authors + seen = set() + for full, _, _, _, _, email, country, company in d.get_author_list(): + full, email, country, company = [ unicode(s) for s in [full, email, country, company, ] ] + if email in seen: + continue + seen.add(email) + + old_author = None + if email: + old_author = old_authors_by_email.get(email) + if not old_author: + old_author = old_authors_by_name.get(full) + + if not old_author: + say("UNKNOWN AUTHOR: %s, %s, %s, %s, %s" % (doc.name, full, email, country, company)) + continue + + if old_author.affiliation != company: + say("new affiliation: %s [ %s <%s> ] %s -> %s" % (canonical_name, full, email, old_author.affiliation, company)) + old_author.affiliation = company + old_author.save(update_fields=["affiliation"]) + updated = True + + if country is None: + country = "" + + if old_author.country != country: + say("new country: %s [ %s <%s> ] %s -> %s" % (canonical_name , full, email, old_author.country, country)) + old_author.country = country + old_author.save(update_fields=["country"]) + updated = True + + + if updates: + Document.objects.filter(pk=doc.pk).update(**updates) + updated = True + + if updated: + say("updated: %s" % canonical_name) + +stop = time.time() +dur = stop-start +sec = dur%60 +min = dur//60 +say("Processing time %d:%02d" % (min, sec)) + +print("\n\nWrote log to %s" % os.path.abspath(logfile.name)) +logfile.close() + diff --git a/ietf/stats/management/__init__.py b/ietf/stats/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ietf/stats/management/commands/__init__.py b/ietf/stats/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ietf/stats/management/commands/fetch_meeting_attendance.py b/ietf/stats/management/commands/fetch_meeting_attendance.py new file mode 100755 index 000000000..52b286c51 --- /dev/null +++ b/ietf/stats/management/commands/fetch_meeting_attendance.py @@ -0,0 +1,44 @@ +# Copyright 2016 IETF Trust + +import syslog + +from django.core.management.base import BaseCommand, CommandError + +import debug # pyflakes:ignore + +from ietf.meeting.models import Meeting +from ietf.stats.utils import get_meeting_registration_data + +logtag = __name__.split('.')[-1] +logname = "user.log" +syslog.openlog(logtag, syslog.LOG_PID, syslog.LOG_USER) + +class Command(BaseCommand): + help = "Fetch meeting attendee figures from ietf.org/registration/attendees." + + def add_arguments(self, parser): + parser.add_argument("--meeting", help="meeting to fetch data for") + parser.add_argument("--all", action="store_true", help="fetch data for all meetings") + parser.add_argument("--latest", type=int, help="fetch data for latest N meetings") + + def handle(self, *args, **options): + self.verbosity = options['verbosity'] + + meetings = Meeting.objects.none() + if options['meeting']: + meetings = Meeting.objects.filter(number=options['meeting'], type="ietf") + elif options['all']: + meetings = Meeting.objects.filter(type="ietf").order_by("date") + elif options['latest']: + meetings = Meeting.objects.filter(type="ietf").order_by("-date")[:options['latest']] + else: + raise CommandError("Please use one of --meeting, --all or --latest") + + for meeting in meetings: + added, processed, total = get_meeting_registration_data(meeting) + msg = "Fetched data for meeting %3s: %4d processed, %4d added, %4d in table" % (meeting.number, processed, added, total) + if self.stdout.isatty(): + self.stdout.write(msg+'\n') # make debugging a bit easier + else: + syslog.syslog(msg) + diff --git a/ietf/stats/migrations/0001_initial.py b/ietf/stats/migrations/0001_initial.py new file mode 100644 index 000000000..19f1b49a9 --- /dev/null +++ b/ietf/stats/migrations/0001_initial.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0023_add_country_continent_names'), + ] + + operations = [ + migrations.CreateModel( + name='AffiliationAlias', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('alias', models.CharField(help_text=b"Note that aliases will be matched case-insensitive and both before and after some clean-up.", max_length=255, unique=True)), + ('name', models.CharField(max_length=255)), + ], + options={'verbose_name_plural': 'affiliation aliases'}, + ), + migrations.CreateModel( + name='AffiliationIgnoredEnding', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('ending', models.CharField(help_text=b"Regexp with ending, e.g. 'Inc\\.?' - remember to escape .!", max_length=255)), + ], + ), + migrations.CreateModel( + name='CountryAlias', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('alias', models.CharField(help_text=b"Note that lower-case aliases are matched case-insensitive while aliases with at least one uppercase letter is matched case-sensitive. So 'United States' is best entered as 'united states' so it both matches 'United States' and 'United states' and 'UNITED STATES', whereas 'US' is best entered as 'US' so it doesn't accidentally match an ordinary word like 'us'.", max_length=255)), + ('country', models.ForeignKey(to='name.CountryName', max_length=255)), + ], + options={'verbose_name_plural': 'country aliases'}, + ), + ] diff --git a/ietf/stats/migrations/0002_add_initial_aliases.py b/ietf/stats/migrations/0002_add_initial_aliases.py new file mode 100644 index 000000000..d317267d8 --- /dev/null +++ b/ietf/stats/migrations/0002_add_initial_aliases.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def add_affiliation_info(apps, schema_editor): + AffiliationAlias = apps.get_model("stats", "AffiliationAlias") + + AffiliationAlias.objects.get_or_create(alias="cisco", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco system", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco systems (india) private limited", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco systems india pvt", name="Cisco Systems") + + AffiliationIgnoredEnding = apps.get_model("stats", "AffiliationIgnoredEnding") + AffiliationIgnoredEnding.objects.get_or_create(ending="LLC\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="Ltd\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="Inc\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="GmbH\.?") + + CountryAlias = apps.get_model("stats", "CountryAlias") + for iso_country_code in ['AD', 'AE', 'AF', 'AG', 'AI', 'AL', 'AM', 'AO', 'AQ', 'AR', 'AS', 'AT', 'AU', 'AW', + 'AX', 'AZ', 'BA', 'BB', 'BD', 'BE', 'BF', 'BG', 'BH', 'BI', 'BJ', 'BL', 'BM', 'BN', + 'BO', 'BQ', 'BR', 'BS', 'BT', 'BV', 'BW', 'BY', 'BZ', 'CA', 'CC', 'CD', 'CF', 'CG', + 'CH', 'CI', 'CK', 'CL', 'CM', 'CN', 'CO', 'CR', 'CU', 'CV', 'CW', 'CX', 'CY', 'CZ', + 'DE', 'DJ', 'DK', 'DM', 'DO', 'DZ', 'EC', 'EE', 'EG', 'EH', 'ER', 'ES', 'ET', 'FI', + 'FJ', 'FK', 'FM', 'FO', 'FR', 'GA', 'GB', 'GD', 'GE', 'GF', 'GG', 'GH', 'GI', 'GL', + 'GM', 'GN', 'GP', 'GQ', 'GR', 'GS', 'GT', 'GU', 'GW', 'GY', 'HK', 'HM', 'HN', 'HR', + 'HT', 'HU', 'ID', 'IE', 'IL', 'IM', 'IN', 'IO', 'IQ', 'IR', 'IS', 'IT', 'JE', 'JM', + 'JO', 'JP', 'KE', 'KG', 'KH', 'KI', 'KM', 'KN', 'KP', 'KR', 'KW', 'KY', 'KZ', 'LA', + 'LB', 'LC', 'LI', 'LK', 'LR', 'LS', 'LT', 'LU', 'LV', 'LY', 'MA', 'MC', 'MD', 'ME', + 'MF', 'MG', 'MH', 'MK', 'ML', 'MM', 'MN', 'MO', 'MP', 'MQ', 'MR', 'MS', 'MT', 'MU', + 'MV', 'MW', 'MX', 'MY', 'MZ', 'NA', 'NC', 'NE', 'NF', 'NG', 'NI', 'NL', 'NO', 'NP', + 'NR', 'NU', 'NZ', 'OM', 'PA', 'PE', 'PF', 'PG', 'PH', 'PK', 'PL', 'PM', 'PN', 'PR', + 'PS', 'PT', 'PW', 'PY', 'QA', 'RE', 'RO', 'RS', 'RU', 'RW', 'SA', 'SB', 'SC', 'SD', + 'SE', 'SG', 'SH', 'SI', 'SJ', 'SK', 'SL', 'SM', 'SN', 'SO', 'SR', 'SS', 'ST', 'SV', + 'SX', 'SY', 'SZ', 'TC', 'TD', 'TF', 'TG', 'TH', 'TJ', 'TK', 'TL', 'TM', 'TN', 'TO', + 'TR', 'TT', 'TV', 'TW', 'TZ', 'UA', 'UG', 'UM', 'US', 'UY', 'UZ', 'VA', 'VC', 'VE', + 'VG', 'VI', 'VN', 'VU', 'WF', 'WS', 'YE', 'YT', 'ZA', 'ZM', 'ZW']: + CountryAlias.objects.get_or_create(alias=iso_country_code, country_id=iso_country_code) + + CountryAlias.objects.get_or_create(alias="russian federation", country_id="RU") + CountryAlias.objects.get_or_create(alias="p. r. china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r. china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r.china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r. of china", country_id="CN") + CountryAlias.objects.get_or_create(alias="PRC", country_id="CN") + CountryAlias.objects.get_or_create(alias="P.R.C", country_id="CN") + CountryAlias.objects.get_or_create(alias="P.R.C.", country_id="CN") + CountryAlias.objects.get_or_create(alias="beijing", country_id="CN") + CountryAlias.objects.get_or_create(alias="shenzhen", country_id="CN") + CountryAlias.objects.get_or_create(alias="R.O.C.", country_id="TW") + CountryAlias.objects.get_or_create(alias="usa", country_id="US") + CountryAlias.objects.get_or_create(alias="UAS", country_id="US") + CountryAlias.objects.get_or_create(alias="USA.", country_id="US") + CountryAlias.objects.get_or_create(alias="u.s.a.", country_id="US") + CountryAlias.objects.get_or_create(alias="u. s. a.", country_id="US") + CountryAlias.objects.get_or_create(alias="u.s.a", country_id="US") + CountryAlias.objects.get_or_create(alias="u.s.", country_id="US") + CountryAlias.objects.get_or_create(alias="U.S", country_id="GB") + CountryAlias.objects.get_or_create(alias="US of A", country_id="US") + CountryAlias.objects.get_or_create(alias="united sates", country_id="US") + CountryAlias.objects.get_or_create(alias="united state", country_id="US") + CountryAlias.objects.get_or_create(alias="united states", country_id="US") + CountryAlias.objects.get_or_create(alias="unites states", country_id="US") + CountryAlias.objects.get_or_create(alias="texas", country_id="US") + CountryAlias.objects.get_or_create(alias="UK", country_id="GB") + CountryAlias.objects.get_or_create(alias="united kingcom", country_id="GB") + CountryAlias.objects.get_or_create(alias="great britain", country_id="GB") + CountryAlias.objects.get_or_create(alias="england", country_id="GB") + CountryAlias.objects.get_or_create(alias="U.K.", country_id="GB") + CountryAlias.objects.get_or_create(alias="U.K", country_id="GB") + CountryAlias.objects.get_or_create(alias="Uk", country_id="GB") + CountryAlias.objects.get_or_create(alias="scotland", country_id="GB") + CountryAlias.objects.get_or_create(alias="republic of korea", country_id="KR") + CountryAlias.objects.get_or_create(alias="korea", country_id="KR") + CountryAlias.objects.get_or_create(alias="korea rep", country_id="KR") + CountryAlias.objects.get_or_create(alias="korea (the republic of)", country_id="KR") + CountryAlias.objects.get_or_create(alias="the netherlands", country_id="NL") + CountryAlias.objects.get_or_create(alias="netherland", country_id="NL") + CountryAlias.objects.get_or_create(alias="danmark", country_id="DK") + CountryAlias.objects.get_or_create(alias="sweeden", country_id="SE") + CountryAlias.objects.get_or_create(alias="swede", country_id="SE") + CountryAlias.objects.get_or_create(alias="belgique", country_id="BE") + CountryAlias.objects.get_or_create(alias="madrid", country_id="ES") + CountryAlias.objects.get_or_create(alias="espana", country_id="ES") + CountryAlias.objects.get_or_create(alias="hellas", country_id="GR") + CountryAlias.objects.get_or_create(alias="gemany", country_id="DE") + CountryAlias.objects.get_or_create(alias="deutschland", country_id="DE") + CountryAlias.objects.get_or_create(alias="italia", country_id="IT") + CountryAlias.objects.get_or_create(alias="isreal", country_id="IL") + CountryAlias.objects.get_or_create(alias="tel aviv", country_id="IL") + CountryAlias.objects.get_or_create(alias="UAE", country_id="AE") + CountryAlias.objects.get_or_create(alias="grand-duchy of luxembourg", country_id="LU") + CountryAlias.objects.get_or_create(alias="brasil", country_id="BR") + + +class Migration(migrations.Migration): + + dependencies = [ + ('stats', '0001_initial'), + ] + + operations = [ + migrations.RunPython(add_affiliation_info, migrations.RunPython.noop) + ] diff --git a/ietf/stats/migrations/0003_registration_registrationstats.py b/ietf/stats/migrations/0003_registration_registrationstats.py new file mode 100644 index 000000000..46bc6a84e --- /dev/null +++ b/ietf/stats/migrations/0003_registration_registrationstats.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.6 on 2017-03-30 14:38 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('person', '0015_clean_primary'), + ('meeting', '0047_import_shared_audio_files'), + ('stats', '0002_add_initial_aliases'), + ] + + operations = [ + migrations.CreateModel( + name='MeetingRegistration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('first_name', models.CharField(max_length=255)), + ('last_name', models.CharField(max_length=255)), + ('affiliation', models.CharField(blank=True, max_length=255)), + ('country_code', models.CharField(max_length=2)), + ('meeting', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='meeting.Meeting')), + ('person', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='person.Person')), + ], + ), + ] diff --git a/ietf/stats/migrations/__init__.py b/ietf/stats/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ietf/stats/models.py b/ietf/stats/models.py new file mode 100644 index 000000000..441d7e0cf --- /dev/null +++ b/ietf/stats/models.py @@ -0,0 +1,56 @@ +from django.db import models +from ietf.meeting.models import Meeting +from ietf.name.models import CountryName +from ietf.person.models import Person + + +class AffiliationAlias(models.Model): + """Records that alias should be treated as name for statistical + purposes.""" + + alias = models.CharField(max_length=255, help_text="Note that aliases will be matched case-insensitive and both before and after some clean-up.", unique=True) + name = models.CharField(max_length=255) + + def __unicode__(self): + return u"{} -> {}".format(self.alias, self.name) + + def save(self, *args, **kwargs): + self.alias = self.alias.lower() + super(AffiliationAlias, self).save(*args, **kwargs) + + class Meta: + verbose_name_plural = "affiliation aliases" + +class AffiliationIgnoredEnding(models.Model): + """Records that ending should be stripped from the affiliation for statistical purposes.""" + + ending = models.CharField(max_length=255, help_text="Regexp with ending, e.g. 'Inc\\.?' - remember to escape .!") + + def __unicode__(self): + return self.ending + +class CountryAlias(models.Model): + """Records that alias should be treated as country for statistical + purposes.""" + + alias = models.CharField(max_length=255, help_text="Note that lower-case aliases are matched case-insensitive while aliases with at least one uppercase letter is matched case-sensitive. So 'United States' is best entered as 'united states' so it both matches 'United States' and 'United states' and 'UNITED STATES', whereas 'US' is best entered as 'US' so it doesn't accidentally match an ordinary word like 'us'.") + country = models.ForeignKey(CountryName, max_length=255) + + def __unicode__(self): + return u"{} -> {}".format(self.alias, self.country.name) + + class Meta: + verbose_name_plural = "country aliases" + +class MeetingRegistration(models.Model): + """Registration attendee records from the IETF registration system""" + meeting = models.ForeignKey(Meeting) + first_name = models.CharField(max_length=255) + last_name = models.CharField(max_length=255) + affiliation = models.CharField(blank=True, max_length=255) + country_code = models.CharField(max_length=2) # ISO 3166 + person = models.ForeignKey(Person, blank=True, null=True) + email = models.EmailField(blank=True, null=True) + + def __unicode__(self): + return u"{} {}".format(self.first_name, self.last_name) diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py new file mode 100644 index 000000000..ab5f50803 --- /dev/null +++ b/ietf/stats/resources.py @@ -0,0 +1,70 @@ +# Autogenerated by the makeresources management command 2017-02-15 10:10 PST +from tastypie.resources import ModelResource +from tastypie.fields import ToManyField # pyflakes:ignore +from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore +from tastypie.cache import SimpleCache + +from ietf import api +from ietf.api import ToOneField # pyflakes:ignore + +from ietf.stats.models import CountryAlias, AffiliationIgnoredEnding, AffiliationAlias, MeetingRegistration + + +from ietf.name.resources import CountryNameResource +class CountryAliasResource(ModelResource): + country = ToOneField(CountryNameResource, 'country') + class Meta: + queryset = CountryAlias.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'countryalias' + filtering = { + "id": ALL, + "alias": ALL, + "country": ALL_WITH_RELATIONS, + } +api.stats.register(CountryAliasResource()) + +class AffiliationIgnoredEndingResource(ModelResource): + class Meta: + queryset = AffiliationIgnoredEnding.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'affiliationignoredending' + filtering = { + "id": ALL, + "ending": ALL, + } +api.stats.register(AffiliationIgnoredEndingResource()) + +class AffiliationAliasResource(ModelResource): + class Meta: + queryset = AffiliationAlias.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'affiliationalias' + filtering = { + "id": ALL, + "alias": ALL, + "name": ALL, + } +api.stats.register(AffiliationAliasResource()) + +class MeetingRegistrationResource(ModelResource): + class Meta: + queryset = MeetingRegistration.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'meetingregistration' + filtering = { + "id": ALL, + "meeting": ALL_WITH_RELATIONS, + "first_name": ALL, + "last_name": ALL, + "affiliation": ALL, + "country_code": ALL, + "email": ALL, + "person": ALL_WITH_RELATIONS + } +api.stats.register(MeetingRegistrationResource()) + diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 57d0faa28..ad30b4ef3 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,17 +1,149 @@ +import datetime + +from mock import patch from pyquery import PyQuery +from requests import Response from django.urls import reverse as urlreverse from ietf.utils.test_data import make_test_data, make_review_data -from ietf.utils.test_utils import login_testing_unauthorized, TestCase +from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent import ietf.stats.views +from ietf.submit.models import Submission +from ietf.doc.models import Document, DocAlias, State, RelatedDocument, NewRevisionDocEvent +from ietf.meeting.factories import MeetingFactory +from ietf.person.models import Person +from ietf.name.models import FormalLanguageName, DocRelationshipName, CountryName +from ietf.stats.models import MeetingRegistration, CountryAlias +from ietf.stats.utils import get_meeting_registration_data + class StatisticsTests(TestCase): def test_stats_index(self): url = urlreverse(ietf.stats.views.stats_index) r = self.client.get(url) self.assertEqual(r.status_code, 200) + def test_document_stats(self): + draft = make_test_data() + + # create some data for the statistics + Submission.objects.create( + authors=[ { "name": "Some Body", "email": "somebody@example.com", "affiliation": "Some Inc.", "country": "US" }], + pages=30, + rev=draft.rev, + words=4000, + draft=draft, + file_types=".txt", + state_id="posted", + ) + + draft.formal_languages.add(FormalLanguageName.objects.get(slug="xml")) + Document.objects.filter(pk=draft.pk).update(words=4000) + # move it back so it shows up in the yearly summaries + NewRevisionDocEvent.objects.filter(doc=draft, rev=draft.rev).update( + time=datetime.datetime.now() - datetime.timedelta(days=500)) + + referencing_draft = Document.objects.create( + name="draft-ietf-mars-referencing", + type_id="draft", + title="Referencing", + stream_id="ietf", + abstract="Test", + rev="00", + pages=2, + words=100 + ) + referencing_draft.set_state(State.objects.get(used=True, type="draft", slug="active")) + DocAlias.objects.create(document=referencing_draft, name=referencing_draft.name) + RelatedDocument.objects.create( + source=referencing_draft, + target=draft.docalias_set.first(), + relationship=DocRelationshipName.objects.get(slug="refinfo") + ) + NewRevisionDocEvent.objects.create( + type="new_revision", + by=Person.objects.get(name="(System)"), + doc=referencing_draft, + desc="New revision available", + rev=referencing_draft.rev, + time=datetime.datetime.now() - datetime.timedelta(days=1000) + ) + + + # check redirect + url = urlreverse(ietf.stats.views.document_stats) + + authors_url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": "authors" }) + + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertTrue(authors_url in r["Location"]) + + # check various stats types + for stats_type in ["authors", "pages", "words", "format", "formlang", + "author/documents", "author/affiliation", "author/country", + "author/continent", "author/citations", "author/hindex", + "yearly/affiliation", "yearly/country", "yearly/continent"]: + for document_type in ["", "rfc", "draft"]: + for time_choice in ["", "5y"]: + url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type }) + r = self.client.get(url, { + "type": document_type, + "time": time_choice, + }) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(q('#chart')) + if not stats_type.startswith("yearly"): + self.assertTrue(q('table.stats-data')) + + def test_meeting_stats(self): + # create some data for the statistics + make_test_data() + meeting = MeetingFactory(type_id='ietf', date=datetime.date.today(), number="96") + MeetingRegistration.objects.create(first_name='John', last_name='Smith', country_code='US', email="john.smith@example.us", meeting=meeting) + CountryAlias.objects.get_or_create(alias="US", country=CountryName.objects.get(slug="US")) + MeetingRegistration.objects.create(first_name='Jaume', last_name='Guillaume', country_code='FR', email="jaume.guillaume@example.fr", meeting=meeting) + CountryAlias.objects.get_or_create(alias="FR", country=CountryName.objects.get(slug="FR")) + + # check redirect + url = urlreverse(ietf.stats.views.meeting_stats) + + authors_url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": "overview" }) + + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertTrue(authors_url in r["Location"]) + + # check various stats types + for stats_type in ["overview", "country", "continent"]: + url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": stats_type }) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(q('#chart')) + if stats_type == "overview": + self.assertTrue(q('table.stats-data')) + + for stats_type in ["country", "continent"]: + url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": stats_type, "num": meeting.number }) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(q('#chart')) + self.assertTrue(q('table.stats-data')) + + def test_known_country_list(self): + make_test_data() + + # check redirect + url = urlreverse(ietf.stats.views.known_countries_list) + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertTrue("United States" in unicontent(r)) + def test_review_stats(self): doc = make_test_data() review_req = make_review_data(doc) @@ -56,3 +188,14 @@ class StatisticsTests(TestCase): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('.review-stats td:contains("1")')) + + @patch('requests.get') + def test_get_meeting_registration_data(self, mock_get): + response = Response() + response.status_code = 200 + response._content = '[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US","Email":"john.doe@example.us"}]' + mock_get.return_value = response + meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96") + get_meeting_registration_data(meeting) + query = MeetingRegistration.objects.filter(first_name='John',last_name='Smith',country_code='US') + self.assertTrue(query.count(), 1) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index f19c313c0..94fbe6951 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -5,5 +5,9 @@ from ietf.utils.urls import url urlpatterns = [ url("^$", views.stats_index), + url("^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", views.document_stats), + url("^knowncountries/$", views.known_countries_list), + url("^meeting/(?P\d+)/(?Pcountry|continent)/$", views.meeting_stats), + url("^meeting/(?:(?Poverview|country|continent)/)?$", views.meeting_stats), url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), ] diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py new file mode 100644 index 000000000..d700395d6 --- /dev/null +++ b/ietf/stats/utils.py @@ -0,0 +1,248 @@ +import re +import requests +from collections import defaultdict + +from django.conf import settings + +from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias, MeetingRegistration +from ietf.name.models import CountryName + +def compile_affiliation_ending_stripping_regexp(): + parts = [] + for ending_re in AffiliationIgnoredEnding.objects.values_list("ending", flat=True): + try: + re.compile(ending_re) + except re.error: + pass + + parts.append(ending_re) + + re_str = ",? *({}) *$".format("|".join(parts)) + + return re.compile(re_str, re.IGNORECASE) + + +def get_aliased_affiliations(affiliations): + """Given non-unique sequence of affiliations, returns dictionary with + aliases needed. + + We employ the following strategies, interleaved: + + - Stripping company endings like Inc., GmbH etc. from database + + - Looking up aliases stored directly in the database, like + "Examplar International" -> "Examplar" + + - Case-folding so Examplar and EXAMPLAR is merged with the + winner being the one with most occurrences (so input should not + be made unique) or most upper case letters in case of ties. + Case folding can be overridden by the aliases in the database.""" + + res = {} + + ending_re = compile_affiliation_ending_stripping_regexp() + + known_aliases = { alias.lower(): name for alias, name in AffiliationAlias.objects.values_list("alias", "name") } + + affiliations_with_case_spellings = defaultdict(set) + case_spelling_count = defaultdict(int) + for affiliation in affiliations: + original_affiliation = affiliation + + # check aliases from DB + name = known_aliases.get(affiliation.lower()) + if name is not None: + affiliation = name + res[original_affiliation] = affiliation + + # strip ending + name = ending_re.sub("", affiliation) + if name != affiliation: + affiliation = name + res[original_affiliation] = affiliation + + # check aliases from DB + name = known_aliases.get(affiliation.lower()) + if name is not None: + affiliation = name + res[original_affiliation] = affiliation + + affiliations_with_case_spellings[affiliation.lower()].add(original_affiliation) + case_spelling_count[affiliation] += 1 + + def affiliation_sort_key(affiliation): + count = case_spelling_count[affiliation] + uppercase_letters = sum(1 for c in affiliation if c.isupper()) + return (count, uppercase_letters) + + # now we just need to pick the most popular uppercase/lowercase + # spelling for each affiliation with more than one + for similar_affiliations in affiliations_with_case_spellings.itervalues(): + if len(similar_affiliations) > 1: + most_popular = sorted(similar_affiliations, key=affiliation_sort_key, reverse=True)[0] + for affiliation in similar_affiliations: + if affiliation != most_popular: + res[affiliation] = most_popular + + return res + + +def get_aliased_countries(countries): + known_aliases = dict(CountryAlias.objects.values_list("alias", "country__name")) + + # add aliases for known countries + for slug, name in CountryName.objects.values_list("slug", "name"): + known_aliases[name.lower()] = name + + def lookup_alias(possible_alias): + name = known_aliases.get(possible_alias) + if name is not None: + return name + + name = known_aliases.get(possible_alias.lower()) + if name is not None: + return name + + return possible_alias + + known_re_aliases = { + re.compile(u"\\b{}\\b".format(re.escape(alias))): name + for alias, name in known_aliases.iteritems() + } + + # specific hack: check for zip codes from the US since in the + # early days, the addresses often didn't include the country + us_zipcode_re = re.compile(r"\b(AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY|AS|GU|MP|PR|VI|UM|FM|MH|PW|Ca|Cal.|California|CALIFORNIA|Colorado|Georgia|Illinois|Ill|Maryland|Ma|Ma.|Mass|Massachuss?etts|Michigan|Minnesota|New Jersey|New York|Ny|N.Y.|North Carolina|NORTH CAROLINA|Ohio|Oregon|Pennsylvania|Tx|Texas|Tennessee|Utah|Vermont|Virginia|Va.|Washington)[., -]*[0-9]{5}\b") + + us_country_name = CountryName.objects.get(slug="US").name + + def last_text_part_stripped(split): + for t in reversed(split): + t = t.strip() + if t: + return t + return u"" + + known_countries = set(CountryName.objects.values_list("name", flat=True)) + + res = {} + + for country in countries: + if country in res or country in known_countries: + continue + + original_country = country + + # aliased name + country = lookup_alias(country) + if country in known_countries: + res[original_country] = country + continue + + # contains US zipcode + if us_zipcode_re.search(country): + res[original_country] = us_country_name + continue + + # do a little bit of cleanup + if len(country) > 1 and country[-1] == "." and not country[-2].isupper(): + country = country.rstrip(".") + + country = country.strip("-,").strip() + + # aliased name + country = lookup_alias(country) + if country in known_countries: + res[original_country] = country + continue + + # country name at end, separated by comma + last_part = lookup_alias(last_text_part_stripped(country.split(","))) + if last_part in known_countries: + res[original_country] = last_part + continue + + # country name at end, separated by whitespace + last_part = lookup_alias(last_text_part_stripped(country.split())) + if last_part in known_countries: + res[original_country] = last_part + continue + + # country name anywhere + country_lower = country.lower() + found = False + for alias_re, name in known_re_aliases.iteritems(): + if alias_re.search(country) or alias_re.search(country_lower): + res[original_country] = name + found = True + break + + if found: + continue + + # unknown country + res[original_country] = "" + + return res + + +def clean_country_name(country_name): + if country_name: + country_name = get_aliased_countries([country_name]).get(country_name, country_name) + if country_name and CountryName.objects.filter(name=country_name).exists(): + return country_name + + return "" + + +def compute_hirsch_index(citation_counts): + """Computes the h-index given a sequence containing the number of + citations for each document.""" + + i = 0 + + for count in sorted(citation_counts, reverse=True): + if i + 1 > count: + break + + i += 1 + + return i + + +def get_meeting_registration_data(meeting): + """"Retrieve registration attendee data and summary statistics. Returns number + of Registration records created.""" + num_created = 0 + num_processed = 0 + response = requests.get(settings.REGISTRATION_ATTENDEES_BASE_URL + meeting.number) + if response.status_code == 200: + decoded = [] + try: + decoded = response.json() + except ValueError: + if response.content.strip() == 'Invalid meeting': + pass + else: + raise RuntimeError("Could not decode response from registrations API: '%s...'" % (response.content[:64], )) + + for registration in decoded: + object, created = MeetingRegistration.objects.get_or_create( + meeting_id=meeting.pk, + first_name=registration['FirstName'], + last_name=registration['LastName'], + affiliation=registration['Company'], + country_code=registration['Country'], + email=registration['Email'], + ) + if created: + num_created += 1 + num_processed += 1 + else: + raise RuntimeError("Bad response from registrations API: %s, '%s'" % (response.status_code, response.content)) + num_total = MeetingRegistration.objects.filter(meeting_id=meeting.pk).count() + return num_created, num_processed, num_total + + + + diff --git a/ietf/stats/views.py b/ietf/stats/views.py index e161a408c..f567ca973 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,25 +1,966 @@ -import datetime, itertools, json, calendar - -from django.shortcuts import render -from django.contrib.auth.decorators import login_required -from django.urls import reverse as urlreverse -from django.http import HttpResponseRedirect, HttpResponseForbidden - +import os +import calendar +import datetime +import email.utils +import itertools +import json import dateutil.relativedelta +from collections import defaultdict + +from django.conf import settings +from django.contrib.auth.decorators import login_required +from django.core.cache import cache +from django.db.models import Count, Q +from django.http import HttpResponseRedirect, HttpResponseForbidden +from django.shortcuts import get_object_or_404, render +from django.urls import reverse as urlreverse +from django.utils.safestring import mark_safe + +import debug # pyflakes:ignore from ietf.review.utils import (extract_review_request_data, aggregate_raw_review_request_stats, ReviewRequestData, compute_review_request_stats, sum_raw_review_request_aggregations) +from ietf.submit.models import Submission from ietf.group.models import Role, Group from ietf.person.models import Person -from ietf.name.models import ReviewRequestStateName, ReviewResultName +from ietf.name.models import ReviewRequestStateName, ReviewResultName, CountryName, DocRelationshipName +from ietf.person.name import plain_name +from ietf.doc.models import DocAlias, Document, State, DocEvent +from ietf.meeting.models import Meeting +from ietf.stats.models import MeetingRegistration, CountryAlias +from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries, compute_hirsch_index from ietf.ietfauth.utils import has_role +from ietf.utils.log import log def stats_index(request): return render(request, "stats/index.html") +def generate_query_string(query_dict, overrides): + query_part = u"" + + if query_dict or overrides: + d = query_dict.copy() + for k, v in overrides.iteritems(): + if type(v) in (list, tuple): + if not v: + if k in d: + del d[k] + else: + d.setlist(k, v) + else: + if v is None or v == u"": + if k in d: + del d[k] + else: + d[k] = v + + if d: + query_part = u"?" + d.urlencode() + + return query_part + +def get_choice(request, get_parameter, possible_choices, multiple=False): + # the statistics are built with links to make navigation faster, + # so we don't really have a form in most cases, so just use this + # helper instead to select between the choices + values = request.GET.getlist(get_parameter) + found = [t[0] for t in possible_choices if t[0] in values] + + if multiple: + return found + else: + if found: + return found[0] + else: + return None + +def add_url_to_choices(choices, url_builder): + return [ (slug, label, url_builder(slug)) for slug, label in choices] + +def put_into_bin(value, bin_size): + if value is None: + return (value, value) + + v = (value // bin_size) * bin_size + return (v, "{} - {}".format(v, v + bin_size - 1)) + +def prune_unknown_bin_with_known(bins): + # remove from the unknown bin all authors within the + # named/known bins + all_known = { n for b, names in bins.iteritems() if b for n in names } + bins[""] = [name for name in bins[""] if name not in all_known] + if not bins[""]: + del bins[""] + +def count_bins(bins): + return len({ n for b, names in bins.iteritems() if b for n in names }) + +def add_labeled_top_series_from_bins(chart_data, bins, limit): + """Take bins on the form (x, label): [name1, name2, ...], figure out + how many there are per label, take the overall top ones and put + them into sorted series like [(x1, len(names1)), (x2, len(names2)), ...].""" + aggregated_bins = defaultdict(set) + xs = set() + for (x, label), names in bins.iteritems(): + xs.add(x) + aggregated_bins[label].update(names) + + xs = list(sorted(xs)) + + sorted_bins = sorted(aggregated_bins.iteritems(), key=lambda t: len(t[1]), reverse=True) + top = [ label for label, names in list(sorted_bins)[:limit]] + + for label in top: + series_data = [] + + for x in xs: + names = bins.get((x, label), set()) + + series_data.append((x, len(names))) + + chart_data.append({ + "data": series_data, + "name": label + }) + +def document_stats(request, stats_type=None): + def build_document_stats_url(stats_type_override=Ellipsis, get_overrides={}): + kwargs = { + "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, + } + + return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) + + cache_key = ("stats:document:%s:%s" % (stats_type, request.META.get('QUERY_STRING',''))) + data = cache.get(cache_key) + if not data: + names_limit = settings.STATS_NAMES_LIMIT + # statistics types + possible_document_stats_types = add_url_to_choices([ + ("authors", "Number of authors"), + ("pages", "Pages"), + ("words", "Words"), + ("format", "Format"), + ("formlang", "Formal languages"), + ], lambda slug: build_document_stats_url(stats_type_override=slug)) + + possible_author_stats_types = add_url_to_choices([ + ("author/documents", "Number of documents"), + ("author/affiliation", "Affiliation"), + ("author/country", "Country"), + ("author/continent", "Continent"), + ("author/citations", "Citations"), + ("author/hindex", "h-index"), + ], lambda slug: build_document_stats_url(stats_type_override=slug)) + + possible_yearly_stats_types = add_url_to_choices([ + ("yearly/affiliation", "Affiliation"), + ("yearly/country", "Country"), + ("yearly/continent", "Continent"), + ], lambda slug: build_document_stats_url(stats_type_override=slug)) + + + if not stats_type: + return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_document_stats_types[0][0])) + + + possible_document_types = add_url_to_choices([ + ("", "All"), + ("rfc", "RFCs"), + ("draft", "Drafts"), + ], lambda slug: build_document_stats_url(get_overrides={ "type": slug })) + + document_type = get_choice(request, "type", possible_document_types) or "" + + + possible_time_choices = add_url_to_choices([ + ("", "All time"), + ("5y", "Past 5 years"), + ], lambda slug: build_document_stats_url(get_overrides={ "time": slug })) + + time_choice = request.GET.get("time") or "" + + from_time = None + if "y" in time_choice: + try: + y = int(time_choice.rstrip("y")) + from_time = datetime.datetime.today() - dateutil.relativedelta.relativedelta(years=y) + except ValueError: + pass + + chart_data = [] + table_data = [] + stats_title = "" + template_name = stats_type.replace("/", "_") + bin_size = 1 + alias_data = [] + eu_countries = None + + + if any(stats_type == t[0] for t in possible_document_stats_types): + # filter documents + docalias_filters = Q(document__type="draft") + + rfc_state = State.objects.get(type="draft", slug="rfc") + if document_type == "rfc": + docalias_filters &= Q(document__states=rfc_state) + elif document_type == "draft": + docalias_filters &= ~Q(document__states=rfc_state) + + if from_time: + # this is actually faster than joining in the database, + # despite the round-trip back and forth + docs_within_time_constraint = list(Document.objects.filter( + type="draft", + docevent__time__gte=from_time, + docevent__type__in=["published_rfc", "new_revision"], + ).values_list("pk")) + + docalias_filters &= Q(document__in=docs_within_time_constraint) + + docalias_qs = DocAlias.objects.filter(docalias_filters) + + if document_type == "rfc": + doc_label = "RFC" + elif document_type == "draft": + doc_label = "draft" + else: + doc_label = "document" + + total_docs = docalias_qs.values_list("document").distinct().count() + + def generate_canonical_names(docalias_qs): + for doc_id, ts in itertools.groupby(docalias_qs.order_by("document"), lambda t: t[0]): + chosen = None + for t in ts: + if chosen is None: + chosen = t + else: + if t[1].startswith("rfc"): + chosen = t + elif t[1].startswith("draft") and not chosen[1].startswith("rfc"): + chosen = t + + yield chosen + + if stats_type == "authors": + stats_title = "Number of authors for each {}".format(doc_label) + + bins = defaultdict(set) + + for name, canonical_name, author_count in generate_canonical_names(docalias_qs.values_list("document", "name").annotate(Count("document__documentauthor"))): + bins[author_count].add(canonical_name) + + series_data = [] + for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / (total_docs or 1) + series_data.append((author_count, percentage)) + table_data.append((author_count, percentage, len(names), list(names)[:names_limit])) + + chart_data.append({ "data": series_data }) + + elif stats_type == "pages": + stats_title = "Number of pages for each {}".format(doc_label) + + bins = defaultdict(set) + + for name, canonical_name, pages in generate_canonical_names(docalias_qs.values_list("document", "name", "document__pages")): + bins[pages].add(canonical_name) + + series_data = [] + for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / (total_docs or 1) + if pages is not None: + series_data.append((pages, len(names))) + table_data.append((pages, percentage, len(names), list(names)[:names_limit])) + + chart_data.append({ "data": series_data }) + + elif stats_type == "words": + stats_title = "Number of words for each {}".format(doc_label) + + bin_size = 500 + + bins = defaultdict(set) + + for name, canonical_name, words in generate_canonical_names(docalias_qs.values_list("document", "name", "document__words")): + bins[put_into_bin(words, bin_size)].add(canonical_name) + + series_data = [] + for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]): + percentage = len(names) * 100.0 / (total_docs or 1) + if words is not None: + series_data.append((value, len(names))) + + table_data.append((words, percentage, len(names), list(names)[:names_limit])) + + chart_data.append({ "data": series_data }) + + elif stats_type == "format": + stats_title = "Submission formats for each {}".format(doc_label) + + bins = defaultdict(set) + + # on new documents, we should have a Submission row with the file types + submission_types = {} + + for doc_name, file_types in Submission.objects.values_list("draft", "file_types").order_by("submission_date", "id"): + submission_types[doc_name] = file_types + + doc_names_with_missing_types = {} + for doc_name, canonical_name, rev in generate_canonical_names(docalias_qs.values_list("document", "name", "document__rev")): + types = submission_types.get(doc_name) + if types: + for dot_ext in types.split(","): + bins[dot_ext.lstrip(".").upper()].add(canonical_name) + + else: + + if canonical_name.startswith("rfc"): + filename = canonical_name + else: + filename = canonical_name + "-" + rev + + doc_names_with_missing_types[filename] = canonical_name + + # look up the remaining documents on disk + for filename in itertools.chain(os.listdir(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR), os.listdir(settings.RFC_PATH)): + t = filename.split(".", 1) + if len(t) != 2: + continue + + basename, ext = t + ext = ext.lower() + if not any(ext==whitelisted_ext for whitelisted_ext in settings.DOCUMENT_FORMAT_WHITELIST): + continue + + canonical_name = doc_names_with_missing_types.get(basename) + + if canonical_name: + bins[ext.upper()].add(canonical_name) + + series_data = [] + for fmt, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / (total_docs or 1) + series_data.append((fmt, len(names))) + + table_data.append((fmt, percentage, len(names), list(names)[:names_limit])) + + chart_data.append({ "data": series_data }) + + elif stats_type == "formlang": + stats_title = "Formal languages used for each {}".format(doc_label) + + bins = defaultdict(set) + + for name, canonical_name, formal_language_name in generate_canonical_names(docalias_qs.values_list("document", "name", "document__formal_languages__name")): + bins[formal_language_name].add(canonical_name) + + series_data = [] + for formal_language, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / (total_docs or 1) + if formal_language is not None: + series_data.append((formal_language, len(names))) + table_data.append((formal_language, percentage, len(names), list(names)[:names_limit])) + + chart_data.append({ "data": series_data }) + + elif any(stats_type == t[0] for t in possible_author_stats_types): + person_filters = Q(documentauthor__document__type="draft") + + # filter persons + rfc_state = State.objects.get(type="draft", slug="rfc") + if document_type == "rfc": + person_filters &= Q(documentauthor__document__states=rfc_state) + elif document_type == "draft": + person_filters &= ~Q(documentauthor__document__states=rfc_state) + + if from_time: + # this is actually faster than joining in the database, + # despite the round-trip back and forth + docs_within_time_constraint = set(Document.objects.filter( + type="draft", + docevent__time__gte=from_time, + docevent__type__in=["published_rfc", "new_revision"], + ).values_list("pk")) + + person_filters &= Q(documentauthor__document__in=docs_within_time_constraint) + + person_qs = Person.objects.filter(person_filters) + + if document_type == "rfc": + doc_label = "RFC" + elif document_type == "draft": + doc_label = "draft" + else: + doc_label = "document" + + if stats_type == "author/documents": + stats_title = "Number of {}s per author".format(doc_label) + + bins = defaultdict(set) + + person_qs = Person.objects.filter(person_filters) + + for name, document_count in person_qs.values_list("name").annotate(Count("documentauthor")): + bins[document_count].add(name) + + total_persons = count_bins(bins) + + series_data = [] + for document_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / (total_persons or 1) + series_data.append((document_count, percentage)) + plain_names = [ plain_name(n) for n in names ] + table_data.append((document_count, percentage, len(plain_names), list(plain_names)[:names_limit])) + + chart_data.append({ "data": series_data }) + + elif stats_type == "author/affiliation": + stats_title = "Number of {} authors per affiliation".format(doc_label) + + bins = defaultdict(set) + + person_qs = Person.objects.filter(person_filters) + + # Since people don't write the affiliation names in the + # same way, and we don't want to go back and edit them + # either, we transform them here. + + name_affiliation_set = { + (name, affiliation) + for name, affiliation in person_qs.values_list("name", "documentauthor__affiliation") + } + + aliases = get_aliased_affiliations(affiliation for _, affiliation in name_affiliation_set) + + for name, affiliation in name_affiliation_set: + bins[aliases.get(affiliation, affiliation)].add(name) + + prune_unknown_bin_with_known(bins) + total_persons = count_bins(bins) + + series_data = [] + for affiliation, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / (total_persons or 1) + if affiliation: + series_data.append((affiliation, len(names))) + plain_names = [ plain_name(n) for n in names ] + table_data.append((affiliation, percentage, len(plain_names), list(plain_names)[:names_limit])) + + series_data.sort(key=lambda t: t[1], reverse=True) + series_data = series_data[:30] + + chart_data.append({ "data": series_data }) + + for alias, name in sorted(aliases.iteritems(), key=lambda t: t[1]): + alias_data.append((name, alias)) + + elif stats_type == "author/country": + stats_title = "Number of {} authors per country".format(doc_label) + + bins = defaultdict(set) + + person_qs = Person.objects.filter(person_filters) + + # Since people don't write the country names in the + # same way, and we don't want to go back and edit them + # either, we transform them here. + + name_country_set = { + (name, country) + for name, country in person_qs.values_list("name", "documentauthor__country") + } + + aliases = get_aliased_countries(country for _, country in name_country_set) + + countries = { c.name: c for c in CountryName.objects.all() } + eu_name = "EU" + eu_countries = { c for c in countries.itervalues() if c.in_eu } + + for name, country in name_country_set: + country_name = aliases.get(country, country) + bins[country_name].add(name) + + c = countries.get(country_name) + if c and c.in_eu: + bins[eu_name].add(name) + + prune_unknown_bin_with_known(bins) + total_persons = count_bins(bins) + + series_data = [] + for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / (total_persons or 1) + if country: + series_data.append((country, len(names))) + plain_names = [ plain_name(n) for n in names ] + table_data.append((country, percentage, len(plain_names), list(plain_names)[:names_limit])) + + series_data.sort(key=lambda t: t[1], reverse=True) + series_data = series_data[:30] + + chart_data.append({ "data": series_data }) + + for alias, country_name in aliases.iteritems(): + alias_data.append((country_name, alias, countries.get(country_name))) + + alias_data.sort() + + elif stats_type == "author/continent": + stats_title = "Number of {} authors per continent".format(doc_label) + + bins = defaultdict(set) + + person_qs = Person.objects.filter(person_filters) + + name_country_set = { + (name, country) + for name, country in person_qs.values_list("name", "documentauthor__country") + } + + aliases = get_aliased_countries(country for _, country in name_country_set) + + country_to_continent = dict(CountryName.objects.values_list("name", "continent__name")) + + for name, country in name_country_set: + country_name = aliases.get(country, country) + continent_name = country_to_continent.get(country_name, "") + bins[continent_name].add(name) + + prune_unknown_bin_with_known(bins) + total_persons = count_bins(bins) + + series_data = [] + for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / (total_persons or 1) + if continent: + series_data.append((continent, len(names))) + plain_names = [ plain_name(n) for n in names ] + table_data.append((continent, percentage, len(plain_names), list(plain_names)[:names_limit])) + + series_data.sort(key=lambda t: t[1], reverse=True) + + chart_data.append({ "data": series_data }) + + elif stats_type == "author/citations": + stats_title = "Number of citations of {}s written by author".format(doc_label) + + bins = defaultdict(set) + + cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold'])) + person_filters &= Q(documentauthor__document__docalias__relateddocument__relationship__in=cite_relationships) + + person_qs = Person.objects.filter(person_filters) + + for name, citations in person_qs.values_list("name").annotate(Count("documentauthor__document__docalias__relateddocument")): + bins[citations].add(name) + + total_persons = count_bins(bins) + + series_data = [] + for citations, names in sorted(bins.iteritems(), key=lambda t: t[0], reverse=True): + percentage = len(names) * 100.0 / (total_persons or 1) + series_data.append((citations, percentage)) + plain_names = [ plain_name(n) for n in names ] + table_data.append((citations, percentage, len(plain_names), list(plain_names)[:names_limit])) + + chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) }) + + elif stats_type == "author/hindex": + stats_title = "h-index for {}s written by author".format(doc_label) + + bins = defaultdict(set) + + cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold'])) + person_filters &= Q(documentauthor__document__docalias__relateddocument__relationship__in=cite_relationships) + + person_qs = Person.objects.filter(person_filters) + + values = person_qs.values_list("name", "documentauthor__document").annotate(Count("documentauthor__document__docalias__relateddocument")) + for name, ts in itertools.groupby(values.order_by("name"), key=lambda t: t[0]): + h_index = compute_hirsch_index([citations for _, document, citations in ts]) + bins[h_index].add(name) + + total_persons = count_bins(bins) + + series_data = [] + for citations, names in sorted(bins.iteritems(), key=lambda t: t[0], reverse=True): + percentage = len(names) * 100.0 / (total_persons or 1) + series_data.append((citations, percentage)) + plain_names = [ plain_name(n) for n in names ] + table_data.append((citations, percentage, len(plain_names), list(plain_names)[:names_limit])) + + chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) }) + + elif any(stats_type == t[0] for t in possible_yearly_stats_types): + + person_filters = Q(documentauthor__document__type="draft") + + # filter persons + rfc_state = State.objects.get(type="draft", slug="rfc") + if document_type == "rfc": + person_filters &= Q(documentauthor__document__states=rfc_state) + elif document_type == "draft": + person_filters &= ~Q(documentauthor__document__states=rfc_state) + + doc_years = defaultdict(set) + + docevent_qs = DocEvent.objects.filter( + doc__type="draft", + type__in=["published_rfc", "new_revision"], + ).values_list("doc", "time").order_by("doc") + + for doc, time in docevent_qs.iterator(): + doc_years[doc].add(time.year) + + person_qs = Person.objects.filter(person_filters) + + if document_type == "rfc": + doc_label = "RFC" + elif document_type == "draft": + doc_label = "draft" + else: + doc_label = "document" + + template_name = "yearly" + + years_from = from_time.year if from_time else 1 + years_to = datetime.date.today().year - 1 + + + if stats_type == "yearly/affiliation": + stats_title = "Number of {} authors per affiliation over the years".format(doc_label) + + person_qs = Person.objects.filter(person_filters) + + name_affiliation_doc_set = { + (name, affiliation, doc) + for name, affiliation, doc in person_qs.values_list("name", "documentauthor__affiliation", "documentauthor__document") + } + + aliases = get_aliased_affiliations(affiliation for _, affiliation, _ in name_affiliation_doc_set) + + bins = defaultdict(set) + for name, affiliation, doc in name_affiliation_doc_set: + a = aliases.get(affiliation, affiliation) + if a: + for year in doc_years.get(doc): + if years_from <= year <= years_to: + bins[(year, a)].add(name) + + add_labeled_top_series_from_bins(chart_data, bins, limit=8) + + elif stats_type == "yearly/country": + stats_title = "Number of {} authors per country over the years".format(doc_label) + + person_qs = Person.objects.filter(person_filters) + + name_country_doc_set = { + (name, country, doc) + for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document") + } + + aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set) + + countries = { c.name: c for c in CountryName.objects.all() } + eu_name = "EU" + eu_countries = { c for c in countries.itervalues() if c.in_eu } + + bins = defaultdict(set) + + for name, country, doc in name_country_doc_set: + country_name = aliases.get(country, country) + c = countries.get(country_name) + + if country_name: + for year in doc_years.get(doc): + if years_from <= year <= years_to: + bins[(year, country_name)].add(name) + + if c and c.in_eu: + bins[(year, eu_name)].add(name) + + add_labeled_top_series_from_bins(chart_data, bins, limit=8) + + + elif stats_type == "yearly/continent": + stats_title = "Number of {} authors per continent".format(doc_label) + + person_qs = Person.objects.filter(person_filters) + + name_country_doc_set = { + (name, country, doc) + for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document") + } + + aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set) + + country_to_continent = dict(CountryName.objects.values_list("name", "continent__name")) + + bins = defaultdict(set) + + for name, country, doc in name_country_doc_set: + country_name = aliases.get(country, country) + continent_name = country_to_continent.get(country_name, "") + + if continent_name: + for year in doc_years.get(doc): + if years_from <= year <= years_to: + bins[(year, continent_name)].add(name) + + add_labeled_top_series_from_bins(chart_data, bins, limit=8) + + data = { + "chart_data": mark_safe(json.dumps(chart_data)), + "table_data": table_data, + "stats_title": stats_title, + "possible_document_stats_types": possible_document_stats_types, + "possible_author_stats_types": possible_author_stats_types, + "possible_yearly_stats_types": possible_yearly_stats_types, + "stats_type": stats_type, + "possible_document_types": possible_document_types, + "document_type": document_type, + "possible_time_choices": possible_time_choices, + "time_choice": time_choice, + "doc_label": doc_label, + "bin_size": bin_size, + "show_aliases_url": build_document_stats_url(get_overrides={ "showaliases": "1" }), + "hide_aliases_url": build_document_stats_url(get_overrides={ "showaliases": None }), + "alias_data": alias_data, + "eu_countries": sorted(eu_countries or [], key=lambda c: c.name), + "content_template": "stats/document_stats_{}.html".format(template_name), + } + log("Cache miss for '%s'. Data size: %sk" % (cache_key, len(str(data))/1000)) + cache.set(cache_key, data, 24*60*60) + return render(request, "stats/document_stats.html", data) + +def known_countries_list(request, stats_type=None, acronym=None): + countries = CountryName.objects.prefetch_related("countryalias_set") + for c in countries: + # the sorting is a bit of a hack - it puts the ISO code first + # since it was added in a migration + c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) + + return render(request, "stats/known_countries_list.html", { + "countries": countries, + "ticket_email_address": settings.SECRETARIAT_TICKET_EMAIL, + }) + +def meeting_stats(request, num=None, stats_type=None): + meeting = None + if num is not None: + meeting = get_object_or_404(Meeting, number=num, type="ietf") + + def build_meeting_stats_url(number=None, stats_type_override=Ellipsis, get_overrides={}): + kwargs = { + "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, + } + + if number is not None: + kwargs["num"] = number + + return urlreverse(meeting_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) + + cache_key = "stats:meeting:%s:%s:%s" % (num, stats_type, request.META.get('QUERY_STRING','')) + data = cache.get(cache_key) + if not data: + names_limit = settings.STATS_NAMES_LIMIT + # statistics types + if meeting: + possible_stats_types = add_url_to_choices([ + ("country", "Country"), + ("continent", "Continent"), + ], lambda slug: build_meeting_stats_url(number=meeting.number, stats_type_override=slug)) + else: + possible_stats_types = add_url_to_choices([ + ("overview", "Overview"), + ("country", "Country"), + ("continent", "Continent"), + ], lambda slug: build_meeting_stats_url(number=None, stats_type_override=slug)) + + if not stats_type: + return HttpResponseRedirect(build_meeting_stats_url(number=num, stats_type_override=possible_stats_types[0][0])) + + chart_data = [] + piechart_data = [] + table_data = [] + stats_title = "" + template_name = stats_type + bin_size = 1 + eu_countries = None + + def get_country_mapping(attendees): + return { + alias.alias: alias.country + for alias in CountryAlias.objects.filter(alias__in=set(r.country_code for r in attendees)).select_related("country", "country__continent") + if alias.alias.isupper() + } + + def reg_name(r): + return email.utils.formataddr(((r.first_name + u" " + r.last_name).strip(), r.email)) + + if meeting and any(stats_type == t[0] for t in possible_stats_types): + attendees = MeetingRegistration.objects.filter(meeting=meeting) + + if stats_type == "country": + stats_title = "Number of attendees for {} {} per country".format(meeting.type.name, meeting.number) + + bins = defaultdict(set) + + country_mapping = get_country_mapping(attendees) + + eu_name = "EU" + eu_countries = set(CountryName.objects.filter(in_eu=True)) + + for r in attendees: + name = reg_name(r) + c = country_mapping.get(r.country_code) + bins[c.name if c else ""].add(name) + + if c and c.in_eu: + bins[eu_name].add(name) + + prune_unknown_bin_with_known(bins) + total_attendees = count_bins(bins) + + series_data = [] + for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / (total_attendees or 1) + if country: + series_data.append((country, len(names))) + table_data.append((country, percentage, len(names), list(names)[:names_limit])) + + if country and country != eu_name: + piechart_data.append({ "name": country, "y": percentage }) + + series_data.sort(key=lambda t: t[1], reverse=True) + series_data = series_data[:20] + + piechart_data.sort(key=lambda d: d["y"], reverse=True) + pie_cut_off = 8 + piechart_data = piechart_data[:pie_cut_off] + [{ "name": "Other", "y": sum(d["y"] for d in piechart_data[pie_cut_off:])}] + + chart_data.append({ "data": series_data }) + + elif stats_type == "continent": + stats_title = "Number of attendees for {} {} per continent".format(meeting.type.name, meeting.number) + + bins = defaultdict(set) + + country_mapping = get_country_mapping(attendees) + + for r in attendees: + name = reg_name(r) + c = country_mapping.get(r.country_code) + bins[c.continent.name if c else ""].add(name) + + prune_unknown_bin_with_known(bins) + total_attendees = count_bins(bins) + + series_data = [] + for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / (total_attendees or 1) + if continent: + series_data.append((continent, len(names))) + table_data.append((continent, percentage, len(names), list(names)[:names_limit])) + + series_data.sort(key=lambda t: t[1], reverse=True) + + chart_data.append({ "data": series_data }) + + + elif not meeting and any(stats_type == t[0] for t in possible_stats_types): + template_name = "overview" + + attendees = MeetingRegistration.objects.filter(meeting__type="ietf").select_related('meeting') + + if stats_type == "overview": + stats_title = "Number of attendees per meeting" + + bins = defaultdict(set) + + for r in attendees: + meeting_number = int(r.meeting.number) + name = reg_name(r) + + bins[meeting_number].add(name) + + meeting_cities = dict(Meeting.objects.filter(number__in=bins.iterkeys()).values_list("number", "city")) + + series_data = [] + for meeting_number, names in sorted(bins.iteritems()): + series_data.append((meeting_number, len(names))) + url = build_meeting_stats_url(number=meeting_number, stats_type_override="country") + label = "IETF {} - {}".format(meeting_number, meeting_cities.get(str(meeting_number), "")) + table_data.append((meeting_number, label, url, len(names), list(names)[:names_limit])) + + series_data.sort(key=lambda t: t[0]) + table_data.sort(key=lambda t: t[0], reverse=True) + + chart_data.append({ "name": "Attendees", "data": series_data }) + + + elif stats_type == "country": + stats_title = "Number of attendees per country across meetings" + + country_mapping = get_country_mapping(attendees) + + eu_name = "EU" + eu_countries = set(CountryName.objects.filter(in_eu=True)) + + bins = defaultdict(set) + + for r in attendees: + meeting_number = int(r.meeting.number) + name = reg_name(r) + c = country_mapping.get(r.country_code) + + if c: + bins[(meeting_number, c.name)].add(name) + if c.in_eu: + bins[(meeting_number, eu_name)].add(name) + + add_labeled_top_series_from_bins(chart_data, bins, limit=8) + + + elif stats_type == "continent": + stats_title = "Number of attendees per continent across meetings" + + country_mapping = get_country_mapping(attendees) + + bins = defaultdict(set) + + for r in attendees: + meeting_number = int(r.meeting.number) + name = reg_name(r) + c = country_mapping.get(r.country_code) + + if c: + bins[(meeting_number, c.continent.name)].add(name) + + add_labeled_top_series_from_bins(chart_data, bins, limit=8) + data = { + "chart_data": mark_safe(json.dumps(chart_data)), + "piechart_data": mark_safe(json.dumps(piechart_data)), + "table_data": table_data, + "stats_title": stats_title, + "possible_stats_types": possible_stats_types, + "stats_type": stats_type, + "bin_size": bin_size, + "meeting": meeting, + "eu_countries": sorted(eu_countries or [], key=lambda c: c.name), + "content_template": "stats/meeting_stats_{}.html".format(template_name), + } + log("Cache miss for '%s'. Data size: %sk" % (cache_key, len(str(data))/1000)) + cache.set(cache_key, data, 24*60*60) + # + return render(request, "stats/meeting_stats.html", data) + + @login_required def review_stats(request, stats_type=None, acronym=None): # This view is a bit complex because we want to show a bunch of @@ -39,41 +980,7 @@ def review_stats(request, stats_type=None, acronym=None): if acr: kwargs["acronym"] = acr - base_url = urlreverse(review_stats, kwargs=kwargs) - query_part = u"" - - if request.GET or get_overrides: - d = request.GET.copy() - for k, v in get_overrides.iteritems(): - if type(v) in (list, tuple): - if not v: - if k in d: - del d[k] - else: - d.setlist(k, v) - else: - if v is None or v == u"": - if k in d: - del d[k] - else: - d[k] = v - - if d: - query_part = u"?" + d.urlencode() - - return base_url + query_part - - def get_choice(get_parameter, possible_choices, multiple=False): - values = request.GET.getlist(get_parameter) - found = [t[0] for t in possible_choices if t[0] in values] - - if multiple: - return found - else: - if found: - return found[0] - else: - return None + return urlreverse(review_stats, kwargs=kwargs) + generate_query_string(request.GET, get_overrides) # which overview - team or reviewer if acronym: @@ -91,21 +998,19 @@ def review_stats(request, stats_type=None, acronym=None): if level == "team": possible_stats_types.append(("time", "Changes over time")) - possible_stats_types = [ (slug, label, build_review_stats_url(stats_type_override=slug)) - for slug, label in possible_stats_types ] + possible_stats_types = add_url_to_choices(possible_stats_types, + lambda slug: build_review_stats_url(stats_type_override=slug)) if not stats_type: return HttpResponseRedirect(build_review_stats_url(stats_type_override=possible_stats_types[0][0])) # what to count - possible_count_choices = [ + possible_count_choices = add_url_to_choices([ ("", "Review requests"), ("pages", "Reviewed pages"), - ] + ], lambda slug: build_review_stats_url(get_overrides={ "count": slug })) - possible_count_choices = [ (slug, label, build_review_stats_url(get_overrides={ "count": slug })) for slug, label in possible_count_choices ] - - count = get_choice("count", possible_count_choices) or "" + count = get_choice(request, "count", possible_count_choices) or "" # time range def parse_date(s): @@ -190,7 +1095,7 @@ def review_stats(request, stats_type=None, acronym=None): if stats_type == "time": possible_teams = [(t.acronym, t.acronym) for t in teams] - selected_teams = get_choice("team", possible_teams, multiple=True) + selected_teams = get_choice(request, "team", possible_teams, multiple=True) def add_if_exists_else_subtract(element, l): if element in l: @@ -198,9 +1103,12 @@ def review_stats(request, stats_type=None, acronym=None): else: return l + [element] - possible_teams = [(slug, label, build_review_stats_url(get_overrides={ - "team": add_if_exists_else_subtract(slug, selected_teams) - })) for slug, label in possible_teams] + possible_teams = add_url_to_choices( + possible_teams, + lambda slug: build_review_stats_url(get_overrides={ + "team": add_if_exists_else_subtract(slug, selected_teams) + }) + ) query_teams = [t for t in query_teams if t.acronym in selected_teams] extracted_data = extract_review_request_data(query_teams, query_reviewers, from_time, to_time) @@ -232,33 +1140,28 @@ def review_stats(request, stats_type=None, acronym=None): # choice - possible_completion_types = [ + possible_completion_types = add_url_to_choices([ ("completed_in_time", "Completed in time"), ("completed_late", "Completed late"), ("not_completed", "Not completed"), ("average_assignment_to_closure_days", "Avg. compl. days"), - ] + ], lambda slug: build_review_stats_url(get_overrides={ "completion": slug, "result": None, "state": None })) - possible_completion_types = [ - (slug, label, build_review_stats_url(get_overrides={ "completion": slug, "result": None, "state": None })) - for slug, label in possible_completion_types - ] + selected_completion_type = get_choice(request, "completion", possible_completion_types) - selected_completion_type = get_choice("completion", possible_completion_types) + possible_results = add_url_to_choices( + [(r.slug, r.name) for r in results], + lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": slug, "state": None }) + ) - possible_results = [ - (r.slug, r.name, build_review_stats_url(get_overrides={ "completion": None, "result": r.slug, "state": None })) - for r in results - ] - - selected_result = get_choice("result", possible_results) + selected_result = get_choice(request, "result", possible_results) - possible_states = [ - (s.slug, s.name, build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": s.slug })) - for s in states - ] + possible_states = add_url_to_choices( + [(s.slug, s.name) for s in states], + lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": slug }) + ) - selected_state = get_choice("state", possible_states) + selected_state = get_choice(request, "state", possible_states) if not selected_completion_type and not selected_result and not selected_state: selected_completion_type = "completed_in_time" diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index 790936907..3f07f7658 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -5,6 +5,7 @@ import email import pytz import xml2rfc import tempfile +from email.utils import formataddr from unidecode import unidecode from django import forms @@ -21,6 +22,7 @@ from ietf.doc.fields import SearchableDocAliasesField from ietf.ipr.mail import utc_from_string from ietf.meeting.models import Meeting from ietf.message.models import Message +from ietf.name.models import FormalLanguageName from ietf.submit.models import Submission, Preapproval from ietf.submit.utils import validate_submission_rev, validate_submission_document_date from ietf.submit.parsers.pdf_parser import PDFParser @@ -29,7 +31,6 @@ from ietf.submit.parsers.ps_parser import PSParser from ietf.submit.parsers.xml_parser import XMLParser from ietf.utils.draft import Draft - class SubmissionUploadForm(forms.Form): txt = forms.FileField(label=u'.txt format', required=False) xml = forms.FileField(label=u'.xml format', required=False) @@ -177,18 +178,14 @@ class SubmissionUploadForm(forms.Form): self.abstract = self.xmlroot.findtext('front/abstract').strip() if type(self.abstract) is unicode: self.abstract = unidecode(self.abstract) - self.author_list = [] author_info = self.xmlroot.findall('front/author') for author in author_info: - author_dict = dict( - company = author.findtext('organization').strip(), - last_name = author.attrib.get('surname').strip(), - full_name = author.attrib.get('fullname').strip(), - email = author.findtext('address/email').strip(), - ) - self.author_list.append(author_dict) - line = email.utils.formataddr((author_dict['full_name'], author_dict['email'])) - self.authors.append(line) + self.authors.append({ + "name": author.attrib.get('fullname').strip(), + "email": author.findtext('address/email').strip(), + "affiliation": author.findtext('organization').strip(), + "country": author.findtext('address/postal/country').strip(), + }) except forms.ValidationError: raise except Exception as e: @@ -324,18 +321,12 @@ class SubmissionUploadForm(forms.Form): return None class NameEmailForm(forms.Form): - """For validating supplied submitter and author information.""" name = forms.CharField(required=True) - email = forms.EmailField(label=u'Email address') - - #Fields for secretariat only - approvals_received = forms.BooleanField(label=u'Approvals received', required=False, initial=False) + email = forms.EmailField(label=u'Email address', required=True) def __init__(self, *args, **kwargs): - email_required = kwargs.pop("email_required", True) super(NameEmailForm, self).__init__(*args, **kwargs) - self.fields["email"].required = email_required self.fields["name"].widget.attrs["class"] = "name" self.fields["email"].widget.attrs["class"] = "email" @@ -345,11 +336,23 @@ class NameEmailForm(forms.Form): def clean_email(self): return self.cleaned_data["email"].replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() +class AuthorForm(NameEmailForm): + affiliation = forms.CharField(max_length=100, required=False) + country = forms.CharField(max_length=255, required=False) + + def __init__(self, *args, **kwargs): + super(AuthorForm, self).__init__(*args, **kwargs) + self.fields["email"].required = False + +class SubmitterForm(NameEmailForm): + #Fields for secretariat only + approvals_received = forms.BooleanField(label=u'Approvals received', required=False, initial=False) + def cleaned_line(self): line = self.cleaned_data["name"] email = self.cleaned_data.get("email") if email: - line += u" <%s>" % email + line = formataddr((line, email)) return line class ReplacesForm(forms.Form): @@ -376,13 +379,14 @@ class EditSubmissionForm(forms.ModelForm): rev = forms.CharField(label=u'Revision', max_length=2, required=True) document_date = forms.DateField(required=True) pages = forms.IntegerField(required=True) + formal_languages = forms.ModelMultipleChoiceField(queryset=FormalLanguageName.objects.filter(used=True), widget=forms.CheckboxSelectMultiple, required=False) abstract = forms.CharField(widget=forms.Textarea, required=True, strip=False) note = forms.CharField(label=mark_safe(u'Comment to the Secretariat'), widget=forms.Textarea, required=False, strip=False) class Meta: model = Submission - fields = ['title', 'rev', 'document_date', 'pages', 'abstract', 'note'] + fields = ['title', 'rev', 'document_date', 'pages', 'formal_languages', 'abstract', 'note'] def clean_rev(self): rev = self.cleaned_data["rev"] diff --git a/ietf/submit/migrations/0019_add_formal_languages_and_words.py b/ietf/submit/migrations/0019_add_formal_languages_and_words.py new file mode 100644 index 000000000..fd2ba1164 --- /dev/null +++ b/ietf/submit/migrations/0019_add_formal_languages_and_words.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0021_add_formlang_names'), + ('submit', '0018_fix_more_bad_submission_docevents'), + ] + + operations = [ + migrations.AddField( + model_name='submission', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), + migrations.AddField( + model_name='submission', + name='words', + field=models.IntegerField(null=True, blank=True), + ), + ] diff --git a/ietf/submit/migrations/0020_authors_as_jsonfield.py b/ietf/submit/migrations/0020_authors_as_jsonfield.py new file mode 100644 index 000000000..2d8b41eca --- /dev/null +++ b/ietf/submit/migrations/0020_authors_as_jsonfield.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations +import jsonfield.fields + +def parse_email_line(line): + """Split line on the form 'Some Name '""" + import re + m = re.match("([^<]+) <([^>]+)>$", line) + if m: + return dict(name=m.group(1), email=m.group(2)) + else: + return dict(name=line, email="") + +def parse_authors(author_lines): + res = [] + for line in author_lines.replace("\r", "").split("\n"): + line = line.strip() + if line: + res.append(parse_email_line(line)) + return res + +def convert_author_lines_to_json(apps, schema_editor): + import json + + Submission = apps.get_model("submit", "Submission") + for s in Submission.objects.all().iterator(): + Submission.objects.filter(pk=s.pk).update(authors=json.dumps(parse_authors(s.authors))) + +class Migration(migrations.Migration): + + dependencies = [ + ('submit', '0019_add_formal_languages_and_words'), + ] + + operations = [ + migrations.RunPython(convert_author_lines_to_json, migrations.RunPython.noop), + migrations.AlterField( + model_name='submission', + name='authors', + field=jsonfield.fields.JSONField(default=list, help_text=b'List of authors with name, email, affiliation and country.'), + ), + ] diff --git a/ietf/submit/models.py b/ietf/submit/models.py index 8788400f4..90a6a1378 100644 --- a/ietf/submit/models.py +++ b/ietf/submit/models.py @@ -10,7 +10,7 @@ from ietf.doc.models import Document from ietf.person.models import Person from ietf.group.models import Group from ietf.message.models import Message -from ietf.name.models import DraftSubmissionStateName +from ietf.name.models import DraftSubmissionStateName, FormalLanguageName from ietf.utils.accesstoken import generate_random_key, generate_access_token @@ -36,7 +36,10 @@ class Submission(models.Model): abstract = models.TextField(blank=True) rev = models.CharField(max_length=3, blank=True) pages = models.IntegerField(null=True, blank=True) - authors = models.TextField(blank=True, help_text="List of author names and emails, one author per line, e.g. \"John Doe <john@example.org>\".") + words = models.IntegerField(null=True, blank=True) + formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document") + + authors = jsonfield.JSONField(default=list, help_text="List of authors with name, email, affiliation and country.") note = models.TextField(blank=True) replaces = models.CharField(max_length=1000, blank=True) @@ -53,20 +56,6 @@ class Submission(models.Model): def __unicode__(self): return u"%s-%s" % (self.name, self.rev) - def authors_parsed(self): - if not hasattr(self, '_cached_authors_parsed'): - from ietf.submit.utils import ensure_person_email_info_exists - res = [] - for line in self.authors.replace("\r", "").split("\n"): - line = line.strip() - if line: - parsed = parse_email_line(line) - if not parsed["email"]: - parsed["email"] = ensure_person_email_info_exists(**parsed).address - res.append(parsed) - self._cached_authors_parsed = res - return self._cached_authors_parsed - def submitter_parsed(self): return parse_email_line(self.submitter) diff --git a/ietf/submit/test_submission.txt b/ietf/submit/test_submission.txt index 97c95b3c4..b6551bf2d 100644 --- a/ietf/submit/test_submission.txt +++ b/ietf/submit/test_submission.txt @@ -61,8 +61,10 @@ Internet-Draft Testing Tests %(month)s %(year)s Table of Contents 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 - 2. Security Considerations . . . . . . . . . . . . . . . . . . . 2 - 3. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 2 + 2. Yang . . . . . . . . . . . . . . . . . . . . . . . . . . . . 2 + 3. JSON example . . . . . . . . . . . . . . . . . . . . . . . . 2 + 4. Security Considerations . . . . . . . . . . . . . . . . . . . 2 + 5. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 2 Author's Address . . . . . . . . . . . . . . . . . . . . . . . . 2 1. Introduction @@ -169,11 +171,19 @@ Table of Contents -3. Security Considerations +3. JSON example + + The JSON object should look like this: + + { + "test": 1234 + } + +4. Security Considerations There are none. -4. IANA Considerations +5. IANA Considerations No new registrations for IANA. diff --git a/ietf/submit/test_submission.xml b/ietf/submit/test_submission.xml index a313e3706..a8b6d9cde 100644 --- a/ietf/submit/test_submission.xml +++ b/ietf/submit/test_submission.xml @@ -137,6 +137,15 @@ module ietf-mpls { +
+ + The JSON object should look like this: + + { + "test": 1234 + } + +
There are none. diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 2348be203..ce93184bb 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -18,7 +18,8 @@ from ietf.group.models import Group from ietf.group.utils import setup_default_community_list_for_group from ietf.meeting.models import Meeting from ietf.message.models import Message -from ietf.person.models import Person, Email +from ietf.name.models import FormalLanguageName +from ietf.person.models import Person from ietf.person.factories import UserFactory, PersonFactory from ietf.submit.models import Submission, Preapproval from ietf.submit.mail import add_submission_email, process_response_email @@ -48,7 +49,6 @@ def submission_file(name, rev, group, format, templatename, author=None): surname=author.last_name(), email=author.email().address.lower(), ) - file = StringIO(submission_text) file.name = "%s-%s.%s" % (name, rev, format) return file @@ -118,7 +118,7 @@ class SubmitTests(TestCase): q = PyQuery(r.content) print(q('div.has-error div.alert').text()) - self.assertEqual(r.status_code, 302) + self.assertNoFormPostErrors(r, ".has-error,.alert-danger") status_url = r["Location"] for format in formats: @@ -126,10 +126,12 @@ class SubmitTests(TestCase): self.assertEqual(Submission.objects.filter(name=name).count(), 1) submission = Submission.objects.get(name=name) self.assertTrue(all([ c.passed!=False for c in submission.checks.all() ])) - self.assertEqual(len(submission.authors_parsed()), 1) - author = submission.authors_parsed()[0] + self.assertEqual(len(submission.authors), 1) + author = submission.authors[0] self.assertEqual(author["name"], "Author Name") self.assertEqual(author["email"], "author@example.com") + self.assertEqual(author["affiliation"], "Test Centre Inc.") + self.assertEqual(author["country"], "UK") return status_url @@ -151,7 +153,7 @@ class SubmitTests(TestCase): if r.status_code == 302: submission = Submission.objects.get(name=name) - self.assertEqual(submission.submitter, u"%s <%s>" % (submitter_name, submitter_email)) + self.assertEqual(submission.submitter, email.utils.formataddr((submitter_name, submitter_email))) self.assertEqual(submission.replaces, ",".join(d.name for d in DocAlias.objects.filter(pk__in=replaces.split(",") if replaces else []))) return r @@ -184,6 +186,7 @@ class SubmitTests(TestCase): abstract="Blahblahblah.", rev="01", pages=2, + words=100, intended_std_level_id="ps", ad=draft.ad, expires=datetime.datetime.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), @@ -241,9 +244,11 @@ class SubmitTests(TestCase): self.assertEqual(draft.stream_id, "ietf") self.assertTrue(draft.expires >= datetime.datetime.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE - 1)) self.assertEqual(draft.get_state("draft-stream-%s" % draft.stream_id).slug, "wg-doc") - self.assertEqual(draft.authors.count(), 1) - self.assertEqual(draft.authors.all()[0].get_name(), "Author Name") - self.assertEqual(draft.authors.all()[0].address, "author@example.com") + authors = draft.documentauthor_set.all() + self.assertEqual(len(authors), 1) + self.assertEqual(authors[0].person.plain_name(), "Author Name") + self.assertEqual(authors[0].email.address, "author@example.com") + self.assertEqual(set(draft.formal_languages.all()), set(FormalLanguageName.objects.filter(slug="json"))) self.assertEqual(draft.relations_that_doc("replaces").count(), 1) self.assertTrue(draft.relations_that_doc("replaces").first().target, replaced_alias) self.assertEqual(draft.relations_that_doc("possibly-replaces").count(), 1) @@ -281,12 +286,12 @@ class SubmitTests(TestCase): draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="added_comment", by=Person.objects.get(user__username="secretary"), desc="Test")]) if not change_authors: draft.documentauthor_set.all().delete() - ensure_person_email_info_exists('Author Name','author@example.com') - draft.documentauthor_set.create(author=Email.objects.get(address='author@example.com')) + author_person, author_email = ensure_person_email_info_exists('Author Name','author@example.com') + draft.documentauthor_set.create(person=author_person, email=author_email) else: # Make it such that one of the previous authors has an invalid email address - bogus_email = ensure_person_email_info_exists('Bogus Person',None) - DocumentAuthor.objects.create(document=draft,author=bogus_email,order=draft.documentauthor_set.latest('order').order+1) + bogus_person, bogus_email = ensure_person_email_info_exists('Bogus Person',None) + DocumentAuthor.objects.create(document=draft, person=bogus_person, email=bogus_email, order=draft.documentauthor_set.latest('order').order+1) prev_author = draft.documentauthor_set.all()[0] @@ -334,7 +339,7 @@ class SubmitTests(TestCase): confirm_email = outbox[-1] self.assertTrue("Confirm submission" in confirm_email["Subject"]) self.assertTrue(name in confirm_email["Subject"]) - self.assertTrue(prev_author.author.address in confirm_email["To"]) + self.assertTrue(prev_author.email.address in confirm_email["To"]) if change_authors: self.assertTrue("author@example.com" not in confirm_email["To"]) self.assertTrue("submitter@example.com" not in confirm_email["To"]) @@ -415,9 +420,10 @@ class SubmitTests(TestCase): self.assertEqual(draft.stream_id, "ietf") self.assertEqual(draft.get_state_slug("draft-stream-%s" % draft.stream_id), "wg-doc") self.assertEqual(draft.get_state_slug("draft-iana-review"), "changed") - self.assertEqual(draft.authors.count(), 1) - self.assertEqual(draft.authors.all()[0].get_name(), "Author Name") - self.assertEqual(draft.authors.all()[0].address, "author@example.com") + authors = draft.documentauthor_set.all() + self.assertEqual(len(authors), 1) + self.assertEqual(authors[0].person.plain_name(), "Author Name") + self.assertEqual(authors[0].email.address, "author@example.com") self.assertEqual(len(outbox), mailbox_before + 3) self.assertTrue((u"I-D Action: %s" % name) in outbox[-3]["Subject"]) self.assertTrue((u"I-D Action: %s" % name) in draft.message_set.order_by("-time")[0].subject) @@ -654,7 +660,7 @@ class SubmitTests(TestCase): "authors-prefix": ["authors-", "authors-0", "authors-1", "authors-2"], }) - self.assertEqual(r.status_code, 302) + self.assertNoFormPostErrors(r, ".has-error,.alert-danger") submission = Submission.objects.get(name=name) self.assertEqual(submission.title, "some title") @@ -666,14 +672,14 @@ class SubmitTests(TestCase): self.assertEqual(submission.replaces, draft.docalias_set.all().first().name) self.assertEqual(submission.state_id, "manual") - authors = submission.authors_parsed() + authors = submission.authors self.assertEqual(len(authors), 3) self.assertEqual(authors[0]["name"], "Person 1") self.assertEqual(authors[0]["email"], "person1@example.com") self.assertEqual(authors[1]["name"], "Person 2") self.assertEqual(authors[1]["email"], "person2@example.com") self.assertEqual(authors[2]["name"], "Person 3") - self.assertEqual(authors[2]["email"], "unknown-email-Person-3") + self.assertEqual(authors[2]["email"], "") self.assertEqual(len(outbox), mailbox_before + 1) self.assertTrue("Manual Post Requested" in outbox[-1]["Subject"]) @@ -929,7 +935,6 @@ class SubmitTests(TestCase): files = {"txt": submission_file(name, rev, group, "txt", "test_submission.nonascii", author=author) } r = self.client.post(url, files) - self.assertEqual(r.status_code, 302) status_url = r["Location"] r = self.client.get(status_url) @@ -1433,8 +1438,8 @@ Subject: test self.assertEqual(Submission.objects.filter(name=name).count(), 1) submission = Submission.objects.get(name=name) self.assertTrue(all([ c.passed!=False for c in submission.checks.all() ])) - self.assertEqual(len(submission.authors_parsed()), 1) - author = submission.authors_parsed()[0] + self.assertEqual(len(submission.authors), 1) + author = submission.authors[0] self.assertEqual(author["name"], "Author Name") self.assertEqual(author["email"], "author@example.com") @@ -1459,6 +1464,6 @@ Subject: test if r.status_code == 302: submission = Submission.objects.get(name=name) - self.assertEqual(submission.submitter, u"%s <%s>" % (submitter_name, submitter_email)) + self.assertEqual(submission.submitter, email.utils.formataddr((submitter_name, submitter_email))) return r diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 62758c20a..b6cbb5bbf 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -47,7 +47,7 @@ def validate_submission(submission): if not submission.abstract: errors['abstract'] = 'Abstract is empty or was not found' - if not submission.authors_parsed(): + if not submission.authors: errors['authors'] = 'No authors found' # revision @@ -130,7 +130,7 @@ def docevent_from_submission(request, submission, desc, who=None): else: submitter_parsed = submission.submitter_parsed() if submitter_parsed["name"] and submitter_parsed["email"]: - by = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]).person + by, _ = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]) else: by = system @@ -184,7 +184,7 @@ def post_submission(request, submission, approvedDesc): system = Person.objects.get(name="(System)") submitter_parsed = submission.submitter_parsed() if submitter_parsed["name"] and submitter_parsed["email"]: - submitter = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]).person + submitter, _ = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]) submitter_info = u'%s <%s>' % (submitter_parsed["name"], submitter_parsed["email"]) else: submitter = system @@ -258,6 +258,8 @@ def post_submission(request, submission, approvedDesc): update_authors(draft, submission) + draft.formal_languages = submission.formal_languages.all() + trouble = rebuild_reference_relations(draft, filename=os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (submission.name, submission.rev))) if trouble: log.log('Rebuild_reference_relations trouble: %s'%trouble) @@ -344,10 +346,9 @@ def update_replaces_from_submission(request, submission, draft): if rdoc == draft: continue - # TODO - I think the .exists() is in the wrong place below.... if (is_secretariat or (draft.group in is_chair_of and (rdoc.group.type_id == "individ" or rdoc.group in is_chair_of)) - or (submitter_email and rdoc.authors.filter(address__iexact=submitter_email)).exists()): + or (submitter_email and rdoc.documentauthor_set.filter(email__address__iexact=submitter_email).exists())): approved.append(r) else: if r not in existing_suggested: @@ -434,24 +435,27 @@ def ensure_person_email_info_exists(name, email): email.time = datetime.datetime.now() email.save() - return email + return person, email def update_authors(draft, submission): - authors = [] - for order, author in enumerate(submission.authors_parsed()): - email = ensure_person_email_info_exists(author["name"], author["email"]) + persons = [] + for order, author in enumerate(submission.authors): + person, email = ensure_person_email_info_exists(author["name"], author.get("email")) - a = DocumentAuthor.objects.filter(document=draft, author=email).first() + a = DocumentAuthor.objects.filter(document=draft, person=person).first() if not a: - a = DocumentAuthor(document=draft, author=email) + a = DocumentAuthor(document=draft, person=person) + a.email = email + a.affiliation = author.get("affiliation") or "" + a.country = author.get("country") or "" a.order = order a.save() - log.assertion('a.author_id != "none"') + log.assertion('a.email_id != "none"') - authors.append(email) + persons.append(person) - draft.documentauthor_set.exclude(author__in=authors).delete() + draft.documentauthor_set.exclude(person__in=persons).delete() def cancel_submission(submission): submission.state = DraftSubmissionStateName.objects.get(slug="cancel") diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 99f043260..ca07dfd50 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -3,6 +3,7 @@ import base64 import datetime import os import xml2rfc +import optparse from django.conf import settings from django.contrib import messages @@ -20,7 +21,8 @@ from ietf.group.models import Group from ietf.ietfauth.utils import has_role, role_required from ietf.mailtrigger.utils import gather_address_lists from ietf.message.models import Message, MessageAttachment -from ietf.submit.forms import ( SubmissionUploadForm, NameEmailForm, EditSubmissionForm, +from ietf.name.models import FormalLanguageName +from ietf.submit.forms import ( SubmissionUploadForm, AuthorForm, SubmitterForm, EditSubmissionForm, PreapprovalForm, ReplacesForm, SubmissionEmailForm, MessageModelForm ) from ietf.submit.mail import ( send_full_url, send_approval_request_to_group, send_submission_confirmation, send_manual_post_request, add_submission_email, get_reply_to ) @@ -30,6 +32,7 @@ from ietf.submit.utils import ( approvable_submissions_for_user, preapprovals_fo recently_approved_by_user, validate_submission, create_submission_event, docevent_from_submission, post_submission, cancel_submission, rename_submission_files, get_person_from_name_email ) +from ietf.stats.utils import clean_country_name from ietf.utils.accesstoken import generate_random_key, generate_access_token from ietf.utils.draft import Draft from ietf.utils.log import log @@ -60,7 +63,7 @@ def upload_submission(request): if not form.cleaned_data['txt']: file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (form.filename, form.revision)) try: - pagedwriter = xml2rfc.PaginatedTextRfcWriter(form.xmltree, quiet=True) + pagedwriter = xml2rfc.PaginatedTextRfcWriter(form.xmltree, options=optparse.Values(defaults=dict(quiet=True, verbose=False, utf8=False))) pagedwriter.write(file_name['txt']) except Exception as e: raise ValidationError("Error from xml2rfc: %s" % e) @@ -80,10 +83,9 @@ def upload_submission(request): # If we don't have an xml file, try to extract the # relevant information from the text file for author in form.parsed_draft.get_author_list(): - full_name, first_name, middle_initial, last_name, name_suffix, email, company = author + full_name, first_name, middle_initial, last_name, name_suffix, email, country, company = author - line = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() - email = (email or "").strip() + name = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() if email: try: @@ -91,29 +93,31 @@ def upload_submission(request): except ValidationError: email = "" - if email: - # Try various ways of handling name and email, in order to avoid - # triggering a 500 error here. If the document contains non-ascii - # characters, it will be flagged later by the idnits check. - try: - line += u" <%s>" % email - except UnicodeDecodeError: + def turn_into_unicode(s): + if s is None: + return u"" + + if isinstance(s, unicode): + return s + else: try: - line = line.decode('utf-8') - email = email.decode('utf-8') - line += u" <%s>" % email + return s.decode("utf-8") except UnicodeDecodeError: try: - line = line.decode('latin-1') - email = email.decode('latin-1') - line += u" <%s>" % email + return s.decode("latin-1") except UnicodeDecodeError: - try: - line += " <%s>" % email - except UnicodeDecodeError: - pass + return "" - authors.append(line) + name = turn_into_unicode(name) + email = turn_into_unicode(email) + company = turn_into_unicode(company) + + authors.append({ + "name": name, + "email": email, + "affiliation": company, + "country": country + }) if form.abstract: abstract = form.abstract @@ -124,55 +128,39 @@ def upload_submission(request): # for this revision. # If so - we're going to update it otherwise we create a new object - submission = Submission.objects.filter(name=form.filename, - rev=form.revision, - state_id = "waiting-for-draft").distinct() - if (len(submission) == 0): - submission = None - elif (len(submission) == 1): - submission = submission[0] - - submission.state = DraftSubmissionStateName.objects.get(slug="uploaded") - submission.remote_ip=form.remote_ip - submission.title=form.title - submission.abstract=abstract - submission.rev=form.revision - submission.pages=form.parsed_draft.get_pagecount() - submission.authors="\n".join(authors) - submission.first_two_pages=''.join(form.parsed_draft.pages[:2]) - submission.file_size=file_size - submission.file_types=','.join(form.file_types) - submission.submission_date=datetime.date.today() - submission.document_date=form.parsed_draft.get_creation_date() - submission.replaces="" - - submission.save() + submissions = Submission.objects.filter(name=form.filename, + rev=form.revision, + state_id = "waiting-for-draft").distinct() + + if not submissions: + submission = Submission(name=form.filename, rev=form.revision, group=form.group) + elif len(submissions) == 1: + submission = submissions[0] else: raise Exception("Multiple submissions found waiting for upload") - if (submission == None): - try: - submission = Submission.objects.create( - state=DraftSubmissionStateName.objects.get(slug="uploaded"), - remote_ip=form.remote_ip, - name=form.filename, - group=form.group, - title=form.title, - abstract=abstract, - rev=form.revision, - pages=form.parsed_draft.get_pagecount(), - authors="\n".join(authors), - note="", - first_two_pages=''.join(form.parsed_draft.pages[:2]), - file_size=file_size, - file_types=','.join(form.file_types), - submission_date=datetime.date.today(), - document_date=form.parsed_draft.get_creation_date(), - replaces="", - ) - except Exception as e: - log("Exception: %s\n" % e) - raise + try: + submission.state = DraftSubmissionStateName.objects.get(slug="uploaded") + submission.remote_ip = form.remote_ip + submission.title = form.title + submission.abstract = abstract + submission.pages = form.parsed_draft.get_pagecount() + submission.words = form.parsed_draft.get_wordcount() + submission.authors = authors + submission.first_two_pages = ''.join(form.parsed_draft.pages[:2]) + submission.file_size = file_size + submission.file_types = ','.join(form.file_types) + submission.submission_date = datetime.date.today() + submission.document_date = form.parsed_draft.get_creation_date() + submission.replaces = "" + + submission.save() + + submission.formal_languages = FormalLanguageName.objects.filter(slug__in=form.parsed_draft.get_formal_languages()) + + except Exception as e: + log("Exception: %s\n" % e) + raise # run submission checkers def apply_check(submission, checker, method, fn): @@ -274,8 +262,8 @@ def submission_status(request, submission_id, access_token=None): group_authors_changed = False doc = submission.existing_document() if doc and doc.group: - old_authors = [ i.author.person for i in doc.documentauthor_set.all() ] - new_authors = [ get_person_from_name_email(**p) for p in submission.authors_parsed() ] + old_authors = [ author.person for author in doc.documentauthor_set.all() ] + new_authors = [ get_person_from_name_email(author["name"], author.get("email")) for author in submission.authors ] group_authors_changed = set(old_authors)!=set(new_authors) message = None @@ -290,7 +278,7 @@ def submission_status(request, submission_id, access_token=None): message = ('success', 'The submission is pending approval by the authors of the previous version. An email has been sent to: %s' % ", ".join(confirmation_list)) - submitter_form = NameEmailForm(initial=submission.submitter_parsed(), prefix="submitter") + submitter_form = SubmitterForm(initial=submission.submitter_parsed(), prefix="submitter") replaces_form = ReplacesForm(name=submission.name,initial=DocAlias.objects.filter(name__in=submission.replaces.split(","))) if request.method == 'POST': @@ -299,7 +287,7 @@ def submission_status(request, submission_id, access_token=None): if not can_edit: return HttpResponseForbidden("You do not have permission to perform this action") - submitter_form = NameEmailForm(request.POST, prefix="submitter") + submitter_form = SubmitterForm(request.POST, prefix="submitter") replaces_form = ReplacesForm(request.POST, name=submission.name) validations = [submitter_form.is_valid(), replaces_form.is_valid()] if all(validations): @@ -415,6 +403,9 @@ def submission_status(request, submission_id, access_token=None): # something went wrong, turn this into a GET and let the user deal with it return HttpResponseRedirect("") + for author in submission.authors: + author["cleaned_country"] = clean_country_name(author.get("country")) + return render(request, 'submit/submission_status.html', { 'selected': 'status', 'submission': submission, @@ -447,7 +438,7 @@ def edit_submission(request, submission_id, access_token=None): # submission itself, one for the submitter, and a list of forms # for the authors - empty_author_form = NameEmailForm(email_required=False) + empty_author_form = AuthorForm() if request.method == 'POST': # get a backup submission now, the model form may change some @@ -455,21 +446,23 @@ def edit_submission(request, submission_id, access_token=None): prev_submission = Submission.objects.get(pk=submission.pk) edit_form = EditSubmissionForm(request.POST, instance=submission, prefix="edit") - submitter_form = NameEmailForm(request.POST, prefix="submitter") + submitter_form = SubmitterForm(request.POST, prefix="submitter") replaces_form = ReplacesForm(request.POST,name=submission.name) - author_forms = [ NameEmailForm(request.POST, email_required=False, prefix=prefix) + author_forms = [ AuthorForm(request.POST, prefix=prefix) for prefix in request.POST.getlist("authors-prefix") if prefix != "authors-" ] # trigger validation of all forms validations = [edit_form.is_valid(), submitter_form.is_valid(), replaces_form.is_valid()] + [ f.is_valid() for f in author_forms ] if all(validations): + changed_fields = [] + submission.submitter = submitter_form.cleaned_line() replaces = replaces_form.cleaned_data.get("replaces", []) submission.replaces = ",".join(o.name for o in replaces) - submission.authors = "\n".join(f.cleaned_line() for f in author_forms) - if hasattr(submission, '_cached_authors_parsed'): - del submission._cached_authors_parsed + submission.authors = [ { attr: f.cleaned_data.get(attr) or "" + for attr in ["name", "email", "affiliation", "country"] } + for f in author_forms ] edit_form.save(commit=False) # transfer changes if submission.rev != prev_submission.rev: @@ -478,12 +471,18 @@ def edit_submission(request, submission_id, access_token=None): submission.state = DraftSubmissionStateName.objects.get(slug="manual") submission.save() + formal_languages_changed = False + if set(submission.formal_languages.all()) != set(edit_form.cleaned_data["formal_languages"]): + submission.formal_languages = edit_form.cleaned_data["formal_languages"] + formal_languages_changed = True + send_manual_post_request(request, submission, errors) - changed_fields = [ + changed_fields += [ submission._meta.get_field(f).verbose_name for f in list(edit_form.fields.keys()) + ["submitter", "authors"] - if getattr(submission, f) != getattr(prev_submission, f) + if (f == "formal_languages" and formal_languages_changed) + or getattr(submission, f) != getattr(prev_submission, f) ] if changed_fields: @@ -498,10 +497,10 @@ def edit_submission(request, submission_id, access_token=None): form_errors = True else: edit_form = EditSubmissionForm(instance=submission, prefix="edit") - submitter_form = NameEmailForm(initial=submission.submitter_parsed(), prefix="submitter") + submitter_form = SubmitterForm(initial=submission.submitter_parsed(), prefix="submitter") replaces_form = ReplacesForm(name=submission.name,initial=DocAlias.objects.filter(name__in=submission.replaces.split(","))) - author_forms = [ NameEmailForm(initial=author, email_required=False, prefix="authors-%s" % i) - for i, author in enumerate(submission.authors_parsed()) ] + author_forms = [ AuthorForm(initial=author, prefix="authors-%s" % i) + for i, author in enumerate(submission.authors) ] return render(request, 'submit/edit_submission.html', {'selected': 'status', diff --git a/ietf/templates/doc/bibxml.xml b/ietf/templates/doc/bibxml.xml index 9813c3a00..fdb57c132 100644 --- a/ietf/templates/doc/bibxml.xml +++ b/ietf/templates/doc/bibxml.xml @@ -2,11 +2,11 @@ {{doc.title}} - {% for entry in doc.authors.all %}{% with entry.address as email %}{% with entry.person as author %} - - {{author.affiliation}} + {% for author in doc.documentauthor_set.all %} + + {{ author.affiliation }} - {% endwith %}{% endwith %}{% endfor %} + {% endfor %} {{doc.abstract}} diff --git a/ietf/templates/doc/document_bibtex.bib b/ietf/templates/doc/document_bibtex.bib index f2efdfc0d..0a5895f99 100644 --- a/ietf/templates/doc/document_bibtex.bib +++ b/ietf/templates/doc/document_bibtex.bib @@ -24,7 +24,7 @@ publisher = {% templatetag openbrace %}Internet Engineering Task Force{% templatetag closebrace %}, note = {% templatetag openbrace %}Work in Progress{% templatetag closebrace %}, url = {% templatetag openbrace %}https://datatracker.ietf.org/doc/html/{{doc.name}}-{{doc.rev}}{% templatetag closebrace %},{% endif %} - author = {% templatetag openbrace %}{% for entry in doc.authors.all %}{% with entry.person as author %}{{author.name}}{% endwith %}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, + author = {% templatetag openbrace %}{% for author in doc.documentauthor_set.all %}{{ author.person.name}}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, title = {% templatetag openbrace %}{% templatetag openbrace %}{{doc.title}}{% templatetag closebrace %}{% templatetag closebrace %}, pagetotal = {{ doc.pages }}, year = {{ doc.pub_date.year }}, diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index 7a8caba78..870e188a5 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -572,13 +572,13 @@

Authors

{% for author in doc.documentauthor_set.all %} - {% if not author.author.invalid_address %} + {% if author.email %} - + {% endif %} - {{ author.author.person }} - {% if not author.author.invalid_address %} - ({{ author.author.address }}) + {{ author.person }} + {% if author.email %} + ({{ author.email.address }}) {% endif %} {% if not forloop.last %}
{% endif %} {% endfor %} diff --git a/ietf/templates/doc/document_referenced_by.html b/ietf/templates/doc/document_referenced_by.html index 1ce3bdc89..740d68f7b 100644 --- a/ietf/templates/doc/document_referenced_by.html +++ b/ietf/templates/doc/document_referenced_by.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin staticfiles %} +{% load origin staticfiles ietf_filters %} {% block pagehead %} @@ -44,8 +44,8 @@ {% with ref.source.canonical_name as name %} - {{name}} - {% if ref.target.name != alias_name %} + {{ name|prettystdname }} + {% if ref.target.name != alias_name %}
As {{ref.target.name}} {% endif %} diff --git a/ietf/templates/doc/document_references.html b/ietf/templates/doc/document_references.html index b63801d9a..e8c9a7c03 100644 --- a/ietf/templates/doc/document_references.html +++ b/ietf/templates/doc/document_references.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin staticfiles %} +{% load origin staticfiles ietf_filters %} {% block pagehead %} @@ -35,7 +35,7 @@ {% for ref in refs %} {% with ref.target.name as name %} - {{name}} + {{ name|prettystdname }} {{ref.target.document.title}}
Refs diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html new file mode 100644 index 000000000..a3819d9e5 --- /dev/null +++ b/ietf/templates/stats/document_stats.html @@ -0,0 +1,78 @@ +{% extends "base.html" %} + +{% load origin %} + +{% load ietf_filters staticfiles bootstrap3 %} + +{% block title %}{{ stats_title }}{% endblock %} + +{% block pagehead %} + +{% endblock %} + +{% block content %} + {% origin %} + +

Draft/RFC statistics

+ +
+
+ Documents: + +
+ {% for slug, label, url in possible_document_stats_types %} + {{ label }} + {% endfor %} +
+
+ +
+ Authors: + +
+ {% for slug, label, url in possible_author_stats_types %} + {{ label }} + {% endfor %} +
+
+ +
+ Yearly: + +
+ {% for slug, label, url in possible_yearly_stats_types %} + {{ label }} + {% endfor %} +
+
+ +
Options
+ +
+ Document type: +
+ {% for slug, label, url in possible_document_types %} + {{ label }} + {% endfor %} +
+ + Time: +
+ {% for slug, label, url in possible_time_choices %} + {{ label }} + {% endfor %} +
+
+
+ +
+ {% include content_template %} +
+{% endblock %} + +{% block js %} + + + + +{% endblock %} diff --git a/ietf/templates/stats/document_stats_author_affiliation.html b/ietf/templates/stats/document_stats_author_affiliation.html new file mode 100644 index 000000000..7e603f16d --- /dev/null +++ b/ietf/templates/stats/document_stats_author_affiliation.html @@ -0,0 +1,107 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for affiliation, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
AffiliationPercentage of authorsAuthors
{{ affiliation|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
+ +

The statistics are based entirely on the author affiliation + provided with each draft. Since this may vary across documents, an + author may be counted with more than one affiliation, making the + total sum more than 100%.

+ + + +

Affiliation Aliases

+ +

In generating the above statistics, some heuristics have been + applied to determine the affiliations of each author.

+ +{% if request.GET.showaliases %} +

Hide generated aliases

+ + {% if request.user.is_staff %} +

Note: since you're an admin, you can add an extra known alias or see the existing known aliases and generally ignored endings.

+ {% endif %} + + {% if alias_data %} + + + + + + + {% for name, alias in alias_data %} + + + + + {% endfor %} +
AffiliationAlias
+ {% ifchanged %} + {{ name|default:"(unknown)" }} + {% endifchanged %} + {{ alias }}
+ {% endif %} + +{% else %} +

Show generated aliases

+{% endif %} diff --git a/ietf/templates/stats/document_stats_author_citations.html b/ietf/templates/stats/document_stats_author_citations.html new file mode 100644 index 000000000..6374b50c5 --- /dev/null +++ b/ietf/templates/stats/document_stats_author_citations.html @@ -0,0 +1,71 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for citations, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
CitationsPercentage of authorsAuthors
{{ citations }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}
+ +

Note that the citation counts do not exclude self-references.

diff --git a/ietf/templates/stats/document_stats_author_continent.html b/ietf/templates/stats/document_stats_author_continent.html new file mode 100644 index 000000000..82d578aa6 --- /dev/null +++ b/ietf/templates/stats/document_stats_author_continent.html @@ -0,0 +1,69 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for continent, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
ContinentPercentage of authorsAuthors
{{ continent|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
+ +

The statistics are based entirely on the author addresses provided + with each draft. Since this varies across documents, a travelling + author may be counted in more than country, making the total sum + more than 100%.

diff --git a/ietf/templates/stats/document_stats_author_country.html b/ietf/templates/stats/document_stats_author_country.html new file mode 100644 index 000000000..e745eb668 --- /dev/null +++ b/ietf/templates/stats/document_stats_author_country.html @@ -0,0 +1,130 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for country, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
CountryPercentage of authorsAuthors
{{ country|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
+ +

The statistics are based entirely on the author addresses provided + with each draft. Since this varies across documents, a travelling + author may be counted in more than country, making the total sum + more than 100%.

+ +

In case no country information is found for an author in the time + period, the author is counted as (unknown).

+ +

EU (European Union) is not a country, but has been added for reference, as the sum of + all current EU member countries: + {% for c in eu_countries %}{{ c.name }}{% if not forloop.last %}, {% endif %}{% endfor %}.

+ +

Country Aliases

+ +

In generating the above statistics, some heuristics have been + applied to figure out which country each author is from.

+ +{% if request.GET.showaliases %} +

Hide generated aliases

+ + {% if request.user.is_staff %} +

Note: since you're an admin, some extra links are visible. You + can either correct a document author entry directly in case the + information is obviously missing or add an alias if an unknown + country name + is being used. +

+ + {% endif %} + + {% if alias_data %} + + + + + + + + {% for name, alias, country in alias_data %} + + + + + + {% endfor %} +
CountryAlias
+ {% ifchanged %} + {% if country and request.user.is_staff %} + + {% endif %} + {{ name|default:"(unknown)" }} + {% if country and request.user.is_staff %} + + {% endif %} + {% endifchanged %} + {{ alias }} + {% if request.user.is_staff and name != "EU" %} + Matching authors + {% endif %} +
+ {% endif %} + +{% else %} +

Show generated aliases

+{% endif %} diff --git a/ietf/templates/stats/document_stats_author_documents.html b/ietf/templates/stats/document_stats_author_documents.html new file mode 100644 index 000000000..23b9ce51d --- /dev/null +++ b/ietf/templates/stats/document_stats_author_documents.html @@ -0,0 +1,70 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for document_count, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
DocumentsPercentage of authorsAuthors
{{ document_count }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}
diff --git a/ietf/templates/stats/document_stats_author_hindex.html b/ietf/templates/stats/document_stats_author_hindex.html new file mode 100644 index 000000000..b5d63dad3 --- /dev/null +++ b/ietf/templates/stats/document_stats_author_hindex.html @@ -0,0 +1,79 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for h_index, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
h-indexPercentage of authorsAuthors
{{ h_index }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=25 %}
+ +

Hirsch index or h-index is a + measure of the + productivity and impact of the publications of an author. An + author with an h-index of 5 has had 5 publications each cited at + least 5 times - to increase the index to 6, the 5 publications plus + 1 more would have to have been cited at least 6 times, each. Thus a + high h-index requires many highly-cited publications.

+ +

Note that the h-index calculations do not exclude self-references.

diff --git a/ietf/templates/stats/document_stats_authors.html b/ietf/templates/stats/document_stats_authors.html new file mode 100644 index 000000000..7c4bcf144 --- /dev/null +++ b/ietf/templates/stats/document_stats_authors.html @@ -0,0 +1,69 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for author_count, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
AuthorsPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ author_count }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_format.html b/ietf/templates/stats/document_stats_format.html new file mode 100644 index 000000000..5630f4319 --- /dev/null +++ b/ietf/templates/stats/document_stats_format.html @@ -0,0 +1,65 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for pages, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
FormatPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_formlang.html b/ietf/templates/stats/document_stats_formlang.html new file mode 100644 index 000000000..3cc17589a --- /dev/null +++ b/ietf/templates/stats/document_stats_formlang.html @@ -0,0 +1,65 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for formal_language, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
Formal languagePercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ formal_language }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_pages.html b/ietf/templates/stats/document_stats_pages.html new file mode 100644 index 000000000..2b42266e9 --- /dev/null +++ b/ietf/templates/stats/document_stats_pages.html @@ -0,0 +1,63 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for pages, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
PagesPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_words.html b/ietf/templates/stats/document_stats_words.html new file mode 100644 index 000000000..35f9536ea --- /dev/null +++ b/ietf/templates/stats/document_stats_words.html @@ -0,0 +1,63 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for pages, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
WordsPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_yearly.html b/ietf/templates/stats/document_stats_yearly.html new file mode 100644 index 000000000..b77988448 --- /dev/null +++ b/ietf/templates/stats/document_stats_yearly.html @@ -0,0 +1,53 @@ +

{{ stats_title }}

+ +
+ + diff --git a/ietf/templates/stats/includes/number_with_details_cell.html b/ietf/templates/stats/includes/number_with_details_cell.html new file mode 100644 index 000000000..b007d4158 --- /dev/null +++ b/ietf/templates/stats/includes/number_with_details_cell.html @@ -0,0 +1,8 @@ +{% if content_limit and count <= content_limit %} + {% for n in names %} + {{ n }}
+ {% endfor %} +{% else %} + {# {{ count }} #} + {{ count }} +{% endif %} diff --git a/ietf/templates/stats/index.html b/ietf/templates/stats/index.html index 11b9bb8e5..a0259f7ee 100644 --- a/ietf/templates/stats/index.html +++ b/ietf/templates/stats/index.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% load origin %}{% origin %} +{% load origin %} {% load ietf_filters staticfiles bootstrap3 %} @@ -9,10 +9,12 @@

{% block title %}Statistics{% endblock %}

-

Currently, there are statistics for:

+

Statistics on...

{% endblock %} diff --git a/ietf/templates/stats/known_countries_list.html b/ietf/templates/stats/known_countries_list.html new file mode 100644 index 000000000..37f4fa8cb --- /dev/null +++ b/ietf/templates/stats/known_countries_list.html @@ -0,0 +1,47 @@ +{% extends "base.html" %} + +{% load origin %} + +{% load ietf_filters staticfiles bootstrap3 %} + +{% block content %} + {% origin %} + +

{% block title %}Countries known to the Datatracker{% endblock %}

+ +

In case you think a country or an alias is missing from the list, you can file a ticket.

+ + {% if request.user.is_staff %} +

Note: since you're an admin, the country names are linked to their corresponding admin page.

+ {% endif %} + + + + + + + + + + {% for c in countries %} + + + + + {% endfor %} + +
NameAliases (lowercase aliases are matched case-insensitive)
+ {% if request.user.is_staff %} + + {% endif %} + {{ c.name }} + {% if request.user.is_staff %} + + {% endif %} + + {% for a in c.aliases %} + {{ a.alias }}{% if not forloop.last %},{% endif %} + {% endfor %} +
+ +{% endblock %} diff --git a/ietf/templates/stats/meeting_stats.html b/ietf/templates/stats/meeting_stats.html new file mode 100644 index 000000000..c79c67e57 --- /dev/null +++ b/ietf/templates/stats/meeting_stats.html @@ -0,0 +1,46 @@ +{% extends "base.html" %} + +{% load origin %} + +{% load ietf_filters staticfiles bootstrap3 %} + +{% block title %}{{ stats_title }}{% endblock %} + +{% block pagehead %} + +{% endblock %} + +{% block content %} + {% origin %} + +

Meeting Statistics

+ + {% if meeting %} +

+ « Back to overview +

+ {% endif %} + +
+
+ Attendees: + +
+ {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
+
+
+ +
+ {% include content_template %} +
+{% endblock %} + +{% block js %} + + + + +{% endblock %} diff --git a/ietf/templates/stats/meeting_stats_continent.html b/ietf/templates/stats/meeting_stats_continent.html new file mode 100644 index 000000000..3ef272fc6 --- /dev/null +++ b/ietf/templates/stats/meeting_stats_continent.html @@ -0,0 +1,64 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for continent, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
ContinentPercentage of attendeesAttendees
{{ continent|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/meeting_stats_country.html b/ietf/templates/stats/meeting_stats_country.html new file mode 100644 index 000000000..1940b82e9 --- /dev/null +++ b/ietf/templates/stats/meeting_stats_country.html @@ -0,0 +1,99 @@ +

{{ stats_title }}

+ +
+ + + +
+ + + +

Data

+ + + + + + + + + + + {% for country, percentage, count, names in table_data %} + + + + + + {% endfor %} + +
CountryPercentage of attendeesAttendees
{{ country|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
+ +

EU (European Union) is not a country, but has been added for reference, as the sum of + all current EU member countries: + {% for c in eu_countries %}{{ c.name }}{% if not forloop.last %}, {% endif %}{% endfor %}. +

diff --git a/ietf/templates/stats/meeting_stats_overview.html b/ietf/templates/stats/meeting_stats_overview.html new file mode 100644 index 000000000..821767843 --- /dev/null +++ b/ietf/templates/stats/meeting_stats_overview.html @@ -0,0 +1,77 @@ +

{{ stats_title }}

+ +
+ + + +{% if table_data %} +

Data

+ + + + + + + + + + {% for meeting_number, label, url, count, names in table_data %} + + + + + {% endfor %} + +
MeetingAttendees
{{ label }}{% include "stats/includes/number_with_details_cell.html" %}
+ +{% endif %} diff --git a/ietf/templates/submit/announce_to_lists.txt b/ietf/templates/submit/announce_to_lists.txt index 7ddcad55f..59c00c32f 100644 --- a/ietf/templates/submit/announce_to_lists.txt +++ b/ietf/templates/submit/announce_to_lists.txt @@ -3,7 +3,7 @@ A New Internet-Draft is available from the on-line Internet-Drafts directories. {% if submission.group %}This draft is a work item of the {{ submission.group.name }}{% if group.type.name %} {{ group.type.name }}{% endif %} of the {% if group.type_id == "rg" %}IRTF{% else %}IETF{% endif %}.{% endif %} Title : {{ submission.title }} - Author{{ submission.authors_parsed|pluralize:" ,s" }} : {% for author in submission.authors_parsed %}{{ author.name }}{% if not forloop.last %} + Author{{ submission.authors|pluralize:" ,s" }} : {% for author in submission.authors %}{{ author.name }}{% if not forloop.last %} {% endif %}{% endfor %} Filename : {{ submission.name }}-{{ submission.rev }}.txt Pages : {{ submission.pages }} diff --git a/ietf/templates/submit/approval_request.txt b/ietf/templates/submit/approval_request.txt index 83677d166..828b14bc4 100644 --- a/ietf/templates/submit/approval_request.txt +++ b/ietf/templates/submit/approval_request.txt @@ -22,7 +22,7 @@ To approve the draft, go to this URL (note: you need to login to be able to appr Authors: -{% for author in submission.authors_parsed %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} +{% for author in submission.authors %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} {% endfor %} {% endautoescape %} diff --git a/ietf/templates/submit/manual_post_request.txt b/ietf/templates/submit/manual_post_request.txt index b4e595ab2..78fd563dc 100644 --- a/ietf/templates/submit/manual_post_request.txt +++ b/ietf/templates/submit/manual_post_request.txt @@ -31,7 +31,7 @@ I-D Submission Tool URL: Authors: -{% for author in submission.authors_parsed %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} +{% for author in submission.authors %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} {% endfor %} Comment to the secretariat: diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index 147947c9a..473829130 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -48,9 +48,7 @@ {% if check.errors %}

The {{check.checker}} returned {{ check.errors }} error{{ check.errors|pluralize }} - and {{ check.warnings }} warning - - {{ check.warnings|pluralize }} ; click the button + and {{ check.warnings }} warning{{ check.warnings|pluralize }}; click the button below to see details. Please fix those, and resubmit.

{% elif check.warnings %} @@ -197,17 +195,38 @@ Authors - {% with submission.authors_parsed as authors %} - {{ authors|length }} author{{ authors|pluralize }} - {% endwith %} + {{ submission.authors|length }} author{{ submission.authors|pluralize }} {% if errors.authors %}

{{ errors.authors|safe }}

{% endif %} - {% for author in submission.authors_parsed %} + {% for author in submission.authors %} Author {{ forloop.counter }} - {{ author.name }} {% if author.email %}<{{ author.email }}>{% endif %} + + {{ author.name }} {% if author.email %}<{{ author.email }}>{% endif %} + - + {% if author.affiliation %} + {{ author.affiliation }} + {% else %} + unknown affiliation + {% endif %} + - + {% if author.country %} + {{ author.country }} + {% if author.cleaned_country and author.country != author.cleaned_country %} + (understood to be {{ author.cleaned_country }}) + {% endif %} + {% else %} + unknown country + {% endif %} + + {% if author.country and not author.cleaned_country %} +
+ Unrecognized country: "{{ author.country }}": See recognized country names. + {% endif %} + + {% endfor %} @@ -231,6 +250,14 @@ File size {{ submission.file_size|filesizeformat }} + + + Formal languages used + + {% for l in submission.formal_languages.all %}{{ l.name }}{% if not forloop.last %}, {% endif %}{% empty %}None recognized{% endfor %} + {% if errors.formal_languages %}

{{ errors.formal_languages }}

{% endif %} + + {% if can_edit %} @@ -311,7 +338,7 @@ {% if user|has_role:"Secretariat" %}

- Send Email + Send Email {% endif %} {% if show_send_full_url %} diff --git a/ietf/templates/submit/submitter_form.html b/ietf/templates/submit/submitter_form.html index 93994c10c..1564dd3e5 100644 --- a/ietf/templates/submit/submitter_form.html +++ b/ietf/templates/submit/submitter_form.html @@ -11,8 +11,8 @@ {% load ietf_filters %} {% buttons %} - {% for author in submission.authors_parsed %} - + {% for author in submission.authors %} + {% endfor %} {% endbuttons %} diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index 14a67f8dc..e605c7018 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -263,8 +263,8 @@ class Draft(): sentence = True if re.search("[^ \t]", line): if newpage: - # 33 is a somewhat arbitrary count for a 'short' line - shortthis = len(line.strip()) < 33 # 33 is a somewhat arbitrary count for a 'short' line + # 36 is a somewhat arbitrary count for a 'short' line + shortthis = len(line.strip()) < 36 # 36 is a somewhat arbitrary count for a 'short' line if sentence or (shortprev and not shortthis): stripped += [""] else: @@ -273,7 +273,7 @@ class Draft(): blankcount = 0 sentence = False newpage = False - shortprev = len(line.strip()) < 33 # 33 is a somewhat arbitrary count for a 'short' line + shortprev = len(line.strip()) < 36 # 36 is a somewhat arbitrary count for a 'short' line if re.search("[.:]$", line): sentence = True if re.search("^[ \t]*$", line): @@ -296,6 +296,37 @@ class Draft(): self._pagecount = count_pages return self._pagecount + # ------------------------------------------------------------------ + def get_wordcount(self): + count = 0 + # match any sequence of non-white-space characters like the Unix command "wc" + word_re = re.compile(r'\S+', re.UNICODE) + for l in self.lines: + count += sum(1 for _ in word_re.finditer(l)) + return count + + # ------------------------------------------------------------------ + def get_formal_languages(self): + language_regexps = [ + ("abnf", [re.compile(r"\bABNF"), re.compile(r" +[a-zA-Z][a-zA-Z0-9_-]* +=[/ ]")]), + ("asn1", [re.compile(r'DEFINITIONS +::= +BEGIN')]), + ("cbor", [re.compile(r'\b(?:CBOR|CDDL)\b'), re.compile(r" +[a-zA-Z][a-zA-Z0-9_-]* += +[\{\[\(]")]), + ("ccode", [re.compile(r"(?:\+\+\))|(?:for \(i)|(?: [!=]= 0\) \{)|(?: struct [a-zA-Z_0-9]+ \{)")]), + ("json", [re.compile(r'\bJSON\b'), re.compile(r" \"[^\"]+\" ?: [a-zA-Z0-9\.\"\{\[]")]), + ("xml", [re.compile(r"<\?xml")]), + ] + already_matched = set() + for l in self.lines: + for lang_name, patterns in language_regexps: + for p in patterns: + if p not in already_matched and p.search(l): + already_matched.add(p) + return [ + lang_name + for lang_name, patterns in language_regexps + if all(p in already_matched for p in patterns) + ] + # ---------------------------------------------------------------------- def get_status(self): if self._status == None: @@ -825,7 +856,8 @@ class Draft(): nonblank_count = 0 blanklines = 0 email = None - for line in self.lines[start+1:]: + country = None + for line_offset, line in enumerate(self.lines[start+1:]): _debug( " " + line.strip()) # Break on the second blank line if not line: @@ -866,15 +898,18 @@ class Draft(): else: pass - try: - column = line[beg:end].strip() - except: - column = line - column = re.sub(" *\(at\) *", "@", column) - column = re.sub(" *\(dot\) *", ".", column) - column = re.sub(" +at +", "@", column) - column = re.sub(" +dot +", ".", column) - column = re.sub("&cisco.com", "@cisco.com", column) + def columnify(l): + try: + column = l.replace('\t', 8 * ' ')[max(0, beg - 1):end].strip() + except: + column = l + column = re.sub(" *(?:\(at\)| | at ) *", "@", column) + column = re.sub(" *(?:\(dot\)| | dot ) *", ".", column) + column = re.sub("&cisco.com", "@cisco.com", column) + column = column.replace("\xa0", " ") + return column + + column = columnify(line) # if re.search("^\w+: \w+", column): # keyword = True @@ -885,13 +920,42 @@ class Draft(): # break #_debug( " Column text :: " + column) + if nonblank_count >= 2 and blanklines == 0: + # Usually, the contact info lines will look + # like this: "Email: someone@example.com" or + # "Tel: +1 (412)-2390 23123", but sometimes + # the : is left out. That's okay for things we + # can't misinterpret, but "tel" may match "Tel + # Aviv 69710, Israel" so match + # - misc contact info + # - tel/fax [number] + # - [phone number] + # - [email] + + other_contact_info_regex = re.compile(r'^(((contact )?e|\(e|e-|m|electronic )?mail|email_id|mailto|e-main|(tele)?phone|voice|mobile|work|uri|url|tel:)\b|^((ph|tel\.?|telefax|fax) *[:.]? *\(?( ?\+ ?)?[0-9]+)|^(\++[0-9]+|\(\+*[0-9]+\)|\(dsn\)|[0-9]+)([ -.]*\b|\b[ -.]*)(([0-9]{2,}|\([0-9]{2,}\)|(\([0-9]\)|[0-9])[ -][0-9]{2,}|\([0-9]\)[0-9]+)([ -.]+([0-9]+|\([0-9]+\)))+|([0-9]{7,}|\([0-9]{7,}\)))|^(?|^https?://|^www\.') + next_line_index = start + 1 + line_offset + 1 + + if (not country + and not other_contact_info_regex.search(column.lower()) + and next_line_index < len(self.lines)): + + next_line_lower = columnify(self.lines[next_line_index]).lower().strip() + + if not next_line_lower or other_contact_info_regex.search(next_line_lower): + # country should be here, as the last + # part of the address, right before an + # empty line or other contact info + country = column.strip() or None + _debug(" Country: %s" % country) + _debug("3: authors[%s]: %s" % (i, authors[i])) emailmatch = re.search("[-A-Za-z0-9_.+]+@[-A-Za-z0-9_.]+", column) if emailmatch and not "@" in author: email = emailmatch.group(0).lower() break - authors[i] = authors[i] + ( email, ) + + authors[i] = authors[i] + ( email, country) else: if not author in ignore: companies[i] = authors[i] @@ -917,8 +981,8 @@ class Draft(): _debug(" * Final company list: %s" % (companies,)) _debug(" * Final companies_seen: %s" % (companies_seen,)) self._author_info = authors - self._authors_with_firm = [ "%s <%s> (%s)"%(full,email,company) for full,first,middle,last,suffix,email,company in authors ] # pyflakes:ignore - self._authors = [ "%s <%s>"%(full,email) if email else full for full,first,middle,last,suffix,email,company in authors ] + self._authors_with_firm = [ "%s <%s> (%s)"%(full,email,company) for full,first,middle,last,suffix,email,country,company in authors ] # pyflakes:ignore + self._authors = [ "%s <%s>"%(full,email) if email else full for full,first,middle,last,suffix,email,country,company in authors ] self._authors.sort() _debug(" * Final author list: " + ", ".join(self._authors)) _debug("-"*72) @@ -1138,10 +1202,10 @@ def getmeta(fn): def _output(docname, fields, outfile=sys.stdout): global company_domain if opt_getauthors: - # Output an (incomplete!) getauthors-compatible format. Country - # information is always UNKNOWN, and information about security and - # iana sections presence is missing. - for full,first,middle,last,suffix,email,company in fields["_authorlist"]: + # Output an (incomplete!) getauthors-compatible format. + # Information about security and iana sections presence is + # missing. + for full,first,middle,last,suffix,email,country,company in fields["_authorlist"]: if company in company_domain: company = company_domain[company] else: @@ -1152,7 +1216,7 @@ def _output(docname, fields, outfile=sys.stdout): fields["name"] = full fields["email"] = email fields["company"] = company - fields["country"] = "UNKNOWN" + fields["country"] = country or "UNKNOWN" try: year, month, day = fields["doccreationdate"].split("-") except ValueError: diff --git a/ietf/utils/test_data.py b/ietf/utils/test_data.py index 836231e85..282478a52 100644 --- a/ietf/utils/test_data.py +++ b/ietf/utils/test_data.py @@ -95,7 +95,7 @@ def make_immutable_base_data(): # one area area = create_group(name="Far Future", acronym="farfut", type_id="area", parent=ietf) - create_person(area, "ad", name="Areað Irector", username="ad", email_address="aread@ietf.org") + create_person(area, "ad", name=u"Areað Irector", username="ad", email_address="aread@ietf.org") # second area opsarea = create_group(name="Operations", acronym="ops", type_id="area", parent=ietf) @@ -297,7 +297,10 @@ def make_test_data(): DocumentAuthor.objects.create( document=draft, - author=Email.objects.get(address="aread@ietf.org"), + person=Person.objects.get(email__address="aread@ietf.org"), + email=Email.objects.get(address="aread@ietf.org"), + country="Germany", + affiliation="IETF", order=1 ) @@ -444,8 +447,11 @@ def make_downref_test_data(): DocumentAuthor.objects.create( document=draft, - author=Email.objects.get(address="aread@ietf.org"), - order=1, + person=Person.objects.get(email__address="aread@ietf.org"), + email=Email.objects.get(address="aread@ietf.org"), + country="US", + affiliation="", + order=1 ) rfc_doc_alias = DocAlias.objects.get(name='rfc9998') diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 2970d0a4c..0afa4ad09 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -265,11 +265,11 @@ class CoverageTest(TestCase): if self.runner.run_full_test_suite: # Permit 0.02% variation in results -- otherwise small code changes become a pain fudge_factor = 0.0002 + self.assertLessEqual(len(test_missing), len(master_missing), + msg = "New %s without test coverage since %s: %s" % (test, latest_coverage_version, list(set(test_missing) - set(master_missing)))) self.assertGreaterEqual(test_coverage, master_coverage - fudge_factor, msg = "The %s coverage percentage is now lower (%.2f%%) than for version %s (%.2f%%)" % ( test, test_coverage*100, latest_coverage_version, master_coverage*100, )) - self.assertLessEqual(len(test_missing), len(master_missing), - msg = "New %s without test coverage since %s: %s" % (test, latest_coverage_version, list(set(test_missing) - set(master_missing)))) def template_coverage_test(self): global loaded_templates diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index 0e7b1973d..6e240f763 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -69,7 +69,7 @@ def unicontent(r): def reload_db_objects(*objects): """Rerequest the given arguments from the database so they're refreshed, to be used like - foo, bar = reload_objects(foo, bar)""" + foo, bar = reload_db_objects(foo, bar)""" t = tuple(o.__class__.objects.get(pk=o.pk) for o in objects) if len(objects) == 1: @@ -108,6 +108,28 @@ class TestCase(django.test.TestCase): os.mkdir(path) return path + def assertNoFormPostErrors(self, response, error_css_selector=".has-error"): + """Try to fish out form errors, if none found at least check the + status code to be a redirect. + + Assumptions: + - a POST is followed by a 302 redirect + - form errors can be found with a simple CSS selector + + """ + + if response.status_code == 200: + from pyquery import PyQuery + from lxml import html + self.maxDiff = None + + errors = [html.tostring(n).decode() for n in PyQuery(response.content)(error_css_selector)] + if errors: + explanation = u"{} != {}\nGot form back with errors:\n----\n".format(response.status_code, 302) + u"----\n".join(errors) + self.assertEqual(response.status_code, 302, explanation) + + self.assertEqual(response.status_code, 302) + def __str__(self): return "%s (%s.%s)" % (self._testMethodName, strclass(self.__class__),self._testMethodName) diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 66fe27787..c66ea849d 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -168,7 +168,7 @@ class TemplateChecksTestCase(TestCase): Check that an URLNode's callback is in callbacks. """ cb = node.view_name.token.strip("\"'") - if cb in callbacks: + if cb in callbacks or cb.startswith("admin:"): return [] else: return [ (origin, cb), ]