From dac430c84e1c6eecc8d2534afa17254b46894ed2 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 6 Jan 2017 15:10:49 +0000 Subject: [PATCH 01/49] Add branch from trunk @r12628 for the author statistics project, add document statistics page with the first statistics with the number of authors per document - Legacy-Id: 12629 --- ietf/static/ietf/css/ietf.css | 5 + ietf/static/ietf/js/document-stats.js | 34 +++++ ietf/stats/urls.py | 1 + ietf/stats/views.py | 132 +++++++++++++++--- ietf/templates/stats/document_stats.html | 48 +++++++ .../stats/document_stats_authors.html | 71 ++++++++++ ietf/templates/stats/index.html | 5 +- 7 files changed, 270 insertions(+), 26 deletions(-) create mode 100644 ietf/static/ietf/js/document-stats.js create mode 100644 ietf/templates/stats/document_stats.html create mode 100644 ietf/templates/stats/document_stats_authors.html diff --git a/ietf/static/ietf/css/ietf.css b/ietf/static/ietf/css/ietf.css index 025bc0168..76b391d58 100644 --- a/ietf/static/ietf/css/ietf.css +++ b/ietf/static/ietf/css/ietf.css @@ -568,6 +568,11 @@ table.simple-table td:last-child { width: 7em; } +.popover .docname { + padding-left: 1em; + text-indent: -1em; +} + .stats-time-graph { height: 15em; } diff --git a/ietf/static/ietf/js/document-stats.js b/ietf/static/ietf/js/document-stats.js new file mode 100644 index 000000000..f9a08fa2c --- /dev/null +++ b/ietf/static/ietf/js/document-stats.js @@ -0,0 +1,34 @@ +$(document).ready(function () { + if (window.chartConf) { + var chart = Highcharts.chart('chart', window.chartConf); + } + + $(".popover-docnames").each(function () { + var stdNameRegExp = new RegExp("^(rfc|bcp|fyi|std)[0-9]+$", 'i'); + + var html = []; + $.each(($(this).data("docnames") || "").split(" "), function (i, docname) { + if (!$.trim(docname)) + return; + + var displayName = docname; + + if (stdNameRegExp.test(docname)) + displayName = docname.slice(0, 3).toUpperCase() + " " + docname.slice(3); + + html.push('
' + displayName + '
'); + }); + + if ($(this).data("sliced")) + html.push('
…
'); + + $(this).popover({ + trigger: "focus", + template: '', + content: html.join(""), + html: true + }).on("click", function (e) { + e.preventDefault(); + }); + }); +}); diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 8a05f9659..641187350 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -5,5 +5,6 @@ import ietf.stats.views urlpatterns = patterns('', url("^$", ietf.stats.views.stats_index), + url("^document/(?:(?Pauthors|pages|format|spectech)/)?(?:(?Pall|rfc|draft)/)?$", ietf.stats.views.document_stats), url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, ietf.stats.views.review_stats), ) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 57c9cf5bf..63b3da920 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,9 +1,12 @@ import datetime, itertools, json, calendar +from collections import defaultdict from django.shortcuts import render from django.contrib.auth.decorators import login_required from django.core.urlresolvers import reverse as urlreverse from django.http import HttpResponseRedirect, HttpResponseForbidden +from django.db.models import Count +from django.utils.safestring import mark_safe import dateutil.relativedelta @@ -15,11 +18,116 @@ from ietf.review.utils import (extract_review_request_data, from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewRequestStateName, ReviewResultName +from ietf.doc.models import Document from ietf.ietfauth.utils import has_role def stats_index(request): return render(request, "stats/index.html") +def generate_query_string(query_dict, overrides): + query_part = u"" + + if query_dict or overrides: + d = query_dict.copy() + for k, v in overrides.iteritems(): + if type(v) in (list, tuple): + if not v: + if k in d: + del d[k] + else: + d.setlist(k, v) + else: + if v is None or v == u"": + if k in d: + del d[k] + else: + d[k] = v + + if d: + query_part = u"?" + d.urlencode() + + return query_part + + +def document_stats(request, stats_type=None, document_state=None): + def build_document_stats_url(stats_type_override=Ellipsis, document_state_override=Ellipsis, get_overrides={}): + kwargs = { + "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, + "document_state": document_state if document_state_override is Ellipsis else document_state_override, + } + + return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) + + # statistics type - one of the tables or the chart + possible_stats_types = [ + ("authors", "Number of authors"), +# ("pages", "Pages"), +# ("format", "Format"), +# ("spectech", "Specification techniques"), + ] + + possible_stats_types = [ (slug, label, build_document_stats_url(stats_type_override=slug)) + for slug, label in possible_stats_types ] + + if not stats_type: + return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_stats_types[0][0])) + + possible_document_states = [ + ("all", "All"), + ("rfc", "RFCs"), + ("draft", "Drafts (not published as RFC)"), + ] + + possible_document_states = [ (slug, label, build_document_stats_url(document_state_override=slug)) + for slug, label in possible_document_states ] + + if not document_state: + return HttpResponseRedirect(build_document_stats_url(document_state_override=possible_document_states[0][0])) + + + # filter documents + doc_qs = Document.objects.filter(type="draft") + + if document_state == "rfc": + doc_qs = doc_qs.filter(states__type="draft", states__slug="rfc") + elif document_state == "draft": + doc_qs = doc_qs.exclude(states__type="draft", states__slug="rfc") + + chart_data = [] + table_data = [] + stats_title = "" + + if stats_type == "authors": + stats_title = "Number of authors for each document" + + groups = defaultdict(list) + + for name, author_count in doc_qs.values_list("name").annotate(Count("authors")).iterator(): + groups[author_count].append(name) + + total_docs = sum(len(names) for author_count, names in groups.iteritems()) + + series_data = [] + for author_count, names in sorted(groups.iteritems(), key=lambda t: t[0]): + series_data.append((author_count, len(names) * 100.0 / total_docs)) + table_data.append((author_count, names)) + + chart_data.append({ + "data": series_data, + "name": "Percentage of documents", + }) + + + return render(request, "stats/document_stats.html", { + "chart_data": mark_safe(json.dumps(chart_data)), + "table_data": table_data, + "stats_title": stats_title, + "possible_stats_types": possible_stats_types, + "stats_type": stats_type, + "possible_document_states": possible_document_states, + "document_state": document_state, + }) + @login_required def review_stats(request, stats_type=None, acronym=None): # This view is a bit complex because we want to show a bunch of @@ -39,29 +147,7 @@ def review_stats(request, stats_type=None, acronym=None): if acr: kwargs["acronym"] = acr - base_url = urlreverse(review_stats, kwargs=kwargs) - query_part = u"" - - if request.GET or get_overrides: - d = request.GET.copy() - for k, v in get_overrides.iteritems(): - if type(v) in (list, tuple): - if not v: - if k in d: - del d[k] - else: - d.setlist(k, v) - else: - if v is None or v == u"": - if k in d: - del d[k] - else: - d[k] = v - - if d: - query_part = u"?" + d.urlencode() - - return base_url + query_part + return urlreverse(review_stats, kwargs=kwargs) + generate_query_string(request.GET, get_overrides) def get_choice(get_parameter, possible_choices, multiple=False): values = request.GET.getlist(get_parameter) diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html new file mode 100644 index 000000000..e11aaed4e --- /dev/null +++ b/ietf/templates/stats/document_stats.html @@ -0,0 +1,48 @@ +{% extends "base.html" %} + +{% load origin %} + +{% load ietf_filters staticfiles bootstrap3 %} + +{% block title %}{{ stats_title }}{% endblock %} + +{% block pagehead %} + +{% endblock %} + +{% block content %} + {% origin %} + +

Document statistics

+ +
+
+ Show: +
+ {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
+
+ +
+ Document types: +
+ {% for slug, label, url in possible_document_states %} + {{ label }} + {% endfor %} +
+
+
+ + {% if stats_type == "authors" %} + {% include "stats/document_stats_authors.html" %} + {% endif %} +{% endblock %} + +{% block js %} + + + + +{% endblock %} diff --git a/ietf/templates/stats/document_stats_authors.html b/ietf/templates/stats/document_stats_authors.html new file mode 100644 index 000000000..3ecc19562 --- /dev/null +++ b/ietf/templates/stats/document_stats_authors.html @@ -0,0 +1,71 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + {% for author_count, names in table_data %} + + + + + {% endfor %} + +
AuthorsDocuments
{{ author_count }}{{ names|length }}
diff --git a/ietf/templates/stats/index.html b/ietf/templates/stats/index.html index 11b9bb8e5..77b8b7925 100644 --- a/ietf/templates/stats/index.html +++ b/ietf/templates/stats/index.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% load origin %}{% origin %} +{% load origin %} {% load ietf_filters staticfiles bootstrap3 %} @@ -9,9 +9,8 @@

{% block title %}Statistics{% endblock %}

-

Currently, there are statistics for:

- From 7dc370baa37cdc03eeb3677844b4e124a4d155a9 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 6 Jan 2017 15:31:32 +0000 Subject: [PATCH 02/49] Improve wording a bit in the document statistics, turn of chart animation, it makes the page seem sluggish - Legacy-Id: 12630 --- ietf/stats/urls.py | 2 +- ietf/stats/views.py | 41 ++++++++++++------- ietf/templates/stats/document_stats.html | 6 +-- .../stats/document_stats_authors.html | 9 +++- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 641187350..82fcd5743 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -5,6 +5,6 @@ import ietf.stats.views urlpatterns = patterns('', url("^$", ietf.stats.views.stats_index), - url("^document/(?:(?Pauthors|pages|format|spectech)/)?(?:(?Pall|rfc|draft)/)?$", ietf.stats.views.document_stats), + url("^document/(?:(?Pauthors|pages|format|spectech)/)?(?:(?Pall|rfc|draft)/)?$", ietf.stats.views.document_stats), url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, ietf.stats.views.review_stats), ) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 63b3da920..77c527157 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -49,11 +49,11 @@ def generate_query_string(query_dict, overrides): return query_part -def document_stats(request, stats_type=None, document_state=None): - def build_document_stats_url(stats_type_override=Ellipsis, document_state_override=Ellipsis, get_overrides={}): +def document_stats(request, stats_type=None, document_type=None): + def build_document_stats_url(stats_type_override=Ellipsis, document_type_override=Ellipsis, get_overrides={}): kwargs = { "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, - "document_state": document_state if document_state_override is Ellipsis else document_state_override, + "document_type": document_type if document_type_override is Ellipsis else document_type_override, } return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) @@ -72,33 +72,42 @@ def document_stats(request, stats_type=None, document_state=None): if not stats_type: return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_stats_types[0][0])) - possible_document_states = [ + possible_document_types = [ ("all", "All"), ("rfc", "RFCs"), - ("draft", "Drafts (not published as RFC)"), + ("draft", "Drafts"), ] - possible_document_states = [ (slug, label, build_document_stats_url(document_state_override=slug)) - for slug, label in possible_document_states ] + possible_document_types = [ (slug, label, build_document_stats_url(document_type_override=slug)) + for slug, label in possible_document_types ] - if not document_state: - return HttpResponseRedirect(build_document_stats_url(document_state_override=possible_document_states[0][0])) + if not document_type: + return HttpResponseRedirect(build_document_stats_url(document_type_override=possible_document_types[0][0])) # filter documents doc_qs = Document.objects.filter(type="draft") - if document_state == "rfc": + if document_type == "rfc": doc_qs = doc_qs.filter(states__type="draft", states__slug="rfc") - elif document_state == "draft": + elif document_type == "draft": doc_qs = doc_qs.exclude(states__type="draft", states__slug="rfc") chart_data = [] table_data = [] + + if document_type == "all": + doc_label = "document" + elif document_type == "rfc": + doc_label = "RFC" + elif document_type == "draft": + doc_label = "draft" + stats_title = "" if stats_type == "authors": - stats_title = "Number of authors for each document" + + stats_title = "Number of authors for each {}".format(doc_label) groups = defaultdict(list) @@ -114,7 +123,8 @@ def document_stats(request, stats_type=None, document_state=None): chart_data.append({ "data": series_data, - "name": "Percentage of documents", + "name": "Percentage of {}s".format(doc_label), + "animation": False, }) @@ -124,8 +134,9 @@ def document_stats(request, stats_type=None, document_state=None): "stats_title": stats_title, "possible_stats_types": possible_stats_types, "stats_type": stats_type, - "possible_document_states": possible_document_states, - "document_state": document_state, + "possible_document_types": possible_document_types, + "document_type": document_type, + "doc_label": doc_label, }) @login_required diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html index e11aaed4e..0ba8e3ccb 100644 --- a/ietf/templates/stats/document_stats.html +++ b/ietf/templates/stats/document_stats.html @@ -26,10 +26,10 @@
- Document types: + Document type:
- {% for slug, label, url in possible_document_states %} - {{ label }} + {% for slug, label, url in possible_document_types %} + {{ label }} {% endfor %}
diff --git a/ietf/templates/stats/document_stats_authors.html b/ietf/templates/stats/document_stats_authors.html index 3ecc19562..4548ae58b 100644 --- a/ietf/templates/stats/document_stats_authors.html +++ b/ietf/templates/stats/document_stats_authors.html @@ -7,6 +7,12 @@ chart: { type: 'column' }, + credits: { + enabled: false, + }, + exporting: { + fallbackToExportServer: false, + }, title: { text: '{{ stats_title|escapejs }}' }, @@ -18,7 +24,7 @@ }, yAxis: { title: { - text: 'Percentage of documents' + text: 'Percentage of {{ doc_label }}s' }, labels: { formatter: function () { @@ -32,7 +38,6 @@ tooltip: { formatter: function () { var s = '' + this.x + ' ' + (this.x == 1 ? "author" : 'authors') + ''; - console.log(this.points) $.each(this.points, function () { s += '
' + this.series.name + ': ' + From 13f3b4ed1ad19101a2d1db7bb33f922eae345b36 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 6 Jan 2017 16:46:28 +0000 Subject: [PATCH 03/49] Add simple test for document stats - Legacy-Id: 12631 --- ietf/stats/tests.py | 28 +++++++++++++++++++ .../stats/document_stats_authors.html | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 5852f3577..af8f5b431 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -12,6 +12,34 @@ class StatisticsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) + def test_document_stats(self): + make_test_data() + + # check redirect + url = urlreverse(ietf.stats.views.document_stats) + + authors_url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": "authors" }) + + r = self.client.get(url) + self.assertEqual(r.status_code, 302) + self.assertTrue(authors_url in r["Location"]) + + authors_all_url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": "authors", "document_type": "all" }) + + r = self.client.get(authors_url) + self.assertEqual(r.status_code, 302) + self.assertTrue(authors_all_url in r["Location"]) + + # check various stats types + for stats_type in ["authors"]: + for document_type in ["all", "rfc", "draft"]: + url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type, "document_type": document_type }) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(q('#chart')) + self.assertTrue(q('table.stats-data')) + def test_review_stats(self): doc = make_test_data() review_req = make_review_data(doc) diff --git a/ietf/templates/stats/document_stats_authors.html b/ietf/templates/stats/document_stats_authors.html index 4548ae58b..e8d1141fb 100644 --- a/ietf/templates/stats/document_stats_authors.html +++ b/ietf/templates/stats/document_stats_authors.html @@ -54,7 +54,7 @@

Data

- +
From 656ed8c89d79d40114e3010ff41a06e595d681a1 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Tue, 10 Jan 2017 18:27:15 +0000 Subject: [PATCH 04/49] Add statistics for pages in documents, refactoring a bit to share more code - Legacy-Id: 12639 --- ietf/static/ietf/js/document-stats.js | 12 ++++ ietf/stats/tests.py | 2 +- ietf/stats/views.py | 32 ++++++++-- ietf/templates/stats/document_stats.html | 2 + .../stats/document_stats_authors.html | 24 ++------ .../templates/stats/document_stats_pages.html | 58 +++++++++++++++++++ .../stats/includes/docnames_cell.html | 1 + 7 files changed, 107 insertions(+), 24 deletions(-) create mode 100644 ietf/templates/stats/document_stats_pages.html create mode 100644 ietf/templates/stats/includes/docnames_cell.html diff --git a/ietf/static/ietf/js/document-stats.js b/ietf/static/ietf/js/document-stats.js index f9a08fa2c..922a7205e 100644 --- a/ietf/static/ietf/js/document-stats.js +++ b/ietf/static/ietf/js/document-stats.js @@ -1,5 +1,17 @@ $(document).ready(function () { if (window.chartConf) { + window.chartConf.credits = { + enabled: false + }; + window.chartConf.exporting = { + fallbackToExportServer: false + }; + + if (!window.chartConf.legend) + window.chartConf.legend = { + enabled: false + }; + var chart = Highcharts.chart('chart', window.chartConf); } diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index af8f5b431..33c76409f 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -31,7 +31,7 @@ class StatisticsTests(TestCase): self.assertTrue(authors_all_url in r["Location"]) # check various stats types - for stats_type in ["authors"]: + for stats_type in ["authors", "pages"]: for document_type in ["all", "rfc", "draft"]: url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type, "document_type": document_type }) r = self.client.get(url) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 77c527157..5e1211daa 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -61,7 +61,7 @@ def document_stats(request, stats_type=None, document_type=None): # statistics type - one of the tables or the chart possible_stats_types = [ ("authors", "Number of authors"), -# ("pages", "Pages"), + ("pages", "Pages"), # ("format", "Format"), # ("spectech", "Specification techniques"), ] @@ -106,7 +106,6 @@ def document_stats(request, stats_type=None, document_type=None): stats_title = "" if stats_type == "authors": - stats_title = "Number of authors for each {}".format(doc_label) groups = defaultdict(list) @@ -118,15 +117,38 @@ def document_stats(request, stats_type=None, document_type=None): series_data = [] for author_count, names in sorted(groups.iteritems(), key=lambda t: t[0]): - series_data.append((author_count, len(names) * 100.0 / total_docs)) - table_data.append((author_count, names)) + percentage = len(names) * 100.0 / total_docs + series_data.append((author_count, percentage)) + table_data.append((author_count, percentage, names)) chart_data.append({ "data": series_data, - "name": "Percentage of {}s".format(doc_label), "animation": False, }) + elif stats_type == "pages": + stats_title = "Number of pages for each {}".format(doc_label) + + groups = defaultdict(list) + + for name, pages in doc_qs.values_list("name", "pages"): + groups[pages].append(name) + + total_docs = sum(len(names) for pages, names in groups.iteritems()) + + series_data = [] + for pages, names in sorted(groups.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / total_docs + if pages is not None: + series_data.append((pages, len(names))) + table_data.append((pages, percentage, names)) + + chart_data.append({ + "data": series_data, + "animation": False, + }) + + return render(request, "stats/document_stats.html", { "chart_data": mark_safe(json.dumps(chart_data)), diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html index 0ba8e3ccb..576a62ed5 100644 --- a/ietf/templates/stats/document_stats.html +++ b/ietf/templates/stats/document_stats.html @@ -37,6 +37,8 @@ {% if stats_type == "authors" %} {% include "stats/document_stats_authors.html" %} + {% elif stats_type == "pages" %} + {% include "stats/document_stats_pages.html" %} {% endif %} {% endblock %} diff --git a/ietf/templates/stats/document_stats_authors.html b/ietf/templates/stats/document_stats_authors.html index e8d1141fb..143da1114 100644 --- a/ietf/templates/stats/document_stats_authors.html +++ b/ietf/templates/stats/document_stats_authors.html @@ -7,12 +7,6 @@ chart: { type: 'column' }, - credits: { - enabled: false, - }, - exporting: { - fallbackToExportServer: false, - }, title: { text: '{{ stats_title|escapejs }}' }, @@ -32,16 +26,12 @@ } } }, - legend: { - enabled: false, - }, tooltip: { formatter: function () { var s = '' + this.x + ' ' + (this.x == 1 ? "author" : 'authors') + ''; $.each(this.points, function () { - s += '
' + this.series.name + ': ' + - this.y.toFixed(1) + '%'; + s += '
' + chartConf.yAxis.title.text + ': ' + this.y.toFixed(1) + '%'; }); return s; @@ -58,18 +48,16 @@ - + + - {% for author_count, names in table_data %} + {% for author_count, percentage, names in table_data %} - + + {% endfor %} diff --git a/ietf/templates/stats/document_stats_pages.html b/ietf/templates/stats/document_stats_pages.html new file mode 100644 index 000000000..c9a725d0b --- /dev/null +++ b/ietf/templates/stats/document_stats_pages.html @@ -0,0 +1,58 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ +
Authors
AuthorsDocumentsPercentage of {{ doc_label }}s{{ doc_label }}s
{{ author_count }}{{ names|length }}{{ percentage|floatformat:2 }}%{% include "stats/includes/docnames_cell.html" %}
+ + + + + + + + + {% for pages, percentage, names in table_data %} + + + + + + {% endfor %} + +
AuthorsPercentage of {{ doc_label }}s{{ doc_label }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/docnames_cell.html" %}
diff --git a/ietf/templates/stats/includes/docnames_cell.html b/ietf/templates/stats/includes/docnames_cell.html new file mode 100644 index 000000000..fecdbe3a3 --- /dev/null +++ b/ietf/templates/stats/includes/docnames_cell.html @@ -0,0 +1 @@ +{{ names|length }} From 34a9f36534a907ee52e1a387abe8d4385219a2df Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 16 Jan 2017 11:35:48 +0000 Subject: [PATCH 05/49] Add helper for getting word count from draft - Legacy-Id: 12655 --- ietf/utils/draft.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index 896e15a7b..a6cde5a99 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -292,6 +292,15 @@ class Draft(): self._pagecount = count_pages return self._pagecount + # ------------------------------------------------------------------ + def get_wordcount(self): + count = 0 + # match any sequence of non-white-space characters like the Unix command "wc" + word_re = re.compile(r'\S+', re.UNICODE) + for l in self.lines: + count += sum(1 for _ in word_re.finditer(l)) + return count + # ---------------------------------------------------------------------- def get_status(self): if self._status == None: From 63785940339726dc335f42e64430e954c5ada788 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 16 Jan 2017 11:36:38 +0000 Subject: [PATCH 06/49] Add word count and submit format statistics - Legacy-Id: 12656 --- .../doc/migrations/0020_auto_20170112_0753.py | 24 +++ ietf/doc/models.py | 1 + ietf/settings.py | 2 + ietf/stats/backfill_data.py | 58 +++++++ ietf/stats/tests.py | 2 +- ietf/stats/urls.py | 2 +- ietf/stats/views.py | 148 +++++++++++++++--- ietf/templates/stats/document_stats.html | 6 +- .../stats/document_stats_format.html | 60 +++++++ .../templates/stats/document_stats_pages.html | 4 +- .../templates/stats/document_stats_words.html | 58 +++++++ 11 files changed, 337 insertions(+), 28 deletions(-) create mode 100644 ietf/doc/migrations/0020_auto_20170112_0753.py create mode 100644 ietf/stats/backfill_data.py create mode 100644 ietf/templates/stats/document_stats_format.html create mode 100644 ietf/templates/stats/document_stats_words.html diff --git a/ietf/doc/migrations/0020_auto_20170112_0753.py b/ietf/doc/migrations/0020_auto_20170112_0753.py new file mode 100644 index 000000000..da22265f7 --- /dev/null +++ b/ietf/doc/migrations/0020_auto_20170112_0753.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('doc', '0019_auto_20161207_1036'), + ] + + operations = [ + migrations.AddField( + model_name='dochistory', + name='words', + field=models.IntegerField(null=True, blank=True), + ), + migrations.AddField( + model_name='document', + name='words', + field=models.IntegerField(null=True, blank=True), + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 6b850e98a..6ef127dc3 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -75,6 +75,7 @@ class DocumentInfo(models.Model): abstract = models.TextField(blank=True) rev = models.CharField(verbose_name="revision", max_length=16, blank=True) pages = models.IntegerField(blank=True, null=True) + words = models.IntegerField(blank=True, null=True) order = models.IntegerField(default=1, blank=True) # This is probably obviated by SessionPresentaion.order intended_std_level = models.ForeignKey(IntendedStdLevelName, verbose_name="Intended standardization level", blank=True, null=True) std_level = models.ForeignKey(StdLevelName, verbose_name="Standardization level", blank=True, null=True) diff --git a/ietf/settings.py b/ietf/settings.py index 321029f0f..38ab48e83 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -468,6 +468,8 @@ INTERNET_DRAFT_ARCHIVE_DIR = '/a/www/www6s/draft-archive' INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/a/www/www6s/archive/id' MEETING_RECORDINGS_DIR = '/a/www/audio' +DOCUMENT_FORMAT_BLACKLIST = ["tar", "dtd", "p7s"] + # Mailing list info URL for lists hosted on the IETF servers MAILING_LIST_INFO_URL = "https://www.ietf.org/mailman/listinfo/%(list_addr)s" MAILING_LIST_ARCHIVE_URL = "https://mailarchive.ietf.org" diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py new file mode 100644 index 000000000..f088de081 --- /dev/null +++ b/ietf/stats/backfill_data.py @@ -0,0 +1,58 @@ +import sys, os, argparse + +basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +sys.path = [ basedir ] + sys.path +os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" + +virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py") +if os.path.exists(virtualenv_activation): + execfile(virtualenv_activation, dict(__file__=virtualenv_activation)) + +import django +django.setup() + +from django.conf import settings + +from ietf.doc.models import Document +from ietf.utils.draft import Draft + +parser = argparse.ArgumentParser() +parser.add_argument("--document", help="specific document name") +parser.add_argument("--words", action="store_true", help="fill in word count") +args = parser.parse_args() + + +docs_qs = Document.objects.filter(type="draft") + +if args.document: + docs_qs = docs_qs.filter(docalias__name=args.document) + +for doc in docs_qs.prefetch_related("docalias_set"): + canonical_name = doc.name + for n in doc.docalias_set.all(): + if n.name.startswith("rfc"): + canonical_name = n.name + + if canonical_name.startswith("rfc"): + path = os.path.join(settings.RFC_PATH, canonical_name + ".txt") + else: + path = os.path.join(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR, canonical_name + "-" + doc.rev + ".txt") + + if not os.path.exists(path): + print "skipping", doc.name, "no txt file found at", path + continue + + with open(path, 'r') as f: + d = Draft(f.read(), path) + + updates = {} + + if args.words: + words = d.get_wordcount() + if words != doc.words: + updates["words"] = words + + if updates: + Document.objects.filter(pk=doc.pk).update(**updates) + print "updated", canonical_name + diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 33c76409f..54017208e 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -31,7 +31,7 @@ class StatisticsTests(TestCase): self.assertTrue(authors_all_url in r["Location"]) # check various stats types - for stats_type in ["authors", "pages"]: + for stats_type in ["authors", "pages", "words", "format"]: for document_type in ["all", "rfc", "draft"]: url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type, "document_type": document_type }) r = self.client.get(url) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 82fcd5743..6616d8fdd 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -5,6 +5,6 @@ import ietf.stats.views urlpatterns = patterns('', url("^$", ietf.stats.views.stats_index), - url("^document/(?:(?Pauthors|pages|format|spectech)/)?(?:(?Pall|rfc|draft)/)?$", ietf.stats.views.document_stats), + url("^document/(?:(?Pauthors|pages|words|format|formlang)/)?(?:(?Pall|rfc|draft)/)?$", ietf.stats.views.document_stats), url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, ietf.stats.views.review_stats), ) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 5e1211daa..6601d0266 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,4 +1,9 @@ -import datetime, itertools, json, calendar +import datetime +import itertools +import json +import calendar +import os +import re from collections import defaultdict from django.shortcuts import render @@ -7,6 +12,7 @@ from django.core.urlresolvers import reverse as urlreverse from django.http import HttpResponseRedirect, HttpResponseForbidden from django.db.models import Count from django.utils.safestring import mark_safe +from django.conf import settings import dateutil.relativedelta @@ -15,10 +21,11 @@ from ietf.review.utils import (extract_review_request_data, ReviewRequestData, compute_review_request_stats, sum_raw_review_request_aggregations) +from ietf.submit.models import Submission from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewRequestStateName, ReviewResultName -from ietf.doc.models import Document +from ietf.doc.models import DocAlias from ietf.ietfauth.utils import has_role def stats_index(request): @@ -48,7 +55,6 @@ def generate_query_string(query_dict, overrides): return query_part - def document_stats(request, stats_type=None, document_type=None): def build_document_stats_url(stats_type_override=Ellipsis, document_type_override=Ellipsis, get_overrides={}): kwargs = { @@ -60,10 +66,11 @@ def document_stats(request, stats_type=None, document_type=None): # statistics type - one of the tables or the chart possible_stats_types = [ - ("authors", "Number of authors"), + ("authors", "Authors"), ("pages", "Pages"), -# ("format", "Format"), -# ("spectech", "Specification techniques"), + ("words", "Words"), + ("format", "Format"), + ("formlang", "Formal languages"), ] possible_stats_types = [ (slug, label, build_document_stats_url(stats_type_override=slug)) @@ -85,13 +92,34 @@ def document_stats(request, stats_type=None, document_type=None): return HttpResponseRedirect(build_document_stats_url(document_type_override=possible_document_types[0][0])) + def put_into_bin(value, bin_size): + if value is None: + return (value, value) + + v = (value // bin_size) * bin_size + return (v, "{} - {}".format(v, v + bin_size - 1)) + + def generate_canonical_names(docalias_qs): + for doc_id, ts in itertools.groupby(docalias_qs.order_by("document"), lambda t: t[0]): + chosen = None + for t in ts: + if chosen is None: + chosen = t + else: + if t[0].startswith("rfc"): + chosen = t + elif t[0].startswith("draft") and not chosen[0].startswith("rfc"): + chosen = t + + yield chosen + # filter documents - doc_qs = Document.objects.filter(type="draft") + docalias_qs = DocAlias.objects.filter(document__type="draft") if document_type == "rfc": - doc_qs = doc_qs.filter(states__type="draft", states__slug="rfc") + docalias_qs = docalias_qs.filter(document__states__type="draft", document__states__slug="rfc") elif document_type == "draft": - doc_qs = doc_qs.exclude(states__type="draft", states__slug="rfc") + docalias_qs = docalias_qs.exclude(document__states__type="draft", document__states__slug="rfc") chart_data = [] table_data = [] @@ -104,19 +132,20 @@ def document_stats(request, stats_type=None, document_type=None): doc_label = "draft" stats_title = "" + bin_size = 1 if stats_type == "authors": stats_title = "Number of authors for each {}".format(doc_label) - groups = defaultdict(list) + bins = defaultdict(list) - for name, author_count in doc_qs.values_list("name").annotate(Count("authors")).iterator(): - groups[author_count].append(name) + for name, author_count in generate_canonical_names(docalias_qs.values_list("name").annotate(Count("document__authors"))): + bins[author_count].append(name) - total_docs = sum(len(names) for author_count, names in groups.iteritems()) + total_docs = sum(len(names) for author_count, names in bins.iteritems()) series_data = [] - for author_count, names in sorted(groups.iteritems(), key=lambda t: t[0]): + for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): percentage = len(names) * 100.0 / total_docs series_data.append((author_count, percentage)) table_data.append((author_count, percentage, names)) @@ -129,15 +158,15 @@ def document_stats(request, stats_type=None, document_type=None): elif stats_type == "pages": stats_title = "Number of pages for each {}".format(doc_label) - groups = defaultdict(list) + bins = defaultdict(list) - for name, pages in doc_qs.values_list("name", "pages"): - groups[pages].append(name) + for name, pages in generate_canonical_names(docalias_qs.values_list("name", "document__pages")): + bins[pages].append(name) - total_docs = sum(len(names) for pages, names in groups.iteritems()) + total_docs = sum(len(names) for pages, names in bins.iteritems()) series_data = [] - for pages, names in sorted(groups.iteritems(), key=lambda t: t[0]): + for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]): percentage = len(names) * 100.0 / total_docs if pages is not None: series_data.append((pages, len(names))) @@ -148,7 +177,86 @@ def document_stats(request, stats_type=None, document_type=None): "animation": False, }) + elif stats_type == "words": + stats_title = "Number of words for each {}".format(doc_label) + bin_size = 500 + + bins = defaultdict(list) + + for name, words in generate_canonical_names(docalias_qs.values_list("name", "document__words")): + bins[put_into_bin(words, bin_size)].append(name) + + total_docs = sum(len(names) for words, names in bins.iteritems()) + + series_data = [] + for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]): + percentage = len(names) * 100.0 / total_docs + if words is not None: + series_data.append((value, len(names))) + + table_data.append((words, percentage, names)) + + chart_data.append({ + "data": series_data, + "animation": False, + }) + + elif stats_type == "format": + stats_title = "Formats for each {}".format(doc_label) + + bins = defaultdict(list) + + # on new documents, we should have a Submission row with the file types + submission_types = {} + + for doc_name, file_types in Submission.objects.values_list("draft", "file_types").order_by("submission_date", "id"): + submission_types[doc_name] = file_types + + doc_names_with_missing_types = {} + for canonical_name, rev, doc_name in generate_canonical_names(docalias_qs.values_list("name", "document__rev", "document__name")): + types = submission_types.get(doc_name) + if types: + for dot_ext in types.split(","): + bins[dot_ext.lstrip(".").upper()].append(canonical_name) + + else: + + if canonical_name.startswith("rfc"): + filename = canonical_name + else: + filename = canonical_name + "-" + rev + + doc_names_with_missing_types[filename] = canonical_name + + # look up the remaining documents on disk + for filename in itertools.chain(os.listdir(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR), os.listdir(settings.RFC_PATH)): + t = filename.split(".", 1) + if len(t) != 2: + continue + + basename, ext = t + if any(ext.lower().endswith(blacklisted_ext.lower()) for blacklisted_ext in settings.DOCUMENT_FORMAT_BLACKLIST): + continue + + canonical_name = doc_names_with_missing_types.get(basename) + + if canonical_name: + bins[ext.upper()].append(canonical_name) + + total_docs = sum(len(names) for fmt, names in bins.iteritems()) + + series_data = [] + for fmt, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / total_docs + series_data.append((fmt, len(names))) + + table_data.append((fmt, percentage, names)) + + chart_data.append({ + "data": series_data, + "animation": False, + }) return render(request, "stats/document_stats.html", { "chart_data": mark_safe(json.dumps(chart_data)), @@ -159,6 +267,8 @@ def document_stats(request, stats_type=None, document_type=None): "possible_document_types": possible_document_types, "document_type": document_type, "doc_label": doc_label, + "bin_size": bin_size, + "content_template": "stats/document_stats_{}.html".format(stats_type), }) @login_required diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html index 576a62ed5..532329203 100644 --- a/ietf/templates/stats/document_stats.html +++ b/ietf/templates/stats/document_stats.html @@ -35,11 +35,7 @@ - {% if stats_type == "authors" %} - {% include "stats/document_stats_authors.html" %} - {% elif stats_type == "pages" %} - {% include "stats/document_stats_pages.html" %} - {% endif %} + {% include content_template %} {% endblock %} {% block js %} diff --git a/ietf/templates/stats/document_stats_format.html b/ietf/templates/stats/document_stats_format.html new file mode 100644 index 000000000..7e701343f --- /dev/null +++ b/ietf/templates/stats/document_stats_format.html @@ -0,0 +1,60 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for pages, percentage, names in table_data %} + + + + + + {% endfor %} + +
FormatPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/docnames_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_pages.html b/ietf/templates/stats/document_stats_pages.html index c9a725d0b..f4c930e46 100644 --- a/ietf/templates/stats/document_stats_pages.html +++ b/ietf/templates/stats/document_stats_pages.html @@ -41,9 +41,9 @@ - + - + diff --git a/ietf/templates/stats/document_stats_words.html b/ietf/templates/stats/document_stats_words.html new file mode 100644 index 000000000..d5983f1d6 --- /dev/null +++ b/ietf/templates/stats/document_stats_words.html @@ -0,0 +1,58 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ +
AuthorsPages Percentage of {{ doc_label }}s{{ doc_label }}s{{ doc_label|capfirst }}s
+ + + + + + + + + {% for pages, percentage, names in table_data %} + + + + + + {% endfor %} + +
WordsPercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ pages }}{{ percentage|floatformat:2 }}%{% include "stats/includes/docnames_cell.html" %}
From aebfe44f9e84ddf1cc568fcf2c9919f02357d909 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 16 Jan 2017 16:08:56 +0000 Subject: [PATCH 07/49] Add simple detection of formal languages used in draft, partially based on the code in getauthors by Jari Arkko - Legacy-Id: 12657 --- ietf/utils/draft.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index a6cde5a99..b7e9c4231 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -301,6 +301,28 @@ class Draft(): count += sum(1 for _ in word_re.finditer(l)) return count + # ------------------------------------------------------------------ + def get_formal_languages(self): + language_regexps = [ + ("abnf", [re.compile(r"\bABNF"), re.compile(r" +[a-zA-Z][a-zA-Z0-9_-]* +=[/ ]")]), + ("asn1", [re.compile(r'DEFINITIONS +::= +BEGIN')]), + ("cbor", [re.compile(r'\b(?:CBOR|CDDL)\b'), re.compile(r" +[a-zA-Z][a-zA-Z0-9_-]* += +[\{\[\(]")]), + ("ccode", [re.compile(r"(?:\+\+\))|(?:for \(i)|(?: [!=]= 0\) \{)|(?: struct [a-zA-Z_0-9]+ \{)")]), + ("json", [re.compile(r'\bJSON\b'), re.compile(r" \"[^\"]+\" ?: [a-zA-Z0-9\.\"\{\[]")]), + ("xml", [re.compile(r"<\?xml")]), + ] + already_matched = set() + for l in self.lines: + for lang_name, patterns in language_regexps: + for p in patterns: + if p not in already_matched and p.search(l): + already_matched.add(p) + return [ + lang_name + for lang_name, patterns in language_regexps + if all(p in already_matched for p in patterns) + ] + # ---------------------------------------------------------------------- def get_status(self): if self._status == None: From 641d92cf49e597e0c4475916294cd0c0f89e6b14 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 16 Jan 2017 17:06:54 +0000 Subject: [PATCH 08/49] Add many-to-many field with formal languages to Document and add formal language statistics - Legacy-Id: 12658 --- .../doc/migrations/0020_auto_20170112_0753.py | 11 ++++ ietf/doc/models.py | 3 +- ietf/name/admin.py | 3 +- .../migrations/0017_formallanguagename.py | 28 +++++++++ .../migrations/0018_add_formlang_names.py | 26 ++++++++ ietf/name/models.py | 2 + ietf/stats/backfill_data.py | 25 +++++++- ietf/stats/views.py | 33 +++++++--- .../stats/document_stats_formlang.html | 60 +++++++++++++++++++ 9 files changed, 179 insertions(+), 12 deletions(-) create mode 100644 ietf/name/migrations/0017_formallanguagename.py create mode 100644 ietf/name/migrations/0018_add_formlang_names.py create mode 100644 ietf/templates/stats/document_stats_formlang.html diff --git a/ietf/doc/migrations/0020_auto_20170112_0753.py b/ietf/doc/migrations/0020_auto_20170112_0753.py index da22265f7..9ca0cafa6 100644 --- a/ietf/doc/migrations/0020_auto_20170112_0753.py +++ b/ietf/doc/migrations/0020_auto_20170112_0753.py @@ -7,6 +7,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ + ('name', '0017_formallanguagename'), ('doc', '0019_auto_20161207_1036'), ] @@ -21,4 +22,14 @@ class Migration(migrations.Migration): name='words', field=models.IntegerField(null=True, blank=True), ), + migrations.AddField( + model_name='dochistory', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), + migrations.AddField( + model_name='document', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 6ef127dc3..e4c776004 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -15,7 +15,7 @@ import debug # pyflakes:ignore from ietf.group.models import Group from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName, - DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName ) + DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, FormalLanguageName ) from ietf.person.models import Email, Person from ietf.utils.admin import admin_link @@ -76,6 +76,7 @@ class DocumentInfo(models.Model): rev = models.CharField(verbose_name="revision", max_length=16, blank=True) pages = models.IntegerField(blank=True, null=True) words = models.IntegerField(blank=True, null=True) + formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document") order = models.IntegerField(default=1, blank=True) # This is probably obviated by SessionPresentaion.order intended_std_level = models.ForeignKey(IntendedStdLevelName, verbose_name="Intended standardization level", blank=True, null=True) std_level = models.ForeignKey(StdLevelName, verbose_name="Standardization level", blank=True, null=True) diff --git a/ietf/name/admin.py b/ietf/name/admin.py index 684e930ce..642d36aff 100644 --- a/ietf/name/admin.py +++ b/ietf/name/admin.py @@ -3,7 +3,7 @@ from django.contrib import admin from ietf.name.models import ( BallotPositionName, ConstraintName, DBTemplateTypeName, DocRelationshipName, DocReminderTypeName, DocTagName, DocTypeName, DraftSubmissionStateName, - FeedbackTypeName, GroupMilestoneStateName, GroupStateName, GroupTypeName, + FeedbackTypeName, FormalLanguageName, GroupMilestoneStateName, GroupStateName, GroupTypeName, IntendedStdLevelName, IprDisclosureStateName, IprEventTypeName, IprLicenseTypeName, LiaisonStatementEventTypeName, LiaisonStatementPurposeName, LiaisonStatementState, LiaisonStatementTagName, MeetingTypeName, NomineePositionStateName, @@ -32,6 +32,7 @@ admin.site.register(DBTemplateTypeName, NameAdmin) admin.site.register(DocReminderTypeName, NameAdmin) admin.site.register(DocTagName, NameAdmin) admin.site.register(DraftSubmissionStateName, NameAdmin) +admin.site.register(FormalLanguageName, NameAdmin) admin.site.register(FeedbackTypeName, NameAdmin) admin.site.register(GroupMilestoneStateName, NameAdmin) admin.site.register(GroupStateName, NameAdmin) diff --git a/ietf/name/migrations/0017_formallanguagename.py b/ietf/name/migrations/0017_formallanguagename.py new file mode 100644 index 000000000..94a7dfc4e --- /dev/null +++ b/ietf/name/migrations/0017_formallanguagename.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0016_auto_20161013_1010'), + ] + + operations = [ + migrations.CreateModel( + name='FormalLanguageName', + fields=[ + ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('desc', models.TextField(blank=True)), + ('used', models.BooleanField(default=True)), + ('order', models.IntegerField(default=0)), + ], + options={ + 'ordering': ['order', 'name'], + 'abstract': False, + }, + ), + ] diff --git a/ietf/name/migrations/0018_add_formlang_names.py b/ietf/name/migrations/0018_add_formlang_names.py new file mode 100644 index 000000000..542cb6cf5 --- /dev/null +++ b/ietf/name/migrations/0018_add_formlang_names.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def insert_initial_formal_language_names(apps, schema_editor): + FormalLanguageName = apps.get_model("name", "FormalLanguageName") + FormalLanguageName.objects.get_or_create(slug="abnf", name="ABNF", desc="Augmented Backus-Naur Form", order=1) + FormalLanguageName.objects.get_or_create(slug="asn1", name="ASN.1", desc="Abstract Syntax Notation One", order=2) + FormalLanguageName.objects.get_or_create(slug="cbor", name="CBOR", desc="Concise Binary Object Representation", order=3) + FormalLanguageName.objects.get_or_create(slug="ccode", name="C Code", desc="Code in the C Programming Language", order=4) + FormalLanguageName.objects.get_or_create(slug="json", name="JSON", desc="Javascript Object Notation", order=5) + FormalLanguageName.objects.get_or_create(slug="xml", name="XML", desc="Extensible Markup Language", order=6) + +def noop(apps, schema_editor): + pass + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0017_formallanguagename'), + ] + + operations = [ + migrations.RunPython(insert_initial_formal_language_names, noop) + ] diff --git a/ietf/name/models.py b/ietf/name/models.py index c1c69b4c6..d208a867b 100644 --- a/ietf/name/models.py +++ b/ietf/name/models.py @@ -46,6 +46,8 @@ class StdLevelName(NameModel): class IntendedStdLevelName(NameModel): """Proposed Standard, (Draft Standard), Internet Standard, Experimental, Informational, Best Current Practice, Historic, ...""" +class FormalLanguageName(NameModel): + """ABNF, ASN.1, C code, CBOR, JSON, XML, ...""" class DocReminderTypeName(NameModel): "Stream state" class BallotPositionName(NameModel): diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py index f088de081..bdcf00193 100644 --- a/ietf/stats/backfill_data.py +++ b/ietf/stats/backfill_data.py @@ -14,20 +14,24 @@ django.setup() from django.conf import settings from ietf.doc.models import Document +from ietf.name.models import FormalLanguageName from ietf.utils.draft import Draft parser = argparse.ArgumentParser() parser.add_argument("--document", help="specific document name") parser.add_argument("--words", action="store_true", help="fill in word count") +parser.add_argument("--formlang", action="store_true", help="fill in formal languages") args = parser.parse_args() +formal_language_dict = { l.pk: l for l in FormalLanguageName.objects.all() } + docs_qs = Document.objects.filter(type="draft") if args.document: docs_qs = docs_qs.filter(docalias__name=args.document) -for doc in docs_qs.prefetch_related("docalias_set"): +for doc in docs_qs.prefetch_related("docalias_set", "formal_languages"): canonical_name = doc.name for n in doc.docalias_set.all(): if n.name.startswith("rfc"): @@ -45,6 +49,8 @@ for doc in docs_qs.prefetch_related("docalias_set"): with open(path, 'r') as f: d = Draft(f.read(), path) + updated = False + updates = {} if args.words: @@ -52,7 +58,24 @@ for doc in docs_qs.prefetch_related("docalias_set"): if words != doc.words: updates["words"] = words + if args.formlang: + langs = d.get_formal_languages() + + new_formal_languages = set(formal_language_dict[l] for l in langs) + old_formal_languages = set(doc.formal_languages.all()) + + if new_formal_languages != old_formal_languages: + for l in new_formal_languages - old_formal_languages: + doc.formal_languages.add(l) + updated = True + for l in old_formal_languages - new_formal_languages: + doc.formal_languages.remove(l) + updated = True + if updates: Document.objects.filter(pk=doc.pk).update(**updates) + updated = True + + if updated: print "updated", canonical_name diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 6601d0266..a580bb277 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -134,6 +134,8 @@ def document_stats(request, stats_type=None, document_type=None): stats_title = "" bin_size = 1 + total_docs = docalias_qs.count() + if stats_type == "authors": stats_title = "Number of authors for each {}".format(doc_label) @@ -142,8 +144,6 @@ def document_stats(request, stats_type=None, document_type=None): for name, author_count in generate_canonical_names(docalias_qs.values_list("name").annotate(Count("document__authors"))): bins[author_count].append(name) - total_docs = sum(len(names) for author_count, names in bins.iteritems()) - series_data = [] for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): percentage = len(names) * 100.0 / total_docs @@ -163,8 +163,6 @@ def document_stats(request, stats_type=None, document_type=None): for name, pages in generate_canonical_names(docalias_qs.values_list("name", "document__pages")): bins[pages].append(name) - total_docs = sum(len(names) for pages, names in bins.iteritems()) - series_data = [] for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]): percentage = len(names) * 100.0 / total_docs @@ -187,8 +185,6 @@ def document_stats(request, stats_type=None, document_type=None): for name, words in generate_canonical_names(docalias_qs.values_list("name", "document__words")): bins[put_into_bin(words, bin_size)].append(name) - total_docs = sum(len(names) for words, names in bins.iteritems()) - series_data = [] for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]): percentage = len(names) * 100.0 / total_docs @@ -203,7 +199,7 @@ def document_stats(request, stats_type=None, document_type=None): }) elif stats_type == "format": - stats_title = "Formats for each {}".format(doc_label) + stats_title = "Submission formats for each {}".format(doc_label) bins = defaultdict(list) @@ -244,8 +240,6 @@ def document_stats(request, stats_type=None, document_type=None): if canonical_name: bins[ext.upper()].append(canonical_name) - total_docs = sum(len(names) for fmt, names in bins.iteritems()) - series_data = [] for fmt, names in sorted(bins.iteritems(), key=lambda t: t[0]): percentage = len(names) * 100.0 / total_docs @@ -258,6 +252,27 @@ def document_stats(request, stats_type=None, document_type=None): "animation": False, }) + elif stats_type == "formlang": + stats_title = "Formal languages used for each {}".format(doc_label) + + bins = defaultdict(list) + + for name, formal_language_name in generate_canonical_names(docalias_qs.values_list("name", "document__formal_languages__name")): + bins[formal_language_name].append(name) + + series_data = [] + for formal_language, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / total_docs + if formal_language is not None: + series_data.append((formal_language, len(names))) + table_data.append((formal_language, percentage, names)) + + chart_data.append({ + "data": series_data, + "animation": False, + }) + + return render(request, "stats/document_stats.html", { "chart_data": mark_safe(json.dumps(chart_data)), "table_data": table_data, diff --git a/ietf/templates/stats/document_stats_formlang.html b/ietf/templates/stats/document_stats_formlang.html new file mode 100644 index 000000000..248a45b82 --- /dev/null +++ b/ietf/templates/stats/document_stats_formlang.html @@ -0,0 +1,60 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for formal_language, percentage, names in table_data %} + + + + + + {% endfor %} + +
Formal languagePercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ formal_language }}{{ percentage|floatformat:2 }}%{% include "stats/includes/docnames_cell.html" %}
From b3d75af1af8e446dc2a550ef6befe46f2c604962 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 16 Jan 2017 17:34:46 +0000 Subject: [PATCH 09/49] Fix a spacing bug in the submission HTML - Legacy-Id: 12659 --- ietf/templates/submit/submission_status.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index 147947c9a..cc0e6b2c9 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -311,7 +311,7 @@ {% if user|has_role:"Secretariat" %}

- Send Email + Send Email {% endif %} {% if show_send_full_url %} From 1850a26b23fdeb41023c2a78923f0fb20882a0d7 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 16 Jan 2017 17:37:23 +0000 Subject: [PATCH 10/49] Add words and formal languages to Submission, set them in the submit view, clean up submit view a bit to avoid duplicated attribute setting code - Legacy-Id: 12660 --- .../migrations/0018_auto_20170116_0927.py | 25 ++++++ ietf/submit/models.py | 5 +- ietf/submit/tests.py | 1 + ietf/submit/views.py | 77 ++++++++----------- 4 files changed, 61 insertions(+), 47 deletions(-) create mode 100644 ietf/submit/migrations/0018_auto_20170116_0927.py diff --git a/ietf/submit/migrations/0018_auto_20170116_0927.py b/ietf/submit/migrations/0018_auto_20170116_0927.py new file mode 100644 index 000000000..8fdd0067b --- /dev/null +++ b/ietf/submit/migrations/0018_auto_20170116_0927.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0018_add_formlang_names'), + ('submit', '0017_auto_20161207_1046'), + ] + + operations = [ + migrations.AddField( + model_name='submission', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), + migrations.AddField( + model_name='submission', + name='words', + field=models.IntegerField(null=True, blank=True), + ), + ] diff --git a/ietf/submit/models.py b/ietf/submit/models.py index 6e710cc6b..1e9e29e59 100644 --- a/ietf/submit/models.py +++ b/ietf/submit/models.py @@ -10,7 +10,7 @@ from ietf.doc.models import Document from ietf.person.models import Person from ietf.group.models import Group from ietf.message.models import Message -from ietf.name.models import DraftSubmissionStateName +from ietf.name.models import DraftSubmissionStateName, FormalLanguageName from ietf.utils.accesstoken import generate_random_key, generate_access_token @@ -36,6 +36,9 @@ class Submission(models.Model): abstract = models.TextField(blank=True) rev = models.CharField(max_length=3, blank=True) pages = models.IntegerField(null=True, blank=True) + words = models.IntegerField(null=True, blank=True) + formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document") + authors = models.TextField(blank=True, help_text="List of author names and emails, one author per line, e.g. \"John Doe <john@example.org>\".") note = models.TextField(blank=True) replaces = models.CharField(max_length=1000, blank=True) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index a767e7c34..ad63148ce 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -190,6 +190,7 @@ class SubmitTests(TestCase): abstract="Blahblahblah.", rev="01", pages=2, + words=100, intended_std_level_id="ps", ad=draft.ad, expires=datetime.datetime.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 51ed2c9a3..659e72c63 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -20,6 +20,7 @@ from ietf.group.models import Group from ietf.ietfauth.utils import has_role, role_required from ietf.mailtrigger.utils import gather_address_lists from ietf.message.models import Message, MessageAttachment +from ietf.name.models import FormalLanguageName from ietf.submit.forms import ( SubmissionUploadForm, NameEmailForm, EditSubmissionForm, PreapprovalForm, ReplacesForm, SubmissionEmailForm, MessageModelForm ) from ietf.submit.mail import ( send_full_url, send_approval_request_to_group, @@ -124,55 +125,39 @@ def upload_submission(request): # for this revision. # If so - we're going to update it otherwise we create a new object - submission = Submission.objects.filter(name=form.filename, - rev=form.revision, - state_id = "waiting-for-draft").distinct() - if (len(submission) == 0): - submission = None - elif (len(submission) == 1): - submission = submission[0] - - submission.state = DraftSubmissionStateName.objects.get(slug="uploaded") - submission.remote_ip=form.remote_ip - submission.title=form.title - submission.abstract=abstract - submission.rev=form.revision - submission.pages=form.parsed_draft.get_pagecount() - submission.authors="\n".join(authors) - submission.first_two_pages=''.join(form.parsed_draft.pages[:2]) - submission.file_size=file_size - submission.file_types=','.join(form.file_types) - submission.submission_date=datetime.date.today() - submission.document_date=form.parsed_draft.get_creation_date() - submission.replaces="" - - submission.save() + submissions = Submission.objects.filter(name=form.filename, + rev=form.revision, + state_id = "waiting-for-draft").distinct() + + if not submissions: + submission = Submission(name=form.filename, rev=form.revision, group=form.group) + elif len(submissions) == 1: + submission = submissions[0] else: raise Exception("Multiple submissions found waiting for upload") - if (submission == None): - try: - submission = Submission.objects.create( - state=DraftSubmissionStateName.objects.get(slug="uploaded"), - remote_ip=form.remote_ip, - name=form.filename, - group=form.group, - title=form.title, - abstract=abstract, - rev=form.revision, - pages=form.parsed_draft.get_pagecount(), - authors="\n".join(authors), - note="", - first_two_pages=''.join(form.parsed_draft.pages[:2]), - file_size=file_size, - file_types=','.join(form.file_types), - submission_date=datetime.date.today(), - document_date=form.parsed_draft.get_creation_date(), - replaces="", - ) - except Exception as e: - log("Exception: %s\n" % e) - raise + try: + submission.state = DraftSubmissionStateName.objects.get(slug="uploaded") + submission.remote_ip = form.remote_ip + submission.title = form.title + submission.abstract = abstract + submission.pages = form.parsed_draft.get_pagecount() + submission.words = form.parsed_draft.get_wordcount() + submission.authors = "\n".join(authors) + submission.first_two_pages = ''.join(form.parsed_draft.pages[:2]) + submission.file_size = file_size + submission.file_types = ','.join(form.file_types) + submission.submission_date = datetime.date.today() + submission.document_date = form.parsed_draft.get_creation_date() + submission.replaces = "" + + submission.save() + + submission.formal_languages = FormalLanguageName.objects.filter(slug__in=form.parsed_draft.get_formal_languages()) + + except Exception as e: + log("Exception: %s\n" % e) + raise # run submission checkers def apply_check(submission, checker, method, fn): From f4555c4269f4b49c93c3e995e55ee3ae47e6b386 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Tue, 17 Jan 2017 13:50:41 +0000 Subject: [PATCH 11/49] Fix another spacing bug on the submit page - Legacy-Id: 12661 --- ietf/templates/submit/submission_status.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index cc0e6b2c9..a961f4395 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -48,9 +48,7 @@ {% if check.errors %}

The {{check.checker}} returned {{ check.errors }} error{{ check.errors|pluralize }} - and {{ check.warnings }} warning - - {{ check.warnings|pluralize }} ; click the button + and {{ check.warnings }} warning{{ check.warnings|pluralize }}; click the button below to see details. Please fix those, and resubmit.

{% elif check.warnings %} From 76bcab6efc7ab65b35c2e58649ef9335d090f31a Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Tue, 17 Jan 2017 13:52:16 +0000 Subject: [PATCH 12/49] Show formal languages used on new submissions and make the field editable, also add it to the test with a simple JSON example - Legacy-Id: 12662 --- ietf/submit/forms.py | 4 +++- ietf/submit/test_submission.txt | 18 ++++++++++++++---- ietf/submit/test_submission.xml | 9 +++++++++ ietf/submit/tests.py | 2 ++ ietf/submit/utils.py | 2 ++ ietf/submit/views.py | 12 ++++++++++-- ietf/templates/submit/submission_status.html | 8 ++++++++ 7 files changed, 48 insertions(+), 7 deletions(-) diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index 6e09476ba..f01cb44ab 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -21,6 +21,7 @@ from ietf.doc.fields import SearchableDocAliasesField from ietf.ipr.mail import utc_from_string from ietf.meeting.models import Meeting from ietf.message.models import Message +from ietf.name.models import FormalLanguageName from ietf.submit.models import Submission, Preapproval from ietf.submit.utils import validate_submission_rev, validate_submission_document_date from ietf.submit.parsers.pdf_parser import PDFParser @@ -376,13 +377,14 @@ class EditSubmissionForm(forms.ModelForm): rev = forms.CharField(label=u'Revision', max_length=2, required=True) document_date = forms.DateField(required=True) pages = forms.IntegerField(required=True) + formal_languages = forms.ModelMultipleChoiceField(queryset=FormalLanguageName.objects.filter(used=True), widget=forms.CheckboxSelectMultiple, required=False) abstract = forms.CharField(widget=forms.Textarea, required=True) note = forms.CharField(label=mark_safe(u'Comment to the Secretariat'), widget=forms.Textarea, required=False) class Meta: model = Submission - fields = ['title', 'rev', 'document_date', 'pages', 'abstract', 'note'] + fields = ['title', 'rev', 'document_date', 'pages', 'formal_languages', 'abstract', 'note'] def clean_rev(self): rev = self.cleaned_data["rev"] diff --git a/ietf/submit/test_submission.txt b/ietf/submit/test_submission.txt index 97c95b3c4..b6551bf2d 100644 --- a/ietf/submit/test_submission.txt +++ b/ietf/submit/test_submission.txt @@ -61,8 +61,10 @@ Internet-Draft Testing Tests %(month)s %(year)s Table of Contents 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 - 2. Security Considerations . . . . . . . . . . . . . . . . . . . 2 - 3. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 2 + 2. Yang . . . . . . . . . . . . . . . . . . . . . . . . . . . . 2 + 3. JSON example . . . . . . . . . . . . . . . . . . . . . . . . 2 + 4. Security Considerations . . . . . . . . . . . . . . . . . . . 2 + 5. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 2 Author's Address . . . . . . . . . . . . . . . . . . . . . . . . 2 1. Introduction @@ -169,11 +171,19 @@ Table of Contents -3. Security Considerations +3. JSON example + + The JSON object should look like this: + + { + "test": 1234 + } + +4. Security Considerations There are none. -4. IANA Considerations +5. IANA Considerations No new registrations for IANA. diff --git a/ietf/submit/test_submission.xml b/ietf/submit/test_submission.xml index a313e3706..a8b6d9cde 100644 --- a/ietf/submit/test_submission.xml +++ b/ietf/submit/test_submission.xml @@ -137,6 +137,15 @@ module ietf-mpls { +
+ + The JSON object should look like this: + + { + "test": 1234 + } + +
There are none. diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index ad63148ce..6fb3859f9 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -18,6 +18,7 @@ from ietf.group.models import Group from ietf.group.utils import setup_default_community_list_for_group from ietf.meeting.models import Meeting from ietf.message.models import Message +from ietf.name.models import FormalLanguageName from ietf.person.models import Person, Email from ietf.person.factories import UserFactory, PersonFactory from ietf.submit.models import Submission, Preapproval @@ -251,6 +252,7 @@ class SubmitTests(TestCase): self.assertEqual(draft.authors.count(), 1) self.assertEqual(draft.authors.all()[0].get_name(), "Author Name") self.assertEqual(draft.authors.all()[0].address, "author@example.com") + self.assertEqual(set(draft.formal_languages.all()), set(FormalLanguageName.objects.filter(slug="json"))) self.assertEqual(draft.relations_that_doc("replaces").count(), 1) self.assertTrue(draft.relations_that_doc("replaces").first().target, replaced_alias) self.assertEqual(draft.relations_that_doc("possibly-replaces").count(), 1) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index b3338fb1b..957b5eff2 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -253,6 +253,8 @@ def post_submission(request, submission, approvedDesc): update_authors(draft, submission) + draft.formal_languages = submission.formal_languages.all() + trouble = rebuild_reference_relations(draft, filename=os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (submission.name, submission.rev))) if trouble: log('Rebuild_reference_relations trouble: %s'%trouble) diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 659e72c63..95b9c2ae8 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -449,6 +449,8 @@ def edit_submission(request, submission_id, access_token=None): # trigger validation of all forms validations = [edit_form.is_valid(), submitter_form.is_valid(), replaces_form.is_valid()] + [ f.is_valid() for f in author_forms ] if all(validations): + changed_fields = [] + submission.submitter = submitter_form.cleaned_line() replaces = replaces_form.cleaned_data.get("replaces", []) submission.replaces = ",".join(o.name for o in replaces) @@ -463,12 +465,18 @@ def edit_submission(request, submission_id, access_token=None): submission.state = DraftSubmissionStateName.objects.get(slug="manual") submission.save() + formal_languages_changed = False + if set(submission.formal_languages.all()) != set(edit_form.cleaned_data["formal_languages"]): + submission.formal_languages = edit_form.cleaned_data["formal_languages"] + formal_languages_changed = True + send_manual_post_request(request, submission, errors) - changed_fields = [ + changed_fields += [ submission._meta.get_field(f).verbose_name for f in list(edit_form.fields.keys()) + ["submitter", "authors"] - if getattr(submission, f) != getattr(prev_submission, f) + if (f == "formal_languages" and formal_languages_changed) + or getattr(submission, f) != getattr(prev_submission, f) ] if changed_fields: diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index a961f4395..d4518b571 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -229,6 +229,14 @@ File size {{ submission.file_size|filesizeformat }} + + + Formal languages used + + {% for l in submission.formal_languages.all %}{{ l.name }}{% if not forloop.last %}, {% endif %}{% empty %}None{% endfor %} + {% if errors.formal_languages %}

{{ errors.formal_languages }}

{% endif %} + + {% if can_edit %} From 0479600ed00ada78a4e35c5b731c04841895c261 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Tue, 17 Jan 2017 13:53:13 +0000 Subject: [PATCH 13/49] Simple wording fix - Legacy-Id: 12663 --- ietf/templates/submit/submission_status.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index d4518b571..4f67c29de 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -233,7 +233,7 @@ Formal languages used - {% for l in submission.formal_languages.all %}{{ l.name }}{% if not forloop.last %}, {% endif %}{% empty %}None{% endfor %} + {% for l in submission.formal_languages.all %}{{ l.name }}{% if not forloop.last %}, {% endif %}{% empty %}None recognized{% endfor %} {% if errors.formal_languages %}

{{ errors.formal_languages }}

{% endif %} From e381dac958c58e49bba8be8240b2113ef86b2812 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Wed, 25 Jan 2017 11:29:45 +0000 Subject: [PATCH 14/49] Fix matching pattern in test to be more generic - Legacy-Id: 12729 --- ietf/meeting/tests_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 8268c6c77..46874875d 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -282,7 +282,7 @@ class MeetingTests(TestCase): r = self.client.get(urlreverse("ietf.meeting.views.materials", kwargs=dict(num=meeting.number))) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - row = q('#content td div:contains("%s")' % str(session.group.acronym)).closest("tr") + row = q('#content #%s' % str(session.group.acronym)).closest("tr") self.assertTrue(row.find('a:contains("Agenda")')) self.assertTrue(row.find('a:contains("Minutes")')) self.assertTrue(row.find('a:contains("Slideshow")')) From 9308948195d526abd09df525307711bf531360f4 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Thu, 26 Jan 2017 17:10:08 +0000 Subject: [PATCH 15/49] Add person, affiliation and country (through django-countries) to DocumentAuthor, rename author field to email and make it optional (for modeling old email-less submissions), remove the authors many to many referencing field from Document as it is not really pointing the right place. Update the Secretariat tools to show affiliation and country. Add migration for getting rid of the fake email addresses that the migration script created some years ago (just set the author email field to null). - Legacy-Id: 12739 --- ietf/api/__init__.py | 4 +- ...bmission-confirmation-email-in-postfix-log | 2 +- ietf/bin/generate-draft-aliases | 2 +- .../community/migrations/0004_cleanup_data.py | 2 +- ietf/community/tests.py | 4 +- ietf/community/utils.py | 4 +- ietf/doc/admin.py | 10 +-- ietf/doc/factories.py | 4 +- .../doc/migrations/0020_auto_20170112_0753.py | 81 +++++++++++++++++++ .../0021_remove_fake_email_adresses.py | 50 ++++++++++++ ietf/doc/models.py | 45 +++++++---- ietf/doc/resources.py | 24 +++--- ietf/doc/tests.py | 4 +- ietf/doc/tests_draft.py | 21 +++-- ietf/doc/tests_review.py | 5 +- ietf/doc/views_doc.py | 13 ++- ietf/doc/views_search.py | 2 +- ietf/doc/views_stats.py | 2 +- ietf/idindex/index.py | 12 +-- ietf/idindex/tests.py | 2 +- ietf/ipr/views.py | 6 +- ietf/mailtrigger/models.py | 2 +- .../migrations/0018_add_formlang_names.py | 5 +- ietf/name/resources.py | 20 ++++- ietf/person/models.py | 14 ++-- ietf/review/utils.py | 2 +- ietf/secr/drafts/email.py | 14 ++-- ietf/secr/drafts/forms.py | 4 + ietf/secr/drafts/views.py | 12 ++- ietf/secr/templates/drafts/authors.html | 12 ++- ietf/secr/templates/drafts/makerfc.html | 2 +- ietf/secr/templates/drafts/view.html | 2 +- ietf/settings.py | 1 + ietf/stats/views.py | 3 +- ietf/submit/models.py | 3 +- ietf/submit/tests.py | 26 +++--- ietf/submit/utils.py | 22 ++--- ietf/submit/views.py | 2 +- ietf/templates/doc/bibxml.xml | 8 +- ietf/templates/doc/document_bibtex.bib | 2 +- ietf/templates/doc/document_draft.html | 10 +-- ietf/utils/test_data.py | 5 +- requirements.txt | 1 + 43 files changed, 329 insertions(+), 142 deletions(-) create mode 100644 ietf/doc/migrations/0021_remove_fake_email_adresses.py diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 9a6cdf868..0d50d103d 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -212,9 +212,9 @@ class ToOneField(tastypie.fields.ToOneField): if not foreign_obj: if not self.null: if callable(self.attribute): - raise ApiFieldError("The related resource for resource %s could not be found." % (previous_obj)) + raise ApiFieldError(u"The related resource for resource %s could not be found." % (previous_obj)) else: - raise ApiFieldError("The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr)) + raise ApiFieldError(u"The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr)) return None fk_resource = self.get_related_resource(foreign_obj) diff --git a/ietf/bin/find-submission-confirmation-email-in-postfix-log b/ietf/bin/find-submission-confirmation-email-in-postfix-log index 3df3c78be..69bdcc331 100755 --- a/ietf/bin/find-submission-confirmation-email-in-postfix-log +++ b/ietf/bin/find-submission-confirmation-email-in-postfix-log @@ -66,7 +66,7 @@ if "<" in from_email: submission = Submission.objects.filter(name=draft).latest('submission_date') document = Document.objects.get(name=draft) -emails = [ author.address for author in document.authors.all() ] +emails = [ author.email.address for author in document.documentauthor_set.all() if author.email ] timestrings = [] for file in [ Path(settings.INTERNET_DRAFT_PATH) / ("%s-%s.txt"%(draft, submission.rev)), diff --git a/ietf/bin/generate-draft-aliases b/ietf/bin/generate-draft-aliases index fd3d409af..64b97a846 100755 --- a/ietf/bin/generate-draft-aliases +++ b/ietf/bin/generate-draft-aliases @@ -65,7 +65,7 @@ def get_draft_authors_emails(draft): " Get list of authors for the given draft." # This feels 'correct'; however, it creates fairly large delta - return [email.email_address() for email in draft.authors.all()] + return [author.email.email_address() for author in draft.documentauthor_set.all() if author.email.email_address()] # This gives fairly small delta compared to current state, # however, it seems to be wrong (doesn't check for emails being diff --git a/ietf/community/migrations/0004_cleanup_data.py b/ietf/community/migrations/0004_cleanup_data.py index 6ee14c951..ab1c24f4e 100644 --- a/ietf/community/migrations/0004_cleanup_data.py +++ b/ietf/community/migrations/0004_cleanup_data.py @@ -57,7 +57,7 @@ def port_rules_to_typed_system(apps, schema_editor): elif rule.rule_type in ["author", "author_rfc"]: - found_persons = list(try_to_uniquify_person(rule, Person.objects.filter(email__documentauthor__id__gte=1).filter(name__icontains=rule.value).distinct())) + found_persons = list(try_to_uniquify_person(rule, Person.objects.filter(documentauthor__id__gte=1).filter(name__icontains=rule.value).distinct())) if found_persons: rule.person = found_persons[0] diff --git a/ietf/community/tests.py b/ietf/community/tests.py index b712ee821..01d1f1ece 100644 --- a/ietf/community/tests.py +++ b/ietf/community/tests.py @@ -31,7 +31,7 @@ class CommunityListTests(TestCase): rule_state_iesg = SearchRule.objects.create(rule_type="state_iesg", state=State.objects.get(type="draft-iesg", slug="lc"), community_list=clist) - rule_author = SearchRule.objects.create(rule_type="author", state=State.objects.get(type="draft", slug="active"), person=Person.objects.filter(email__documentauthor__document=draft).first(), community_list=clist) + rule_author = SearchRule.objects.create(rule_type="author", state=State.objects.get(type="draft", slug="active"), person=Person.objects.filter(documentauthor__document=draft).first(), community_list=clist) rule_ad = SearchRule.objects.create(rule_type="ad", state=State.objects.get(type="draft", slug="active"), person=draft.ad, community_list=clist) @@ -113,7 +113,7 @@ class CommunityListTests(TestCase): r = self.client.post(url, { "action": "add_rule", "rule_type": "author_rfc", - "author_rfc-person": Person.objects.filter(email__documentauthor__document=draft).first().pk, + "author_rfc-person": Person.objects.filter(documentauthor__document=draft).first().pk, "author_rfc-state": State.objects.get(type="draft", slug="rfc").pk, }) self.assertEqual(r.status_code, 302) diff --git a/ietf/community/utils.py b/ietf/community/utils.py index ee8c87042..37fce87ac 100644 --- a/ietf/community/utils.py +++ b/ietf/community/utils.py @@ -88,7 +88,7 @@ def docs_matching_community_list_rule(rule): elif rule.rule_type.startswith("state_"): return docs.filter(states=rule.state) elif rule.rule_type in ["author", "author_rfc"]: - return docs.filter(states=rule.state, documentauthor__author__person=rule.person) + return docs.filter(states=rule.state, documentauthor__person=rule.person) elif rule.rule_type == "ad": return docs.filter(states=rule.state, ad=rule.person) elif rule.rule_type == "shepherd": @@ -121,7 +121,7 @@ def community_list_rules_matching_doc(doc): rules |= SearchRule.objects.filter( rule_type__in=["author", "author_rfc"], state__in=states, - person__in=list(Person.objects.filter(email__documentauthor__document=doc)), + person__in=list(Person.objects.filter(documentauthor__document=doc)), ) if doc.ad_id: diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index a9e81a749..c5db20e37 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -25,7 +25,7 @@ class DocAliasInline(admin.TabularInline): class DocAuthorInline(admin.TabularInline): model = DocumentAuthor - raw_id_fields = ['author', ] + raw_id_fields = ['person', 'email'] extra = 1 class RelatedDocumentInline(admin.TabularInline): @@ -99,7 +99,7 @@ class DocumentAdmin(admin.ModelAdmin): list_display = ['name', 'rev', 'group', 'pages', 'intended_std_level', 'author_list', 'time'] search_fields = ['name'] list_filter = ['type'] - raw_id_fields = ['authors', 'group', 'shepherd', 'ad'] + raw_id_fields = ['group', 'shepherd', 'ad'] inlines = [DocAliasInline, DocAuthorInline, RelatedDocumentInline, ] form = DocumentForm @@ -121,7 +121,7 @@ class DocHistoryAdmin(admin.ModelAdmin): list_display = ['doc', 'rev', 'state', 'group', 'pages', 'intended_std_level', 'author_list', 'time'] search_fields = ['doc__name'] ordering = ['time', 'doc', 'rev'] - raw_id_fields = ['doc', 'authors', 'group', 'shepherd', 'ad'] + raw_id_fields = ['doc', 'group', 'shepherd', 'ad'] def state(self, instance): return instance.get_state() @@ -174,7 +174,7 @@ class BallotPositionDocEventAdmin(DocEventAdmin): admin.site.register(BallotPositionDocEvent, BallotPositionDocEventAdmin) class DocumentAuthorAdmin(admin.ModelAdmin): - list_display = ['id', 'document', 'author', 'order'] - search_fields = [ 'document__name', 'author__address', ] + list_display = ['id', 'document', 'person', 'email', 'order'] + search_fields = [ 'document__name', 'person__name', 'email__address', ] admin.site.register(DocumentAuthor, DocumentAuthorAdmin) diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index 9e29c1a19..919c9a520 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -46,8 +46,8 @@ class DocumentFactory(factory.DjangoModelFactory): def authors(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument if create and extracted: order = 0 - for email in extracted: - DocumentAuthor.objects.create(document=obj, author=email, order=order) + for person in extracted: + DocumentAuthor.objects.create(document=obj, person=person, order=order) order += 1 @classmethod diff --git a/ietf/doc/migrations/0020_auto_20170112_0753.py b/ietf/doc/migrations/0020_auto_20170112_0753.py index 9ca0cafa6..9404b8aab 100644 --- a/ietf/doc/migrations/0020_auto_20170112_0753.py +++ b/ietf/doc/migrations/0020_auto_20170112_0753.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from django.db import migrations, models +import django_countries.fields class Migration(migrations.Migration): @@ -32,4 +33,84 @@ class Migration(migrations.Migration): name='formal_languages', field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), ), + migrations.RemoveField( + model_name='dochistory', + name='authors', + ), + migrations.RemoveField( + model_name='document', + name='authors', + ), + migrations.AddField( + model_name='dochistoryauthor', + name='affiliation', + field=models.CharField(help_text=b'Organization/company used by author for submission', max_length=100, blank=True), + ), + migrations.AddField( + model_name='dochistoryauthor', + name='country', + field=django_countries.fields.CountryField(blank=True, help_text=b'Country used by author for submission', max_length=2), + ), + migrations.RenameField( + model_name='dochistoryauthor', + old_name='author', + new_name='email', + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='email', + field=models.ForeignKey(blank=True, to='person.Email', help_text=b'Email address used by author for submission', null=True), + ), + migrations.AddField( + model_name='dochistoryauthor', + name='person', + field=models.ForeignKey(blank=True, to='person.Person', null=True), + ), + migrations.AddField( + model_name='documentauthor', + name='affiliation', + field=models.CharField(help_text=b'Organization/company used by author for submission', max_length=100, blank=True), + ), + migrations.AddField( + model_name='documentauthor', + name='country', + field=django_countries.fields.CountryField(blank=True, help_text=b'Country used by author for submission', max_length=2), + ), + migrations.RenameField( + model_name='documentauthor', + old_name='author', + new_name='email', + ), + migrations.AlterField( + model_name='documentauthor', + name='email', + field=models.ForeignKey(blank=True, to='person.Email', help_text=b'Email address used by author for submission', null=True), + ), + migrations.AddField( + model_name='documentauthor', + name='person', + field=models.ForeignKey(blank=True, to='person.Person', null=True), + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='document', + field=models.ForeignKey(related_name='documentauthor_set', to='doc.DocHistory'), + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='order', + field=models.IntegerField(default=1), + ), + migrations.RunSQL("update doc_documentauthor a inner join person_email e on a.email_id = e.address set a.person_id = e.person_id;", migrations.RunSQL.noop), + migrations.RunSQL("update doc_dochistoryauthor a inner join person_email e on a.email_id = e.address set a.person_id = e.person_id;", migrations.RunSQL.noop), + migrations.AlterField( + model_name='documentauthor', + name='person', + field=models.ForeignKey(to='person.Person'), + ), + migrations.AlterField( + model_name='dochistoryauthor', + name='person', + field=models.ForeignKey(to='person.Person'), + ), ] diff --git a/ietf/doc/migrations/0021_remove_fake_email_adresses.py b/ietf/doc/migrations/0021_remove_fake_email_adresses.py new file mode 100644 index 000000000..d317ed03d --- /dev/null +++ b/ietf/doc/migrations/0021_remove_fake_email_adresses.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def fix_invalid_emails(apps, schema_editor): + Email = apps.get_model("person", "Email") + Role = apps.get_model("group", "Role") + RoleHistory = apps.get_model("group", "RoleHistory") + + e = Email.objects.filter(address="unknown-email-Gigi-Karmous-Edwards").first() + if e: + # according to ftp://ietf.org/ietf/97dec/adsl-minutes-97dec.txt + new_e, _ = Email.objects.get_or_create( + address="GiGi.Karmous-Edwards@pulse.com", + primary=e.primary, + active=e.active, + person=e.person, + ) + Role.objects.filter(email=e).update(email=new_e) + RoleHistory.objects.filter(email=e).update(email=new_e) + e.delete() + + e = Email.objects.filter(address="unknown-email-Pat-Thaler").first() + if e: + # current chair email + new_e = Email.objects.get(address="pat.thaler@broadcom.com") + Role.objects.filter(email=e).update(email=new_e) + RoleHistory.objects.filter(email=e).update(email=new_e) + e.delete() + + Email = apps.get_model("person", "Email") + DocumentAuthor = apps.get_model("doc", "DocumentAuthor") + DocHistoryAuthor = apps.get_model("doc", "DocHistoryAuthor") + + DocumentAuthor.objects.filter(email__address__startswith="unknown-email-").exclude(email__address__contains="@").update(email=None) + DocHistoryAuthor.objects.filter(email__address__startswith="unknown-email-").exclude(email__address__contains="@").update(email=None) + Email.objects.exclude(address__contains="@").filter(address__startswith="unknown-email-").delete() + +class Migration(migrations.Migration): + + dependencies = [ + ('doc', '0020_auto_20170112_0753'), + ('person', '0014_auto_20160613_0751'), + ('group', '0009_auto_20150930_0758'), + ] + + operations = [ + migrations.RunPython(fix_invalid_emails, migrations.RunPython.noop), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index e4c776004..3592b4578 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -11,6 +11,8 @@ from django.contrib.contenttypes.models import ContentType from django.conf import settings from django.utils.html import mark_safe +from django_countries.fields import CountryField + import debug # pyflakes:ignore from ietf.group.models import Group @@ -254,7 +256,7 @@ class DocumentInfo(models.Model): return state.name def author_list(self): - return ", ".join(email.address for email in self.authors.all()) + return u", ".join(author.email_id for author in self.documentauthor_set.all() if author.email_id) # This, and several other ballot related functions here, assume that there is only one active ballot for a document at any point in time. # If that assumption is violated, they will only expose the most recently created ballot @@ -399,20 +401,32 @@ class RelatedDocument(models.Model): return None -class DocumentAuthor(models.Model): - document = models.ForeignKey('Document') - author = models.ForeignKey(Email, help_text="Email address used by author for submission") +class DocumentAuthorInfo(models.Model): + person = models.ForeignKey(Person) + # email should only be null for some historic documents + email = models.ForeignKey(Email, help_text="Email address used by author for submission", blank=True, null=True) + affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission") + country = CountryField(blank=True, help_text="Country used by author for submission") order = models.IntegerField(default=1) - def __unicode__(self): - return u"%s %s (%s)" % (self.document.name, self.author.get_name(), self.order) + def formatted_email(self): + if self.email: + return u'"%s" <%s>' % (self.person.plain_ascii(), self.email.address) + else: + return "" class Meta: + abstract = True ordering = ["document", "order"] - + +class DocumentAuthor(DocumentAuthorInfo): + document = models.ForeignKey('Document') + + def __unicode__(self): + return u"%s %s (%s)" % (self.document.name, self.person, self.order) + class Document(DocumentInfo): name = models.CharField(max_length=255, primary_key=True) # immutable - authors = models.ManyToManyField(Email, through=DocumentAuthor, blank=True) def __unicode__(self): return self.name @@ -609,16 +623,13 @@ class RelatedDocHistory(models.Model): def __unicode__(self): return u"%s %s %s" % (self.source.doc.name, self.relationship.name.lower(), self.target.name) -class DocHistoryAuthor(models.Model): - document = models.ForeignKey('DocHistory') - author = models.ForeignKey(Email) - order = models.IntegerField() +class DocHistoryAuthor(DocumentAuthorInfo): + # use same naming convention as non-history version to make it a bit + # easier to write generic code + document = models.ForeignKey('DocHistory', related_name="documentauthor_set") def __unicode__(self): - return u"%s %s (%s)" % (self.document.doc.name, self.author.get_name(), self.order) - - class Meta: - ordering = ["document", "order"] + return u"%s %s (%s)" % (self.document.doc.name, self.person, self.order) class DocHistory(DocumentInfo): doc = models.ForeignKey(Document, related_name="history_set") @@ -627,7 +638,7 @@ class DocHistory(DocumentInfo): # canonical_name and replace the function on Document with a # property name = models.CharField(max_length=255) - authors = models.ManyToManyField(Email, through=DocHistoryAuthor, blank=True) + def __unicode__(self): return unicode(self.doc.name) diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 945700daa..dd97516e0 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -99,7 +99,6 @@ class DocumentResource(ModelResource): shepherd = ToOneField(EmailResource, 'shepherd', null=True) states = ToManyField(StateResource, 'states', null=True) tags = ToManyField(DocTagNameResource, 'tags', null=True) - authors = ToManyField(EmailResource, 'authors', null=True) rfc = CharField(attribute='rfc_number', null=True) class Meta: cache = SimpleCache() @@ -128,14 +127,14 @@ class DocumentResource(ModelResource): "shepherd": ALL_WITH_RELATIONS, "states": ALL_WITH_RELATIONS, "tags": ALL_WITH_RELATIONS, - "authors": ALL_WITH_RELATIONS, } api.doc.register(DocumentResource()) -from ietf.person.resources import EmailResource +from ietf.person.resources import PersonResource, EmailResource class DocumentAuthorResource(ModelResource): + person = ToOneField(PersonResource, 'person') + email = ToOneField(EmailResource, 'email', null=True) document = ToOneField(DocumentResource, 'document') - author = ToOneField(EmailResource, 'author') class Meta: cache = SimpleCache() queryset = DocumentAuthor.objects.all() @@ -143,9 +142,12 @@ class DocumentAuthorResource(ModelResource): #resource_name = 'documentauthor' filtering = { "id": ALL, + "affiliation": ALL, + "country": ALL, "order": ALL, + "person": ALL_WITH_RELATIONS, + "email": ALL_WITH_RELATIONS, "document": ALL_WITH_RELATIONS, - "author": ALL_WITH_RELATIONS, } api.doc.register(DocumentAuthorResource()) @@ -207,7 +209,6 @@ class DocHistoryResource(ModelResource): doc = ToOneField(DocumentResource, 'doc') states = ToManyField(StateResource, 'states', null=True) tags = ToManyField(DocTagNameResource, 'tags', null=True) - authors = ToManyField(EmailResource, 'authors', null=True) class Meta: cache = SimpleCache() queryset = DocHistory.objects.all() @@ -237,7 +238,6 @@ class DocHistoryResource(ModelResource): "doc": ALL_WITH_RELATIONS, "states": ALL_WITH_RELATIONS, "tags": ALL_WITH_RELATIONS, - "authors": ALL_WITH_RELATIONS, } api.doc.register(DocHistoryResource()) @@ -405,10 +405,11 @@ class InitialReviewDocEventResource(ModelResource): } api.doc.register(InitialReviewDocEventResource()) -from ietf.person.resources import EmailResource +from ietf.person.resources import PersonResource, EmailResource class DocHistoryAuthorResource(ModelResource): + person = ToOneField(PersonResource, 'person') + email = ToOneField(EmailResource, 'email', null=True) document = ToOneField(DocHistoryResource, 'document') - author = ToOneField(EmailResource, 'author') class Meta: cache = SimpleCache() queryset = DocHistoryAuthor.objects.all() @@ -416,9 +417,12 @@ class DocHistoryAuthorResource(ModelResource): #resource_name = 'dochistoryauthor' filtering = { "id": ALL, + "affiliation": ALL, + "country": ALL, "order": ALL, + "person": ALL_WITH_RELATIONS, + "email": ALL_WITH_RELATIONS, "document": ALL_WITH_RELATIONS, - "author": ALL_WITH_RELATIONS, } api.doc.register(DocHistoryAuthorResource()) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index e4a171813..d025b5de9 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -87,7 +87,7 @@ class SearchTests(TestCase): self.assertTrue(draft.title in unicontent(r)) # find by author - r = self.client.get(base_url + "?activedrafts=on&by=author&author=%s" % draft.authors.all()[0].person.name_parts()[1]) + r = self.client.get(base_url + "?activedrafts=on&by=author&author=%s" % draft.documentauthor_set.first().person.name_parts()[1]) self.assertEqual(r.status_code, 200) self.assertTrue(draft.title in unicontent(r)) @@ -1223,7 +1223,7 @@ class ChartTests(ResourceTestCaseMixin, TestCase): person = PersonFactory.create() DocumentFactory.create( states=[('draft','active')], - authors=[person.email(), ], + authors=[person, ], ) conf_url = urlreverse('ietf.doc.views_stats.chart_conf_person_drafts', kwargs=dict(id=person.id)) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 1f024b0ed..3c2c6a32a 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -377,7 +377,10 @@ class EditInfoTests(TestCase): DocumentAuthor.objects.create( document=draft, - author=Email.objects.get(address="aread@ietf.org"), + person=Person.objects.get(email__address="aread@ietf.org"), + email=Email.objects.get(address="aread@ietf.org"), + country="US", + affiliation="", order=1 ) @@ -1361,7 +1364,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.basea.documentauthor_set.create(author=Email.objects.create(address="basea_author@example.com"),order=1) + p = Person.objects.create(address="basea_author") + e = Email.objects.create(address="basea_author@example.com", person=p) + self.basea.documentauthor_set.create(person=p, email=e, order=1) self.baseb = Document.objects.create( name="draft-test-base-b", @@ -1372,7 +1377,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() - datetime.timedelta(days = 365 - settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.baseb.documentauthor_set.create(author=Email.objects.create(address="baseb_author@example.com"),order=1) + p = Person.objects.create(name="baseb_author") + e = Email.objects.create(address="baseb_author@example.com", person=p) + self.baseb.documentauthor_set.create(person=p, email=e, order=1) self.replacea = Document.objects.create( name="draft-test-replace-a", @@ -1383,7 +1390,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() + datetime.timedelta(days = settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.replacea.documentauthor_set.create(author=Email.objects.create(address="replacea_author@example.com"),order=1) + p = Person.objects.create(name="replacea_author") + e = Email.objects.create(address="replacea_author@example.com", person=p) + self.replacea.documentauthor_set.create(person=p, email=e, order=1) self.replaceboth = Document.objects.create( name="draft-test-replace-both", @@ -1394,7 +1403,9 @@ class ChangeReplacesTests(TestCase): expires=datetime.datetime.now() + datetime.timedelta(days = settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), group=mars_wg, ) - self.replaceboth.documentauthor_set.create(author=Email.objects.create(address="replaceboth_author@example.com"),order=1) + p = Person.objects.create(name="replaceboth_author") + e = Email.objects.create(address="replaceboth_author@example.com", person=p) + self.replaceboth.documentauthor_set.create(person=p, email=e, order=1) self.basea.set_state(State.objects.get(used=True, type="draft", slug="active")) self.baseb.set_state(State.objects.get(used=True, type="draft", slug="expired")) diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index f477f3323..58a9ef010 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -258,10 +258,7 @@ class ReviewTests(TestCase): # set up some reviewer-suitability factors reviewer_email = Email.objects.get(person__user__username="reviewer") - DocumentAuthor.objects.create( - author=reviewer_email, - document=doc, - ) + DocumentAuthor.objects.create(person=reviewer_email.person, email=reviewer_email, document=doc) doc.rev = "10" doc.save_with_history([DocEvent.objects.create(doc=doc, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")]) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index ab950d406..7f81d2802 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -57,7 +57,6 @@ from ietf.group.models import Role from ietf.group.utils import can_manage_group, can_manage_materials from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, user_is_person, role_required from ietf.name.models import StreamName, BallotPositionName -from ietf.person.models import Email from ietf.utils.history import find_history_active_at from ietf.doc.forms import TelechatForm, NotifyForm from ietf.doc.mails import email_comment @@ -167,7 +166,7 @@ def document_main(request, name, rev=None): can_edit_replaces = has_role(request.user, ("Area Director", "Secretariat", "IRTF Chair", "WG Chair", "RG Chair", "WG Secretary", "RG Secretary")) - is_author = unicode(request.user) in set([email.address for email in doc.authors.all()]) + is_author = request.user.is_authenticated() and doc.documentauthor_set.filter(person__user=request.user).exists() can_view_possibly_replaces = can_edit_replaces or is_author rfc_number = name[3:] if name.startswith("") else None @@ -957,11 +956,11 @@ def document_json(request, name, rev=None): data["intended_std_level"] = extract_name(doc.intended_std_level) data["std_level"] = extract_name(doc.std_level) data["authors"] = [ - dict(name=e.person.name, - email=e.address, - affiliation=e.person.affiliation) - for e in Email.objects.filter(documentauthor__document=doc).select_related("person").order_by("documentauthor__order") - ] + dict(name=author.person.name, + email=author.email.address, + affiliation=author.affiliation) + for author in doc.documentauthor_set.all().select_related("person", "email").order_by("order") + ] data["shepherd"] = doc.shepherd.formatted_email() if doc.shepherd else None data["ad"] = doc.ad.role_email("ad").formatted_email() if doc.ad else None diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index e1ae950e8..0b22a2b72 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -167,7 +167,7 @@ def retrieve_search_results(form, all_types=False): # radio choices by = query["by"] if by == "author": - docs = docs.filter(authors__person__alias__name__icontains=query["author"]) + docs = docs.filter(documentauthor__person__alias__name__icontains=query["author"]) elif by == "group": docs = docs.filter(group__acronym=query["group"]) elif by == "area": diff --git a/ietf/doc/views_stats.py b/ietf/doc/views_stats.py index 6fa1a094b..9d1d02a3e 100644 --- a/ietf/doc/views_stats.py +++ b/ietf/doc/views_stats.py @@ -170,7 +170,7 @@ def chart_data_person_drafts(request, id): if not person: data = [] else: - data = model_to_timeline_data(DocEvent, doc__authors__person=person, type='new_revision') + data = model_to_timeline_data(DocEvent, doc__documentauthor__person=person, type='new_revision') return JsonResponse(data, safe=False) diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py index ec4e111ef..d02ceee5f 100644 --- a/ietf/idindex/index.py +++ b/ietf/idindex/index.py @@ -115,15 +115,15 @@ def all_id2_txt(): file_types = file_types_for_drafts() authors = {} - for a in DocumentAuthor.objects.filter(document__name__startswith="draft-").order_by("order").select_related("author", "author__person").iterator(): + for a in DocumentAuthor.objects.filter(document__name__startswith="draft-").order_by("order").select_related("email", "person").iterator(): if a.document_id not in authors: l = authors[a.document_id] = [] else: l = authors[a.document_id] - if "@" in a.author.address: - l.append(u'%s <%s>' % (a.author.person.plain_name().replace("@", ""), a.author.address.replace(",", ""))) + if a.email: + l.append(u'%s <%s>' % (a.person.plain_name().replace("@", ""), a.email.address.replace(",", ""))) else: - l.append(a.author.person.plain_name()) + l.append(a.person.plain_name()) shepherds = dict((e.pk, e.formatted_email().replace('"', '')) for e in Email.objects.filter(shepherd_document_set__type="draft").select_related("person").distinct()) @@ -234,12 +234,12 @@ def active_drafts_index_by_group(extra_values=()): d["initial_rev_time"] = time # add authors - for a in DocumentAuthor.objects.filter(document__states=active_state).order_by("order").select_related("author__person"): + for a in DocumentAuthor.objects.filter(document__states=active_state).order_by("order").select_related("person"): d = docs_dict.get(a.document_id) if d: if "authors" not in d: d["authors"] = [] - d["authors"].append(a.author.person.plain_ascii()) # This should probably change to .plain_name() when non-ascii names are permitted + d["authors"].append(a.person.plain_ascii()) # This should probably change to .plain_name() when non-ascii names are permitted # put docs into groups for d in docs_dict.itervalues(): diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py index 5f7ff9d9c..4d7aaa699 100644 --- a/ietf/idindex/tests.py +++ b/ietf/idindex/tests.py @@ -99,7 +99,7 @@ class IndexTests(TestCase): self.assertEqual(t[12], ".pdf,.txt") self.assertEqual(t[13], draft.title) author = draft.documentauthor_set.order_by("order").get() - self.assertEqual(t[14], u"%s <%s>" % (author.author.person.name, author.author.address)) + self.assertEqual(t[14], u"%s <%s>" % (author.person.name, author.email.address)) self.assertEqual(t[15], u"%s <%s>" % (draft.shepherd.person.name, draft.shepherd.address)) self.assertEqual(t[16], u"%s <%s>" % (draft.ad.plain_ascii(), draft.ad.email_address())) diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index 3ec7a0a3d..61da03c23 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -63,15 +63,15 @@ def get_document_emails(ipr): messages = [] for rel in ipr.iprdocrel_set.all(): doc = rel.document.document - authors = doc.authors.all() - + if is_draft(doc): doc_info = 'Internet-Draft entitled "{}" ({})'.format(doc.title,doc.name) else: doc_info = 'RFC entitled "{}" (RFC{})'.format(doc.title,get_rfc_num(doc)) addrs = gather_address_lists('ipr_posted_on_doc',doc=doc).as_strings(compact=False) - author_names = ', '.join([a.person.name for a in authors]) + + author_names = ', '.join(a.person.name for a in doc.documentauthor_set.select_related("person")) context = dict( doc_info=doc_info, diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py index 526b5f5dd..40392e0e4 100644 --- a/ietf/mailtrigger/models.py +++ b/ietf/mailtrigger/models.py @@ -199,7 +199,7 @@ class Recipient(models.Model): submission = kwargs['submission'] doc=submission.existing_document() if doc: - old_authors = [i.author.formatted_email() for i in doc.documentauthor_set.all() if not i.author.invalid_address()] + old_authors = [author.formatted_email() for author in doc.documentauthor_set.all() if author.email] new_authors = [u'"%s" <%s>' % (author["name"], author["email"]) for author in submission.authors_parsed() if author["email"]] addrs.extend(old_authors) if doc.group and set(old_authors)!=set(new_authors): diff --git a/ietf/name/migrations/0018_add_formlang_names.py b/ietf/name/migrations/0018_add_formlang_names.py index 542cb6cf5..a0dd3ff19 100644 --- a/ietf/name/migrations/0018_add_formlang_names.py +++ b/ietf/name/migrations/0018_add_formlang_names.py @@ -12,9 +12,6 @@ def insert_initial_formal_language_names(apps, schema_editor): FormalLanguageName.objects.get_or_create(slug="json", name="JSON", desc="Javascript Object Notation", order=5) FormalLanguageName.objects.get_or_create(slug="xml", name="XML", desc="Extensible Markup Language", order=6) -def noop(apps, schema_editor): - pass - class Migration(migrations.Migration): dependencies = [ @@ -22,5 +19,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(insert_initial_formal_language_names, noop) + migrations.RunPython(insert_initial_formal_language_names, migrations.RunPython.noop) ] diff --git a/ietf/name/resources.py b/ietf/name/resources.py index 15bc6d378..f6a74387d 100644 --- a/ietf/name/resources.py +++ b/ietf/name/resources.py @@ -14,7 +14,8 @@ from ietf.name.models import (TimeSlotTypeName, GroupStateName, DocTagName, Inte ConstraintName, MeetingTypeName, DocRelationshipName, RoomResourceName, IprLicenseTypeName, LiaisonStatementTagName, FeedbackTypeName, LiaisonStatementState, StreamName, BallotPositionName, DBTemplateTypeName, NomineePositionStateName, - ReviewRequestStateName, ReviewTypeName, ReviewResultName) + ReviewRequestStateName, ReviewTypeName, ReviewResultName, + FormalLanguageName) class TimeSlotTypeNameResource(ModelResource): @@ -456,3 +457,20 @@ class ReviewResultNameResource(ModelResource): } api.name.register(ReviewResultNameResource()) + + +class FormalLanguageNameResource(ModelResource): + class Meta: + queryset = FormalLanguageName.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'formallanguagename' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(FormalLanguageNameResource()) + diff --git a/ietf/person/models.py b/ietf/person/models.py index 9713ccb25..61fa6b2c0 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -125,18 +125,18 @@ class PersonInfo(models.Model): def has_drafts(self): from ietf.doc.models import Document - return Document.objects.filter(authors__person=self, type='draft').exists() + return Document.objects.filter(documentauthor__person=self, type='draft').exists() def rfcs(self): from ietf.doc.models import Document - rfcs = list(Document.objects.filter(authors__person=self, type='draft', states__slug='rfc')) + rfcs = list(Document.objects.filter(documentauthor__person=self, type='draft', states__slug='rfc')) rfcs.sort(key=lambda d: d.canonical_name() ) return rfcs def active_drafts(self): from ietf.doc.models import Document - return Document.objects.filter(authors__person=self, type='draft', states__slug='active').order_by('-time') + return Document.objects.filter(documentauthor__person=self, type='draft', states__slug='active').order_by('-time') def expired_drafts(self): from ietf.doc.models import Document - return Document.objects.filter(authors__person=self, type='draft', states__slug__in=['repl', 'expired', 'auth-rm', 'ietf-rm']).order_by('-time') + return Document.objects.filter(documentauthor__person=self, type='draft', states__slug__in=['repl', 'expired', 'auth-rm', 'ietf-rm']).order_by('-time') class Meta: abstract = True @@ -231,15 +231,11 @@ class Email(models.Model): else: return self.address - def invalid_address(self): - # we have some legacy authors with unknown email addresses - return self.address.startswith("unknown-email") and "@" not in self.address - def email_address(self): """Get valid, current email address; in practise, for active, non-invalid addresses it is just the address field. In other cases, we default to person's email address.""" - if self.invalid_address() or not self.active: + if not self.active: if self.person: return self.person.email_address() return diff --git a/ietf/review/utils.py b/ietf/review/utils.py index ae7124fa8..f1270ef7e 100644 --- a/ietf/review/utils.py +++ b/ietf/review/utils.py @@ -752,7 +752,7 @@ def make_assignment_choices(email_queryset, review_req): connections[r.person_id] = "is group {}".format(r.name) if doc.shepherd: connections[doc.shepherd.person_id] = "is shepherd of document" - for author in DocumentAuthor.objects.filter(document=doc, author__person__in=possible_person_ids).values_list("author__person", flat=True): + for author in DocumentAuthor.objects.filter(document=doc, person__in=possible_person_ids).values_list("person", flat=True): connections[author] = "is author of document" # unavailable periods diff --git a/ietf/secr/drafts/email.py b/ietf/secr/drafts/email.py index cd8889efc..78d83d474 100644 --- a/ietf/secr/drafts/email.py +++ b/ietf/secr/drafts/email.py @@ -48,12 +48,12 @@ def get_authors(draft): Takes a draft object and returns a list of authors suitable for a tombstone document """ authors = [] - for a in draft.authors.all(): + for a in draft.documentauthor_set.all(): initial = '' prefix, first, middle, last, suffix = a.person.name_parts() if first: initial = first + '. ' - entry = '%s%s <%s>' % (initial,last,a.address) + entry = '%s%s <%s>' % (initial,last,a.email.address) authors.append(entry) return authors @@ -64,10 +64,10 @@ def get_abbr_authors(draft): """ initial = '' result = '' - authors = DocumentAuthor.objects.filter(document=draft) + authors = DocumentAuthor.objects.filter(document=draft).order_by("order") if authors: - prefix, first, middle, last, suffix = authors[0].author.person.name_parts() + prefix, first, middle, last, suffix = authors[0].person.name_parts() if first: initial = first[0] + '. ' result = '%s%s' % (initial,last) @@ -140,9 +140,9 @@ def get_fullcc_list(draft): """ emails = {} # get authors - for author in draft.authors.all(): - if author.address not in emails: - emails[author.address] = '"%s"' % (author.person.name) + for author in draft.documentauthor_set.all(): + if author.email and author.email.address not in emails: + emails[author.email.address] = '"%s"' % (author.person.name) if draft.group.acronym != 'none': # add chairs diff --git a/ietf/secr/drafts/forms.py b/ietf/secr/drafts/forms.py index 228f98bfb..956c87109 100644 --- a/ietf/secr/drafts/forms.py +++ b/ietf/secr/drafts/forms.py @@ -4,6 +4,8 @@ import os from django import forms +from django_countries.fields import countries + from ietf.doc.models import Document, DocAlias, State from ietf.name.models import IntendedStdLevelName, DocRelationshipName from ietf.group.models import Group @@ -104,6 +106,8 @@ class AuthorForm(forms.Form): ''' person = forms.CharField(max_length=50,widget=forms.TextInput(attrs={'class':'name-autocomplete'}),help_text="To see a list of people type the first name, or last name, or both.") email = forms.CharField(widget=forms.Select(),help_text="Select an email.") + affiliation = forms.CharField(max_length=100, required=False, help_text="Affiliation") + country = forms.ChoiceField(choices=[('', "(Not specified)")] + list(countries), required=False, help_text="Country") # check for id within parenthesis to ensure name was selected from the list def clean_person(self): diff --git a/ietf/secr/drafts/views.py b/ietf/secr/drafts/views.py index e14c408a4..eb77abd64 100644 --- a/ietf/secr/drafts/views.py +++ b/ietf/secr/drafts/views.py @@ -541,7 +541,7 @@ def approvals(request): @role_required('Secretariat') def author_delete(request, id, oid): ''' - This view deletes the specified author(email) from the draft + This view deletes the specified author from the draft ''' DocumentAuthor.objects.get(id=oid).delete() messages.success(request, 'The author was deleted successfully') @@ -574,14 +574,20 @@ def authors(request, id): return redirect('drafts_view', id=id) + print form.is_valid(), form.errors + if form.is_valid(): - author = form.cleaned_data['email'] + person = form.cleaned_data['person'] + email = form.cleaned_data['email'] + affiliation = form.cleaned_data.get('affiliation') or "" + country = form.cleaned_data.get('country') or "" + authors = draft.documentauthor_set.all() if authors: order = authors.aggregate(Max('order')).values()[0] + 1 else: order = 1 - DocumentAuthor.objects.create(document=draft,author=author,order=order) + DocumentAuthor.objects.create(document=draft, person=person, email=email, affiliation=affiliation, country=country, order=order) messages.success(request, 'Author added successfully!') return redirect('drafts_authors', id=id) diff --git a/ietf/secr/templates/drafts/authors.html b/ietf/secr/templates/drafts/authors.html index 2620db1fb..33e818d82 100644 --- a/ietf/secr/templates/drafts/authors.html +++ b/ietf/secr/templates/drafts/authors.html @@ -24,6 +24,8 @@ Name Email + Affiliation + Country Order Action @@ -31,8 +33,10 @@ {% for author in draft.documentauthor_set.all %} - {{ author.author.person }} - {{ author.author }} + {{ author.person }} + {{ author.email }} + {{ author.affiliation }} + {{ author.country.name }} {{ author.order }} Delete @@ -49,6 +53,10 @@ {{ form.person.errors }}{{ form.person }}{% if form.person.help_text %}
{{ form.person.help_text }}{% endif %} {{ form.email.errors }}{{ form.email }}{% if form.email.help_text %}
{{ form.email.help_text }}{% endif %} + + + {{ form.affiliation.errors }}{{ form.affiliation }}{% if form.affiliation.help_text %}
{{ form.affiliation.help_text }}{% endif %} + {{ form.country.errors }}{{ form.country }}{% if form.country.help_text %}
{{ form.country.help_text }}{% endif %} diff --git a/ietf/secr/templates/drafts/makerfc.html b/ietf/secr/templates/drafts/makerfc.html index 962f76093..f5945d558 100644 --- a/ietf/secr/templates/drafts/makerfc.html +++ b/ietf/secr/templates/drafts/makerfc.html @@ -52,7 +52,7 @@

Author(s)

- {% for author in draft.authors.all %} + {% for author in draft.documentauthor_set.all %} {% endfor %}
{{ author.person.name }}
diff --git a/ietf/secr/templates/drafts/view.html b/ietf/secr/templates/drafts/view.html index 6d9f0a5e6..6d0e2f655 100644 --- a/ietf/secr/templates/drafts/view.html +++ b/ietf/secr/templates/drafts/view.html @@ -73,7 +73,7 @@

Author(s)

{% for author in draft.documentauthor_set.all %} - + {% endfor %}
{{ author.author.person.name }}
{{ author.person.name }}
diff --git a/ietf/settings.py b/ietf/settings.py index 38ab48e83..70ba8dda5 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -293,6 +293,7 @@ INSTALLED_APPS = ( 'tastypie', 'widget_tweaks', 'django_markup', + 'django_countries', # IETF apps 'ietf.api', 'ietf.community', diff --git a/ietf/stats/views.py b/ietf/stats/views.py index a580bb277..047b90c66 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -3,7 +3,6 @@ import itertools import json import calendar import os -import re from collections import defaultdict from django.shortcuts import render @@ -141,7 +140,7 @@ def document_stats(request, stats_type=None, document_type=None): bins = defaultdict(list) - for name, author_count in generate_canonical_names(docalias_qs.values_list("name").annotate(Count("document__authors"))): + for name, author_count in generate_canonical_names(docalias_qs.values_list("name").annotate(Count("document__documentauthor"))): bins[author_count].append(name) series_data = [] diff --git a/ietf/submit/models.py b/ietf/submit/models.py index 1e9e29e59..260494360 100644 --- a/ietf/submit/models.py +++ b/ietf/submit/models.py @@ -65,7 +65,8 @@ class Submission(models.Model): if line: parsed = parse_email_line(line) if not parsed["email"]: - parsed["email"] = ensure_person_email_info_exists(**parsed).address + person, email = ensure_person_email_info_exists(**parsed) + parsed["email"] = email.address res.append(parsed) self._cached_authors_parsed = res return self._cached_authors_parsed diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 6fb3859f9..62ab8c3f2 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -19,7 +19,7 @@ from ietf.group.utils import setup_default_community_list_for_group from ietf.meeting.models import Meeting from ietf.message.models import Message from ietf.name.models import FormalLanguageName -from ietf.person.models import Person, Email +from ietf.person.models import Person from ietf.person.factories import UserFactory, PersonFactory from ietf.submit.models import Submission, Preapproval from ietf.submit.mail import add_submission_email, process_response_email @@ -249,9 +249,10 @@ class SubmitTests(TestCase): self.assertEqual(draft.stream_id, "ietf") self.assertTrue(draft.expires >= datetime.datetime.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE - 1)) self.assertEqual(draft.get_state("draft-stream-%s" % draft.stream_id).slug, "wg-doc") - self.assertEqual(draft.authors.count(), 1) - self.assertEqual(draft.authors.all()[0].get_name(), "Author Name") - self.assertEqual(draft.authors.all()[0].address, "author@example.com") + authors = draft.documentauthor_set.all() + self.assertEqual(len(authors), 1) + self.assertEqual(authors[0].person.plain_name(), "Author Name") + self.assertEqual(authors[0].email.address, "author@example.com") self.assertEqual(set(draft.formal_languages.all()), set(FormalLanguageName.objects.filter(slug="json"))) self.assertEqual(draft.relations_that_doc("replaces").count(), 1) self.assertTrue(draft.relations_that_doc("replaces").first().target, replaced_alias) @@ -290,12 +291,12 @@ class SubmitTests(TestCase): draft.save_with_history([DocEvent.objects.create(doc=draft, type="added_comment", by=Person.objects.get(user__username="secretary"), desc="Test")]) if not change_authors: draft.documentauthor_set.all().delete() - ensure_person_email_info_exists('Author Name','author@example.com') - draft.documentauthor_set.create(author=Email.objects.get(address='author@example.com')) + author_person, author_email = ensure_person_email_info_exists('Author Name','author@example.com') + draft.documentauthor_set.create(person=author_person, email=author_email) else: # Make it such that one of the previous authors has an invalid email address - bogus_email = ensure_person_email_info_exists('Bogus Person',None) - DocumentAuthor.objects.create(document=draft,author=bogus_email,order=draft.documentauthor_set.latest('order').order+1) + bogus_person, bogus_email = ensure_person_email_info_exists('Bogus Person',None) + DocumentAuthor.objects.create(document=draft, person=bogus_person, email=bogus_email, order=draft.documentauthor_set.latest('order').order+1) prev_author = draft.documentauthor_set.all()[0] @@ -342,7 +343,7 @@ class SubmitTests(TestCase): confirm_email = outbox[-1] self.assertTrue("Confirm submission" in confirm_email["Subject"]) self.assertTrue(name in confirm_email["Subject"]) - self.assertTrue(prev_author.author.address in confirm_email["To"]) + self.assertTrue(prev_author.email.address in confirm_email["To"]) if change_authors: self.assertTrue("author@example.com" not in confirm_email["To"]) self.assertTrue("submitter@example.com" not in confirm_email["To"]) @@ -423,9 +424,10 @@ class SubmitTests(TestCase): self.assertEqual(draft.stream_id, "ietf") self.assertEqual(draft.get_state_slug("draft-stream-%s" % draft.stream_id), "wg-doc") self.assertEqual(draft.get_state_slug("draft-iana-review"), "changed") - self.assertEqual(draft.authors.count(), 1) - self.assertEqual(draft.authors.all()[0].get_name(), "Author Name") - self.assertEqual(draft.authors.all()[0].address, "author@example.com") + authors = draft.documentauthor_set.all() + self.assertEqual(len(authors), 1) + self.assertEqual(authors[0].person.plain_name(), "Author Name") + self.assertEqual(authors[0].email.address, "author@example.com") self.assertEqual(len(outbox), mailbox_before + 3) self.assertTrue((u"I-D Action: %s" % name) in outbox[-3]["Subject"]) self.assertTrue((u"I-D Action: %s" % name) in draft.message_set.order_by("-time")[0].subject) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 957b5eff2..ad5f02c6a 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -125,7 +125,7 @@ def docevent_from_submission(request, submission, desc, who=None): else: submitter_parsed = submission.submitter_parsed() if submitter_parsed["name"] and submitter_parsed["email"]: - by = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]).person + by, _ = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]) else: by = system @@ -179,7 +179,7 @@ def post_submission(request, submission, approvedDesc): system = Person.objects.get(name="(System)") submitter_parsed = submission.submitter_parsed() if submitter_parsed["name"] and submitter_parsed["email"]: - submitter = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]).person + submitter, _ = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]) submitter_info = u'%s <%s>' % (submitter_parsed["name"], submitter_parsed["email"]) else: submitter = system @@ -341,10 +341,9 @@ def update_replaces_from_submission(request, submission, draft): if rdoc == draft: continue - # TODO - I think the .exists() is in the wrong place below.... if (is_secretariat or (draft.group in is_chair_of and (rdoc.group.type_id == "individ" or rdoc.group in is_chair_of)) - or (submitter_email and rdoc.authors.filter(address__iexact=submitter_email)).exists()): + or (submitter_email and rdoc.documentauthor_set.filter(email__address__iexact=submitter_email).exists())): approved.append(r) else: if r not in existing_suggested: @@ -424,23 +423,24 @@ def ensure_person_email_info_exists(name, email): email.person = person email.save() - return email + return person, email def update_authors(draft, submission): - authors = [] + persons = [] for order, author in enumerate(submission.authors_parsed()): - email = ensure_person_email_info_exists(author["name"], author["email"]) + person, email = ensure_person_email_info_exists(author["name"], author["email"]) - a = DocumentAuthor.objects.filter(document=draft, author=email).first() + a = DocumentAuthor.objects.filter(document=draft, person=person).first() if not a: - a = DocumentAuthor(document=draft, author=email) + a = DocumentAuthor(document=draft, person=person) + a.email = email a.order = order a.save() - authors.append(email) + persons.append(person) - draft.documentauthor_set.exclude(author__in=authors).delete() + draft.documentauthor_set.exclude(person__in=persons).delete() def cancel_submission(submission): submission.state = DraftSubmissionStateName.objects.get(slug="cancel") diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 95b9c2ae8..05a1aa379 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -259,7 +259,7 @@ def submission_status(request, submission_id, access_token=None): group_authors_changed = False doc = submission.existing_document() if doc and doc.group: - old_authors = [ i.author.person for i in doc.documentauthor_set.all() ] + old_authors = [ author.person for author in doc.documentauthor_set.all() ] new_authors = [ get_person_from_name_email(**p) for p in submission.authors_parsed() ] group_authors_changed = set(old_authors)!=set(new_authors) diff --git a/ietf/templates/doc/bibxml.xml b/ietf/templates/doc/bibxml.xml index 9813c3a00..fdb57c132 100644 --- a/ietf/templates/doc/bibxml.xml +++ b/ietf/templates/doc/bibxml.xml @@ -2,11 +2,11 @@ {{doc.title}} - {% for entry in doc.authors.all %}{% with entry.address as email %}{% with entry.person as author %} - - {{author.affiliation}} + {% for author in doc.documentauthor_set.all %} + + {{ author.affiliation }} - {% endwith %}{% endwith %}{% endfor %} + {% endfor %} {{doc.abstract}} diff --git a/ietf/templates/doc/document_bibtex.bib b/ietf/templates/doc/document_bibtex.bib index 489abf3ec..0e10add29 100644 --- a/ietf/templates/doc/document_bibtex.bib +++ b/ietf/templates/doc/document_bibtex.bib @@ -24,7 +24,7 @@ publisher = {% templatetag openbrace %}Internet Engineering Task Force{% templatetag closebrace %}, note = {% templatetag openbrace %}Work in Progress{% templatetag closebrace %}, url = {% templatetag openbrace %}https://tools.ietf.org/html/{{doc.name}}-{{doc.rev}}{% templatetag closebrace %},{% endif %} - author = {% templatetag openbrace %}{% for entry in doc.authors.all %}{% with entry.person as author %}{{author.name}}{% endwith %}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, + author = {% templatetag openbrace %}{% for author in doc.documentauthor_set.all %}{{ author.person.name}}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, title = {% templatetag openbrace %}{% templatetag openbrace %}{{doc.title}}{% templatetag closebrace %}{% templatetag closebrace %}, pagetotal = {{ doc.pages }}, year = {{ doc.pub_date.year }}, diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index bed55b798..7ebd65f90 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -570,13 +570,13 @@

Authors

{% for author in doc.documentauthor_set.all %} - {% if not author.author.invalid_address %} + {% if author.email %} - + {% endif %} - {{ author.author.person }} - {% if not author.author.invalid_address %} - ({{ author.author.address }}) + {{ author.person }} + {% if author.email %} + ({{ author.email.address }}) {% endif %} {% if not forloop.last %}
{% endif %} {% endfor %} diff --git a/ietf/utils/test_data.py b/ietf/utils/test_data.py index 938fcbc5c..80229fdb7 100644 --- a/ietf/utils/test_data.py +++ b/ietf/utils/test_data.py @@ -91,7 +91,7 @@ def make_immutable_base_data(): # one area area = create_group(name="Far Future", acronym="farfut", type_id="area", parent=ietf) - create_person(area, "ad", name="Areað Irector", username="ad", email_address="aread@ietf.org") + create_person(area, "ad", name=u"Areað Irector", username="ad", email_address="aread@ietf.org") # second area opsarea = create_group(name="Operations", acronym="ops", type_id="area", parent=ietf) @@ -276,7 +276,8 @@ def make_test_data(): DocumentAuthor.objects.create( document=draft, - author=Email.objects.get(address="aread@ietf.org"), + person=Person.objects.get(email__address="aread@ietf.org"), + email=Email.objects.get(address="aread@ietf.org"), order=1 ) diff --git a/requirements.txt b/requirements.txt index 16aba8935..d5eb29170 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ decorator>=3.4.0 defusedxml>=0.4.1 # for TastyPie when ussing xml; not a declared dependency Django>=1.8.16,<1.9 django-bootstrap3>=5.1.1,<7.0.0 # django-bootstrap 7.0 requires django 1.8 +django-countries>=4.0 django-formtools>=1.0 # instead of django.contrib.formtools in 1.8 django-markup>=1.1 django-tastypie>=0.13.1 From 698965e3d12fbace2d0e250eae7d9060c8ace874 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 27 Jan 2017 12:33:05 +0000 Subject: [PATCH 16/49] Fix one last fake email addresses that managed to escape the migration - Legacy-Id: 12743 --- .../migrations/0021_remove_fake_email_adresses.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ietf/doc/migrations/0021_remove_fake_email_adresses.py b/ietf/doc/migrations/0021_remove_fake_email_adresses.py index d317ed03d..7e6c73594 100644 --- a/ietf/doc/migrations/0021_remove_fake_email_adresses.py +++ b/ietf/doc/migrations/0021_remove_fake_email_adresses.py @@ -7,6 +7,8 @@ def fix_invalid_emails(apps, schema_editor): Email = apps.get_model("person", "Email") Role = apps.get_model("group", "Role") RoleHistory = apps.get_model("group", "RoleHistory") + DocumentAuthor = apps.get_model("doc", "DocumentAuthor") + DocHistoryAuthor = apps.get_model("doc", "DocHistoryAuthor") e = Email.objects.filter(address="unknown-email-Gigi-Karmous-Edwards").first() if e: @@ -29,14 +31,20 @@ def fix_invalid_emails(apps, schema_editor): RoleHistory.objects.filter(email=e).update(email=new_e) e.delete() - Email = apps.get_model("person", "Email") - DocumentAuthor = apps.get_model("doc", "DocumentAuthor") - DocHistoryAuthor = apps.get_model("doc", "DocHistoryAuthor") + e = Email.objects.filter(address="unknown-email-Greg->").first() + if e: + # current email + new_e = Email.objects.get(address="gregimirsky@gmail.com") + DocumentAuthor.objects.filter(email=e).update(email=new_e) + DocHistoryAuthor.objects.filter(email=e).update(email=new_e) + e.delete() DocumentAuthor.objects.filter(email__address__startswith="unknown-email-").exclude(email__address__contains="@").update(email=None) DocHistoryAuthor.objects.filter(email__address__startswith="unknown-email-").exclude(email__address__contains="@").update(email=None) Email.objects.exclude(address__contains="@").filter(address__startswith="unknown-email-").delete() + assert not Email.objects.filter(address__startswith="unknown-email-") + class Migration(migrations.Migration): dependencies = [ From 4426e3386f7f7fb012553902fd67bee654dbe1fc Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 27 Jan 2017 13:27:19 +0000 Subject: [PATCH 17/49] Add assertNoFormPostErrors for use instead of testing the status code of form post responses to make it easier to see what goes wrong when the form in a test suddently doesn't validate - Legacy-Id: 12744 --- ietf/utils/test_utils.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index 67ef1f338..d8bb1ef31 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -276,7 +276,7 @@ def unicontent(r): def reload_db_objects(*objects): """Rerequest the given arguments from the database so they're refreshed, to be used like - foo, bar = reload_objects(foo, bar)""" + foo, bar = reload_db_objects(foo, bar)""" t = tuple(o.__class__.objects.get(pk=o.pk) for o in objects) if len(objects) == 1: @@ -307,5 +307,27 @@ class TestCase(django.test.TestCase): self.assertTrue(resp['Content-Type'].startswith('text/html')) self.assertValidHTML(resp.content) + def assertNoFormPostErrors(self, response, error_css_selector=".has-error"): + """Try to fish out form errors, if none found at least check the + status code to be a redirect. + + Assumptions: + - a POST is followed by a 302 redirect + - form errors can be found with a simple CSS selector + + """ + + if response.status_code == 200: + from pyquery import PyQuery + from lxml import html + self.maxDiff = None + + errors = [html.tostring(n).decode() for n in PyQuery(response.content)(error_css_selector)] + if errors: + explanation = u"{} != {}\nGot form back with errors:\n----\n".format(response.status_code, 302) + u"----\n".join(errors) + self.assertEqual(response.status_code, 302, explanation) + + self.assertEqual(response.status_code, 302) + def __str__(self): return "%s (%s.%s)" % (self._testMethodName, strclass(self.__class__),self._testMethodName) From 90051a157555dd1ebee5741cb69e26aacd7dc8df Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 27 Jan 2017 16:10:31 +0000 Subject: [PATCH 18/49] Add support for displaying and editing author affiliation and country when submitting a draft, replace the Submission.authors line-based text field with a JSON field - Legacy-Id: 12745 --- ietf/mailtrigger/models.py | 6 +- ietf/secr/drafts/views.py | 2 - ietf/submit/forms.py | 46 ++++++++----- ietf/submit/models.py | 17 +---- ietf/submit/tests.py | 19 ++--- ietf/submit/utils.py | 8 ++- ietf/submit/views.py | 69 ++++++++++--------- ietf/templates/submit/announce_to_lists.txt | 2 +- ietf/templates/submit/approval_request.txt | 2 +- ietf/templates/submit/manual_post_request.txt | 2 +- ietf/templates/submit/submission_status.html | 12 ++-- ietf/templates/submit/submitter_form.html | 4 +- ietf/utils/templatetags/country.py | 14 ++++ 13 files changed, 110 insertions(+), 93 deletions(-) create mode 100644 ietf/utils/templatetags/country.py diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py index 40392e0e4..db691750c 100644 --- a/ietf/mailtrigger/models.py +++ b/ietf/mailtrigger/models.py @@ -177,7 +177,7 @@ class Recipient(models.Model): addrs = [] if 'submission' in kwargs: submission = kwargs['submission'] - addrs.extend(["%s <%s>" % (author["name"], author["email"]) for author in submission.authors_parsed() if author["email"]]) + addrs.extend(["%s <%s>" % (author["name"], author["email"]) for author in submission.authors if author.get("email")]) return addrs def gather_submission_group_chairs(self, **kwargs): @@ -200,7 +200,7 @@ class Recipient(models.Model): doc=submission.existing_document() if doc: old_authors = [author.formatted_email() for author in doc.documentauthor_set.all() if author.email] - new_authors = [u'"%s" <%s>' % (author["name"], author["email"]) for author in submission.authors_parsed() if author["email"]] + new_authors = [u'"%s" <%s>' % (author["name"], author["email"]) for author in submission.authors if author.get("email")] addrs.extend(old_authors) if doc.group and set(old_authors)!=set(new_authors): if doc.group.type_id in ['wg','rg','ag']: @@ -212,7 +212,7 @@ class Recipient(models.Model): if doc.stream_id and doc.stream_id not in ['ietf']: addrs.extend(Recipient.objects.get(slug='stream_managers').gather(**{'streams':[doc.stream_id]})) else: - addrs.extend([u"%s <%s>" % (author["name"], author["email"]) for author in submission.authors_parsed() if author["email"]]) + addrs.extend([u"%s <%s>" % (author["name"], author["email"]) for author in submission.authors if author.get("email")]) if submission.submitter_parsed()["email"]: addrs.append(submission.submitter) return addrs diff --git a/ietf/secr/drafts/views.py b/ietf/secr/drafts/views.py index eb77abd64..f431516f6 100644 --- a/ietf/secr/drafts/views.py +++ b/ietf/secr/drafts/views.py @@ -574,8 +574,6 @@ def authors(request, id): return redirect('drafts_view', id=id) - print form.is_valid(), form.errors - if form.is_valid(): person = form.cleaned_data['person'] email = form.cleaned_data['email'] diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index f01cb44ab..6af82a6c3 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -12,6 +12,8 @@ from django.conf import settings from django.utils.html import mark_safe from django.core.urlresolvers import reverse as urlreverse +from django_countries.fields import countries + import debug # pyflakes:ignore from ietf.doc.models import Document @@ -30,6 +32,14 @@ from ietf.submit.parsers.ps_parser import PSParser from ietf.submit.parsers.xml_parser import XMLParser from ietf.utils.draft import Draft +def clean_country(country): + country = country.upper() + for code, name in countries: + if country == code: + return code + if country == name.upper(): + return code + return "" # unknown class SubmissionUploadForm(forms.Form): txt = forms.FileField(label=u'.txt format', required=False) @@ -178,18 +188,14 @@ class SubmissionUploadForm(forms.Form): self.abstract = self.xmlroot.findtext('front/abstract').strip() if type(self.abstract) is unicode: self.abstract = unidecode(self.abstract) - self.author_list = [] author_info = self.xmlroot.findall('front/author') for author in author_info: - author_dict = dict( - company = author.findtext('organization'), - last_name = author.attrib.get('surname'), - full_name = author.attrib.get('fullname'), - email = author.findtext('address/email'), - ) - self.author_list.append(author_dict) - line = "%(full_name)s <%(email)s>" % author_dict - self.authors.append(line) + self.authors.append({ + "name": author.attrib.get('fullname'), + "email": author.findtext('address/email'), + "affiliation": author.findtext('organization'), + "country": clean_country(author.findtext('address/postal/country')), + }) except forms.ValidationError: raise except Exception as e: @@ -325,18 +331,12 @@ class SubmissionUploadForm(forms.Form): return None class NameEmailForm(forms.Form): - """For validating supplied submitter and author information.""" name = forms.CharField(required=True) - email = forms.EmailField(label=u'Email address') - - #Fields for secretariat only - approvals_received = forms.BooleanField(label=u'Approvals received', required=False, initial=False) + email = forms.EmailField(label=u'Email address', required=True) def __init__(self, *args, **kwargs): - email_required = kwargs.pop("email_required", True) super(NameEmailForm, self).__init__(*args, **kwargs) - self.fields["email"].required = email_required self.fields["name"].widget.attrs["class"] = "name" self.fields["email"].widget.attrs["class"] = "email" @@ -346,6 +346,18 @@ class NameEmailForm(forms.Form): def clean_email(self): return self.cleaned_data["email"].replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() +class AuthorForm(NameEmailForm): + affiliation = forms.CharField(max_length=100, required=False) + country = forms.ChoiceField(choices=[('', "(Not specified)")] + list(countries), required=False) + + def __init__(self, *args, **kwargs): + super(AuthorForm, self).__init__(*args, **kwargs) + self.fields["email"].required = False + +class SubmitterForm(NameEmailForm): + #Fields for secretariat only + approvals_received = forms.BooleanField(label=u'Approvals received', required=False, initial=False) + def cleaned_line(self): line = self.cleaned_data["name"] email = self.cleaned_data.get("email") diff --git a/ietf/submit/models.py b/ietf/submit/models.py index 260494360..396811005 100644 --- a/ietf/submit/models.py +++ b/ietf/submit/models.py @@ -39,7 +39,7 @@ class Submission(models.Model): words = models.IntegerField(null=True, blank=True) formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document") - authors = models.TextField(blank=True, help_text="List of author names and emails, one author per line, e.g. \"John Doe <john@example.org>\".") + authors = jsonfield.JSONField(default=list, help_text="List of authors with name, email, affiliation and country code.") note = models.TextField(blank=True) replaces = models.CharField(max_length=1000, blank=True) @@ -56,21 +56,6 @@ class Submission(models.Model): def __unicode__(self): return u"%s-%s" % (self.name, self.rev) - def authors_parsed(self): - if not hasattr(self, '_cached_authors_parsed'): - from ietf.submit.utils import ensure_person_email_info_exists - res = [] - for line in self.authors.replace("\r", "").split("\n"): - line = line.strip() - if line: - parsed = parse_email_line(line) - if not parsed["email"]: - person, email = ensure_person_email_info_exists(**parsed) - parsed["email"] = email.address - res.append(parsed) - self._cached_authors_parsed = res - return self._cached_authors_parsed - def submitter_parsed(self): return parse_email_line(self.submitter) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 62ab8c3f2..955901761 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -125,7 +125,7 @@ class SubmitTests(TestCase): q = PyQuery(r.content) print(q('div.has-error div.alert').text()) - self.assertEqual(r.status_code, 302) + self.assertNoFormPostErrors(r, ".has-error,.alert-danger") status_url = r["Location"] for format in formats: @@ -133,10 +133,12 @@ class SubmitTests(TestCase): self.assertEqual(Submission.objects.filter(name=name).count(), 1) submission = Submission.objects.get(name=name) self.assertTrue(all([ c.passed!=False for c in submission.checks.all() ])) - self.assertEqual(len(submission.authors_parsed()), 1) - author = submission.authors_parsed()[0] + self.assertEqual(len(submission.authors), 1) + author = submission.authors[0] self.assertEqual(author["name"], "Author Name") self.assertEqual(author["email"], "author@example.com") + self.assertEqual(author["affiliation"], "Test Centre Inc.") + # FIXMEself.assertEqual(author["country"], "UK") return status_url @@ -664,7 +666,7 @@ class SubmitTests(TestCase): "authors-prefix": ["authors-", "authors-0", "authors-1", "authors-2"], }) - self.assertEqual(r.status_code, 302) + self.assertNoFormPostErrors(r, ".has-error,.alert-danger") submission = Submission.objects.get(name=name) self.assertEqual(submission.title, "some title") @@ -676,14 +678,14 @@ class SubmitTests(TestCase): self.assertEqual(submission.replaces, draft.docalias_set.all().first().name) self.assertEqual(submission.state_id, "manual") - authors = submission.authors_parsed() + authors = submission.authors self.assertEqual(len(authors), 3) self.assertEqual(authors[0]["name"], "Person 1") self.assertEqual(authors[0]["email"], "person1@example.com") self.assertEqual(authors[1]["name"], "Person 2") self.assertEqual(authors[1]["email"], "person2@example.com") self.assertEqual(authors[2]["name"], "Person 3") - self.assertEqual(authors[2]["email"], "unknown-email-Person-3") + self.assertEqual(authors[2]["email"], "") self.assertEqual(len(outbox), mailbox_before + 1) self.assertTrue("Manual Post Requested" in outbox[-1]["Subject"]) @@ -939,7 +941,6 @@ class SubmitTests(TestCase): files = {"txt": submission_file(name, rev, group, "txt", "test_submission.nonascii", author=author) } r = self.client.post(url, files) - self.assertEqual(r.status_code, 302) status_url = r["Location"] r = self.client.get(status_url) @@ -1443,8 +1444,8 @@ Subject: test self.assertEqual(Submission.objects.filter(name=name).count(), 1) submission = Submission.objects.get(name=name) self.assertTrue(all([ c.passed!=False for c in submission.checks.all() ])) - self.assertEqual(len(submission.authors_parsed()), 1) - author = submission.authors_parsed()[0] + self.assertEqual(len(submission.authors), 1) + author = submission.authors[0] self.assertEqual(author["name"], "Author Name") self.assertEqual(author["email"], "author@example.com") diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index ad5f02c6a..54852349d 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -42,7 +42,7 @@ def validate_submission(submission): if not submission.abstract: errors['abstract'] = 'Abstract is empty or was not found' - if not submission.authors_parsed(): + if not submission.authors: errors['authors'] = 'No authors found' # revision @@ -427,14 +427,16 @@ def ensure_person_email_info_exists(name, email): def update_authors(draft, submission): persons = [] - for order, author in enumerate(submission.authors_parsed()): - person, email = ensure_person_email_info_exists(author["name"], author["email"]) + for order, author in enumerate(submission.authors): + person, email = ensure_person_email_info_exists(author["name"], author.get("email")) a = DocumentAuthor.objects.filter(document=draft, person=person).first() if not a: a = DocumentAuthor(document=draft, person=person) a.email = email + a.affiliation = author.get("affiliation") or "" + a.country = author.get("country") or "" a.order = order a.save() diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 05a1aa379..bd2305779 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -21,7 +21,7 @@ from ietf.ietfauth.utils import has_role, role_required from ietf.mailtrigger.utils import gather_address_lists from ietf.message.models import Message, MessageAttachment from ietf.name.models import FormalLanguageName -from ietf.submit.forms import ( SubmissionUploadForm, NameEmailForm, EditSubmissionForm, +from ietf.submit.forms import ( SubmissionUploadForm, AuthorForm, SubmitterForm, EditSubmissionForm, PreapprovalForm, ReplacesForm, SubmissionEmailForm, MessageModelForm ) from ietf.submit.mail import ( send_full_url, send_approval_request_to_group, send_submission_confirmation, send_manual_post_request, add_submission_email, get_reply_to ) @@ -83,8 +83,7 @@ def upload_submission(request): for author in form.parsed_draft.get_author_list(): full_name, first_name, middle_initial, last_name, name_suffix, email, company = author - line = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() - email = (email or "").strip() + name = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() if email: try: @@ -92,29 +91,31 @@ def upload_submission(request): except ValidationError: email = "" - if email: - # Try various ways of handling name and email, in order to avoid - # triggering a 500 error here. If the document contains non-ascii - # characters, it will be flagged later by the idnits check. - try: - line += u" <%s>" % email - except UnicodeDecodeError: + def turn_into_unicode(s): + if s is None: + return u"" + + if isinstance(s, unicode): + return s + else: try: - line = line.decode('utf-8') - email = email.decode('utf-8') - line += u" <%s>" % email + return s.decode("utf-8") except UnicodeDecodeError: try: - line = line.decode('latin-1') - email = email.decode('latin-1') - line += u" <%s>" % email + return s.decode("latin-1") except UnicodeDecodeError: - try: - line += " <%s>" % email - except UnicodeDecodeError: - pass + return "" - authors.append(line) + name = turn_into_unicode(name) + email = turn_into_unicode(email) + company = turn_into_unicode(company) + + authors.append({ + "name": name, + "email": email, + "affiliation": company, + # FIXME: missing country + }) if form.abstract: abstract = form.abstract @@ -143,7 +144,7 @@ def upload_submission(request): submission.abstract = abstract submission.pages = form.parsed_draft.get_pagecount() submission.words = form.parsed_draft.get_wordcount() - submission.authors = "\n".join(authors) + submission.authors = authors submission.first_two_pages = ''.join(form.parsed_draft.pages[:2]) submission.file_size = file_size submission.file_types = ','.join(form.file_types) @@ -260,7 +261,7 @@ def submission_status(request, submission_id, access_token=None): doc = submission.existing_document() if doc and doc.group: old_authors = [ author.person for author in doc.documentauthor_set.all() ] - new_authors = [ get_person_from_name_email(**p) for p in submission.authors_parsed() ] + new_authors = [ get_person_from_name_email(author["name"], author.get("email")) for author in submission.authors ] group_authors_changed = set(old_authors)!=set(new_authors) message = None @@ -275,7 +276,7 @@ def submission_status(request, submission_id, access_token=None): message = ('success', 'The submission is pending approval by the authors of the previous version. An email has been sent to: %s' % ", ".join(confirmation_list)) - submitter_form = NameEmailForm(initial=submission.submitter_parsed(), prefix="submitter") + submitter_form = SubmitterForm(initial=submission.submitter_parsed(), prefix="submitter") replaces_form = ReplacesForm(name=submission.name,initial=DocAlias.objects.filter(name__in=submission.replaces.split(","))) if request.method == 'POST': @@ -284,7 +285,7 @@ def submission_status(request, submission_id, access_token=None): if not can_edit: return HttpResponseForbidden("You do not have permission to perform this action") - submitter_form = NameEmailForm(request.POST, prefix="submitter") + submitter_form = SubmitterForm(request.POST, prefix="submitter") replaces_form = ReplacesForm(request.POST, name=submission.name) validations = [submitter_form.is_valid(), replaces_form.is_valid()] if all(validations): @@ -432,7 +433,7 @@ def edit_submission(request, submission_id, access_token=None): # submission itself, one for the submitter, and a list of forms # for the authors - empty_author_form = NameEmailForm(email_required=False) + empty_author_form = AuthorForm() if request.method == 'POST': # get a backup submission now, the model form may change some @@ -440,9 +441,9 @@ def edit_submission(request, submission_id, access_token=None): prev_submission = Submission.objects.get(pk=submission.pk) edit_form = EditSubmissionForm(request.POST, instance=submission, prefix="edit") - submitter_form = NameEmailForm(request.POST, prefix="submitter") + submitter_form = SubmitterForm(request.POST, prefix="submitter") replaces_form = ReplacesForm(request.POST,name=submission.name) - author_forms = [ NameEmailForm(request.POST, email_required=False, prefix=prefix) + author_forms = [ AuthorForm(request.POST, prefix=prefix) for prefix in request.POST.getlist("authors-prefix") if prefix != "authors-" ] @@ -454,9 +455,9 @@ def edit_submission(request, submission_id, access_token=None): submission.submitter = submitter_form.cleaned_line() replaces = replaces_form.cleaned_data.get("replaces", []) submission.replaces = ",".join(o.name for o in replaces) - submission.authors = "\n".join(f.cleaned_line() for f in author_forms) - if hasattr(submission, '_cached_authors_parsed'): - del submission._cached_authors_parsed + submission.authors = [ { attr: f.cleaned_data.get(attr) or "" + for attr in ["name", "email", "affiliation", "country"] } + for f in author_forms ] edit_form.save(commit=False) # transfer changes if submission.rev != prev_submission.rev: @@ -491,10 +492,10 @@ def edit_submission(request, submission_id, access_token=None): form_errors = True else: edit_form = EditSubmissionForm(instance=submission, prefix="edit") - submitter_form = NameEmailForm(initial=submission.submitter_parsed(), prefix="submitter") + submitter_form = SubmitterForm(initial=submission.submitter_parsed(), prefix="submitter") replaces_form = ReplacesForm(name=submission.name,initial=DocAlias.objects.filter(name__in=submission.replaces.split(","))) - author_forms = [ NameEmailForm(initial=author, email_required=False, prefix="authors-%s" % i) - for i, author in enumerate(submission.authors_parsed()) ] + author_forms = [ AuthorForm(initial=author, prefix="authors-%s" % i) + for i, author in enumerate(submission.authors) ] return render(request, 'submit/edit_submission.html', {'selected': 'status', diff --git a/ietf/templates/submit/announce_to_lists.txt b/ietf/templates/submit/announce_to_lists.txt index 0d9817d57..30939a8d0 100644 --- a/ietf/templates/submit/announce_to_lists.txt +++ b/ietf/templates/submit/announce_to_lists.txt @@ -3,7 +3,7 @@ A New Internet-Draft is available from the on-line Internet-Drafts directories. {% if submission.group %}This draft is a work item of the {{ submission.group.name }}{% if group.type.name %} {{ group.type.name }}{% endif %} of the {% if group.type_id == "rg" %}IRTF{% else %}IETF{% endif %}.{% endif %} Title : {{ submission.title }} - Author{{ submission.authors_parsed|pluralize:" ,s" }} : {% for author in submission.authors_parsed %}{{ author.name }}{% if not forloop.last %} + Author{{ submission.authors|pluralize:" ,s" }} : {% for author in submission.authors %}{{ author.name }}{% if not forloop.last %} {% endif %}{% endfor %} Filename : {{ submission.name }}-{{ submission.rev }}.txt Pages : {{ submission.pages }} diff --git a/ietf/templates/submit/approval_request.txt b/ietf/templates/submit/approval_request.txt index 83677d166..828b14bc4 100644 --- a/ietf/templates/submit/approval_request.txt +++ b/ietf/templates/submit/approval_request.txt @@ -22,7 +22,7 @@ To approve the draft, go to this URL (note: you need to login to be able to appr Authors: -{% for author in submission.authors_parsed %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} +{% for author in submission.authors %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} {% endfor %} {% endautoescape %} diff --git a/ietf/templates/submit/manual_post_request.txt b/ietf/templates/submit/manual_post_request.txt index 1620a476f..8ff38a6fa 100644 --- a/ietf/templates/submit/manual_post_request.txt +++ b/ietf/templates/submit/manual_post_request.txt @@ -22,7 +22,7 @@ I-D Submission Tool URL: Authors: -{% for author in submission.authors_parsed %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} +{% for author in submission.authors %} {{ author.name }}{% if author.email %} <{{ author.email }}>{% endif%} {% endfor %} Comment to the secretariat: diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index 4f67c29de..3bf65c096 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -2,7 +2,7 @@ {# Copyright The IETF Trust 2015, All Rights Reserved #} {% load origin %} {% load staticfiles %} -{% load ietf_filters submit_tags %} +{% load ietf_filters submit_tags country %} {% block title %}Submission status of {{ submission.name }}-{{ submission.rev }}{% endblock %} @@ -195,17 +195,21 @@ Authors - {% with submission.authors_parsed as authors %} + {% with submission.authors as authors %} {{ authors|length }} author{{ authors|pluralize }} {% endwith %} {% if errors.authors %}

{{ errors.authors|safe }}

{% endif %} - {% for author in submission.authors_parsed %} + {% for author in submission.authors %} Author {{ forloop.counter }} - {{ author.name }} {% if author.email %}<{{ author.email }}>{% endif %} + + {{ author.name }} {% if author.email %}<{{ author.email }}>{% endif %} + {% if author.affiliation %}- {{ author.affiliation }}{% endif %} + {% if author.country %}- {{ author.country|country_name }}{% endif %} + {% endfor %} diff --git a/ietf/templates/submit/submitter_form.html b/ietf/templates/submit/submitter_form.html index 93994c10c..1564dd3e5 100644 --- a/ietf/templates/submit/submitter_form.html +++ b/ietf/templates/submit/submitter_form.html @@ -11,8 +11,8 @@ {% load ietf_filters %} {% buttons %} - {% for author in submission.authors_parsed %} - + {% for author in submission.authors %} + {% endfor %} {% endbuttons %} diff --git a/ietf/utils/templatetags/country.py b/ietf/utils/templatetags/country.py new file mode 100644 index 000000000..7d730d2f9 --- /dev/null +++ b/ietf/utils/templatetags/country.py @@ -0,0 +1,14 @@ +from django.template.base import Library +from django.template.defaultfilters import stringfilter + +from django_countries import countries + +register = Library() + +@register.filter(is_safe=True) +@stringfilter +def country_name(value): + """ + Converts country code to country name + """ + return dict(countries).get(value, "") From 8fdb0047e10b2becfe36fcbdbdca6872b0c3b4a0 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 27 Jan 2017 16:12:10 +0000 Subject: [PATCH 19/49] Add migration for converting existing Submission.authors to JSON - Legacy-Id: 12746 --- .../migrations/0019_auto_20170127_0538.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 ietf/submit/migrations/0019_auto_20170127_0538.py diff --git a/ietf/submit/migrations/0019_auto_20170127_0538.py b/ietf/submit/migrations/0019_auto_20170127_0538.py new file mode 100644 index 000000000..972edc41e --- /dev/null +++ b/ietf/submit/migrations/0019_auto_20170127_0538.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations +import jsonfield.fields + +def parse_email_line(line): + """Split line on the form 'Some Name '""" + import re + m = re.match("([^<]+) <([^>]+)>$", line) + if m: + return dict(name=m.group(1), email=m.group(2)) + else: + return dict(name=line, email="") + +def parse_authors(author_lines): + res = [] + for line in author_lines.replace("\r", "").split("\n"): + line = line.strip() + if line: + res.append(parse_email_line(line)) + return res + +def convert_author_lines_to_json(apps, schema_editor): + import json + + Submission = apps.get_model("submit", "Submission") + for s in Submission.objects.all().iterator(): + Submission.objects.filter(pk=s.pk).update(authors=json.dumps(parse_authors(s.authors))) + +class Migration(migrations.Migration): + + dependencies = [ + ('submit', '0018_auto_20170116_0927'), + ] + + operations = [ + migrations.RunPython(convert_author_lines_to_json, migrations.RunPython.noop), + migrations.AlterField( + model_name='submission', + name='authors', + field=jsonfield.fields.JSONField(default=list, help_text=b'List of authors with name, email, affiliation and country code.'), + ), + ] From 3395d6f445c074743a0405b5c07fe9d89d05d5bd Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 30 Jan 2017 13:33:25 +0000 Subject: [PATCH 20/49] Add simple back-filling of affiliation on authors to backfilling script - Legacy-Id: 12750 --- ietf/stats/backfill_data.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py index bdcf00193..c62edfd30 100644 --- a/ietf/stats/backfill_data.py +++ b/ietf/stats/backfill_data.py @@ -21,6 +21,7 @@ parser = argparse.ArgumentParser() parser.add_argument("--document", help="specific document name") parser.add_argument("--words", action="store_true", help="fill in word count") parser.add_argument("--formlang", action="store_true", help="fill in formal languages") +parser.add_argument("--authors", action="store_true", help="fill in author info") args = parser.parse_args() formal_language_dict = { l.pk: l for l in FormalLanguageName.objects.all() } @@ -31,7 +32,7 @@ docs_qs = Document.objects.filter(type="draft") if args.document: docs_qs = docs_qs.filter(docalias__name=args.document) -for doc in docs_qs.prefetch_related("docalias_set", "formal_languages"): +for doc in docs_qs.prefetch_related("docalias_set", "formal_languages", "documentauthor_set", "documentauthor_set__person", "documentauthor_set__person__alias_set"): canonical_name = doc.name for n in doc.docalias_set.all(): if n.name.startswith("rfc"): @@ -72,6 +73,35 @@ for doc in docs_qs.prefetch_related("docalias_set", "formal_languages"): doc.formal_languages.remove(l) updated = True + if args.authors: + old_authors = doc.documentauthor_set.all() + old_authors_by_name = {} + old_authors_by_email = {} + for author in old_authors: + for alias in author.person.alias_set.all(): + old_authors_by_name[alias.name] = author + + if author.email_id: + old_authors_by_email[author.email_id] = author + + for full, _, _, _, _, email, company in d.get_author_list(): + old_author = None + if email: + old_author = old_authors_by_email.get(email) + if not old_author: + old_author = old_authors_by_name.get(full) + + if not old_author: + print "UNKNOWN AUTHOR", doc.name, full, email, company + continue + + if old_author.affiliation != company: + print "new affiliation", old_author.affiliation, company + old_author.affiliation = company + old_author.save(update_fields=["affiliation"]) + updated = True + + if updates: Document.objects.filter(pk=doc.pk).update(**updates) updated = True From ce2180971aaae8feb8bfebfc78468f9f37c09719 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 30 Jan 2017 13:34:17 +0000 Subject: [PATCH 21/49] Revamp choice parsing a bit in statistics section, add time choice to documents to be able to choose between all time and recent drafts - Legacy-Id: 12751 --- ietf/stats/urls.py | 2 +- ietf/stats/views.py | 146 +++++++++++++---------- ietf/templates/stats/document_stats.html | 9 ++ 3 files changed, 93 insertions(+), 64 deletions(-) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 6616d8fdd..a700a5a3a 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -5,6 +5,6 @@ import ietf.stats.views urlpatterns = patterns('', url("^$", ietf.stats.views.stats_index), - url("^document/(?:(?Pauthors|pages|words|format|formlang)/)?(?:(?Pall|rfc|draft)/)?$", ietf.stats.views.document_stats), + url("^document/(?:(?Pauthors|pages|words|format|formlang)/)?$", ietf.stats.views.document_stats), url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, ietf.stats.views.review_stats), ) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 047b90c66..de9853f00 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -24,7 +24,7 @@ from ietf.submit.models import Submission from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewRequestStateName, ReviewResultName -from ietf.doc.models import DocAlias +from ietf.doc.models import DocAlias, Document from ietf.ietfauth.utils import has_role def stats_index(request): @@ -54,49 +54,75 @@ def generate_query_string(query_dict, overrides): return query_part -def document_stats(request, stats_type=None, document_type=None): - def build_document_stats_url(stats_type_override=Ellipsis, document_type_override=Ellipsis, get_overrides={}): +def get_choice(request, get_parameter, possible_choices, multiple=False): + # the statistics are built with links to make navigation faster, + # so we don't really have a form in most cases, so just use this + # helper instead to select between the choices + values = request.GET.getlist(get_parameter) + found = [t[0] for t in possible_choices if t[0] in values] + + if multiple: + return found + else: + if found: + return found[0] + else: + return None + +def add_url_to_choices(choices, url_builder): + return [ (slug, label, url_builder(slug)) for slug, label in choices] + +def put_into_bin(value, bin_size): + if value is None: + return (value, value) + + v = (value // bin_size) * bin_size + return (v, "{} - {}".format(v, v + bin_size - 1)) + +def document_stats(request, stats_type=None): + def build_document_stats_url(stats_type_override=Ellipsis, get_overrides={}): kwargs = { "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, - "document_type": document_type if document_type_override is Ellipsis else document_type_override, } return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) # statistics type - one of the tables or the chart - possible_stats_types = [ + possible_stats_types = add_url_to_choices([ ("authors", "Authors"), ("pages", "Pages"), ("words", "Words"), ("format", "Format"), ("formlang", "Formal languages"), - ] - - possible_stats_types = [ (slug, label, build_document_stats_url(stats_type_override=slug)) - for slug, label in possible_stats_types ] + ], lambda slug: build_document_stats_url(stats_type_override=slug)) if not stats_type: return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_stats_types[0][0])) - possible_document_types = [ - ("all", "All"), + + possible_document_types = add_url_to_choices([ + ("", "All"), ("rfc", "RFCs"), ("draft", "Drafts"), - ] + ], lambda slug: build_document_stats_url(get_overrides={ "type": slug })) - possible_document_types = [ (slug, label, build_document_stats_url(document_type_override=slug)) - for slug, label in possible_document_types ] + document_type = get_choice(request, "type", possible_document_types) or "" - if not document_type: - return HttpResponseRedirect(build_document_stats_url(document_type_override=possible_document_types[0][0])) - - def put_into_bin(value, bin_size): - if value is None: - return (value, value) + possible_time_choices = add_url_to_choices([ + ("", "All time"), + ("5y", "Past 5 years"), + ], lambda slug: build_document_stats_url(get_overrides={ "time": slug })) - v = (value // bin_size) * bin_size - return (v, "{} - {}".format(v, v + bin_size - 1)) + time_choice = request.GET.get("time") or "" + + from_time = None + if "y" in time_choice: + try: + years = int(time_choice.rstrip("y")) + from_time = datetime.datetime.today() - dateutil.relativedelta.relativedelta(years=years) + except ValueError: + pass def generate_canonical_names(docalias_qs): for doc_id, ts in itertools.groupby(docalias_qs.order_by("document"), lambda t: t[0]): @@ -120,15 +146,26 @@ def document_stats(request, stats_type=None, document_type=None): elif document_type == "draft": docalias_qs = docalias_qs.exclude(document__states__type="draft", document__states__slug="rfc") + if from_time: + # this is actually faster than joining in the database, + # despite the round-trip back and forth + docs_within_time_constraint = list(Document.objects.filter( + type="draft", + docevent__time__gte=from_time, + docevent__type__in=["published_rfc", "new_revision"], + ).values_list("pk")) + + docalias_qs = docalias_qs.filter(document__in=docs_within_time_constraint) + chart_data = [] table_data = [] - if document_type == "all": - doc_label = "document" - elif document_type == "rfc": + if document_type == "rfc": doc_label = "RFC" elif document_type == "draft": doc_label = "draft" + else: + doc_label = "document" stats_title = "" bin_size = 1 @@ -280,6 +317,8 @@ def document_stats(request, stats_type=None, document_type=None): "stats_type": stats_type, "possible_document_types": possible_document_types, "document_type": document_type, + "possible_time_choices": possible_time_choices, + "time_choice": time_choice, "doc_label": doc_label, "bin_size": bin_size, "content_template": "stats/document_stats_{}.html".format(stats_type), @@ -306,18 +345,6 @@ def review_stats(request, stats_type=None, acronym=None): return urlreverse(review_stats, kwargs=kwargs) + generate_query_string(request.GET, get_overrides) - def get_choice(get_parameter, possible_choices, multiple=False): - values = request.GET.getlist(get_parameter) - found = [t[0] for t in possible_choices if t[0] in values] - - if multiple: - return found - else: - if found: - return found[0] - else: - return None - # which overview - team or reviewer if acronym: level = "reviewer" @@ -334,21 +361,19 @@ def review_stats(request, stats_type=None, acronym=None): if level == "team": possible_stats_types.append(("time", "Changes over time")) - possible_stats_types = [ (slug, label, build_review_stats_url(stats_type_override=slug)) - for slug, label in possible_stats_types ] + possible_stats_types = add_url_to_choices(possible_stats_types, + lambda slug: build_review_stats_url(stats_type_override=slug)) if not stats_type: return HttpResponseRedirect(build_review_stats_url(stats_type_override=possible_stats_types[0][0])) # what to count - possible_count_choices = [ + possible_count_choices = add_url_to_choices([ ("", "Review requests"), ("pages", "Reviewed pages"), - ] + ], lambda slug: build_review_stats_url(get_overrides={ "count": slug })) - possible_count_choices = [ (slug, label, build_review_stats_url(get_overrides={ "count": slug })) for slug, label in possible_count_choices ] - - count = get_choice("count", possible_count_choices) or "" + count = get_choice(request, "count", possible_count_choices) or "" # time range def parse_date(s): @@ -433,7 +458,7 @@ def review_stats(request, stats_type=None, acronym=None): if stats_type == "time": possible_teams = [(t.acronym, t.acronym) for t in teams] - selected_teams = get_choice("team", possible_teams, multiple=True) + selected_teams = get_choice(request, "team", possible_teams, multiple=True) def add_if_exists_else_subtract(element, l): if element in l: @@ -475,33 +500,28 @@ def review_stats(request, stats_type=None, acronym=None): # choice - possible_completion_types = [ + possible_completion_types = add_url_to_choices([ ("completed_in_time", "Completed in time"), ("completed_late", "Completed late"), ("not_completed", "Not completed"), ("average_assignment_to_closure_days", "Avg. compl. days"), - ] + ], lambda slug: build_review_stats_url(get_overrides={ "completion": slug, "result": None, "state": None })) - possible_completion_types = [ - (slug, label, build_review_stats_url(get_overrides={ "completion": slug, "result": None, "state": None })) - for slug, label in possible_completion_types - ] + selected_completion_type = get_choice(request, "completion", possible_completion_types) - selected_completion_type = get_choice("completion", possible_completion_types) + possible_results = add_url_to_choices( + [(r.slug, r.name) for r in results], + lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": slug, "state": None }) + ) - possible_results = [ - (r.slug, r.name, build_review_stats_url(get_overrides={ "completion": None, "result": r.slug, "state": None })) - for r in results - ] - - selected_result = get_choice("result", possible_results) + selected_result = get_choice(request, "result", possible_results) - possible_states = [ - (s.slug, s.name, build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": s.slug })) - for s in states - ] + possible_states = add_url_to_choices( + [(s.slug, s.name) for s in states], + build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": slug }) + ) - selected_state = get_choice("state", possible_states) + selected_state = get_choice(request, "state", possible_states) if not selected_completion_type and not selected_result and not selected_state: selected_completion_type = "completed_in_time" diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html index 532329203..bac17e587 100644 --- a/ietf/templates/stats/document_stats.html +++ b/ietf/templates/stats/document_stats.html @@ -33,6 +33,15 @@ {% endfor %} + +
+ Time: +
+ {% for slug, label, url in possible_time_choices %} + {{ label }} + {% endfor %} +
+
{% include content_template %} From 631295f7db519e8378fa2ebf1c4b8f567fedb853 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 30 Jan 2017 14:00:05 +0000 Subject: [PATCH 22/49] Fix a couple of bugs - Legacy-Id: 12752 --- ietf/stats/views.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index de9853f00..ace78f5cf 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -89,7 +89,7 @@ def document_stats(request, stats_type=None): # statistics type - one of the tables or the chart possible_stats_types = add_url_to_choices([ - ("authors", "Authors"), + ("authors", "Number of authors"), ("pages", "Pages"), ("words", "Words"), ("format", "Format"), @@ -466,9 +466,12 @@ def review_stats(request, stats_type=None, acronym=None): else: return l + [element] - possible_teams = [(slug, label, build_review_stats_url(get_overrides={ - "team": add_if_exists_else_subtract(slug, selected_teams) - })) for slug, label in possible_teams] + possible_teams = add_url_to_choices( + possible_teams, + lambda slug: build_review_stats_url(get_overrides={ + "team": add_if_exists_else_subtract(slug, selected_teams) + }) + ) query_teams = [t for t in query_teams if t.acronym in selected_teams] extracted_data = extract_review_request_data(query_teams, query_reviewers, from_time, to_time) @@ -518,7 +521,7 @@ def review_stats(request, stats_type=None, acronym=None): possible_states = add_url_to_choices( [(s.slug, s.name) for s in states], - build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": slug }) + lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": slug }) ) selected_state = get_choice(request, "state", possible_states) From b645a8c0f9fcd3bab190defd4f974defeff46bc1 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 30 Jan 2017 14:01:40 +0000 Subject: [PATCH 23/49] Expand document stats test - Legacy-Id: 12753 --- ietf/stats/tests.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 54017208e..7bb1ab230 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -24,21 +24,19 @@ class StatisticsTests(TestCase): self.assertEqual(r.status_code, 302) self.assertTrue(authors_url in r["Location"]) - authors_all_url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": "authors", "document_type": "all" }) - - r = self.client.get(authors_url) - self.assertEqual(r.status_code, 302) - self.assertTrue(authors_all_url in r["Location"]) - # check various stats types - for stats_type in ["authors", "pages", "words", "format"]: - for document_type in ["all", "rfc", "draft"]: - url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type, "document_type": document_type }) - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertTrue(q('#chart')) - self.assertTrue(q('table.stats-data')) + for stats_type in ["authors", "pages", "words", "format", "formlang"]: + for document_type in ["", "rfc", "draft"]: + for time_choice in ["", "5y"]: + url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type }) + r = self.client.get(url, { + "type": document_type, + "time": time_choice, + }) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(q('#chart')) + self.assertTrue(q('table.stats-data')) def test_review_stats(self): doc = make_test_data() From a9525ab4f46943ec0b1df3f96ecee54721dbda96 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Tue, 31 Jan 2017 16:32:20 +0000 Subject: [PATCH 24/49] Revamp stats selector UI a bit to accomodate statistics counting authors (instead of documents), add stats with documents per author - Legacy-Id: 12766 --- ietf/static/ietf/js/document-stats.js | 22 +- ietf/stats/tests.py | 2 +- ietf/stats/urls.py | 2 +- ietf/stats/views.py | 355 +++++++++++------- ietf/templates/stats/document_stats.html | 23 +- .../document_stats_author_documents.html | 65 ++++ .../stats/document_stats_authors.html | 4 +- .../stats/document_stats_format.html | 2 +- .../stats/document_stats_formlang.html | 2 +- .../templates/stats/document_stats_pages.html | 2 +- .../templates/stats/document_stats_words.html | 2 +- .../stats/includes/docnames_cell.html | 1 - .../includes/number_with_details_cell.html | 1 + ietf/templates/stats/index.html | 6 +- 14 files changed, 318 insertions(+), 171 deletions(-) create mode 100644 ietf/templates/stats/document_stats_author_documents.html delete mode 100644 ietf/templates/stats/includes/docnames_cell.html create mode 100644 ietf/templates/stats/includes/number_with_details_cell.html diff --git a/ietf/static/ietf/js/document-stats.js b/ietf/static/ietf/js/document-stats.js index 922a7205e..7d97d868e 100644 --- a/ietf/static/ietf/js/document-stats.js +++ b/ietf/static/ietf/js/document-stats.js @@ -15,20 +15,26 @@ $(document).ready(function () { var chart = Highcharts.chart('chart', window.chartConf); } - $(".popover-docnames").each(function () { + $(".popover-details").each(function () { var stdNameRegExp = new RegExp("^(rfc|bcp|fyi|std)[0-9]+$", 'i'); + var draftRegExp = new RegExp("^draft-", 'i'); - var html = []; - $.each(($(this).data("docnames") || "").split(" "), function (i, docname) { - if (!$.trim(docname)) + var html = [];t + $.each(($(this).data("elements") || "").split("|"), function (i, element) { + if (!$.trim(element)) return; - var displayName = docname; + if (draftRegExp.test(element) || stdNameRegExp.test(element)) { + var displayName = element; - if (stdNameRegExp.test(docname)) - displayName = docname.slice(0, 3).toUpperCase() + " " + docname.slice(3); + if (stdNameRegExp.test(element)) + displayName = element.slice(0, 3).toUpperCase() + " " + element.slice(3); - html.push('
' + displayName + '
'); + html.push('
' + displayName + '
'); + } + else { + html.push('
' + element + '
'); + } }); if ($(this).data("sliced")) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 7bb1ab230..fa973544c 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -25,7 +25,7 @@ class StatisticsTests(TestCase): self.assertTrue(authors_url in r["Location"]) # check various stats types - for stats_type in ["authors", "pages", "words", "format", "formlang"]: + for stats_type in ["authors", "pages", "words", "format", "formlang", "author/documents"]: for document_type in ["", "rfc", "draft"]: for time_choice in ["", "5y"]: url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type }) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index a700a5a3a..ee4fb4018 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -5,6 +5,6 @@ import ietf.stats.views urlpatterns = patterns('', url("^$", ietf.stats.views.stats_index), - url("^document/(?:(?Pauthors|pages|words|format|formlang)/)?$", ietf.stats.views.document_stats), + url("^document/(?:(?Pauthors|pages|words|format|formlang|author/documents|author/affiliation|author/country|author/continent|author/citation)/)?$", ietf.stats.views.document_stats), url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, ietf.stats.views.review_stats), ) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index ace78f5cf..c151888b5 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -9,7 +9,7 @@ from django.shortcuts import render from django.contrib.auth.decorators import login_required from django.core.urlresolvers import reverse as urlreverse from django.http import HttpResponseRedirect, HttpResponseForbidden -from django.db.models import Count +from django.db.models import Count, Q from django.utils.safestring import mark_safe from django.conf import settings @@ -88,7 +88,7 @@ def document_stats(request, stats_type=None): return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) # statistics type - one of the tables or the chart - possible_stats_types = add_url_to_choices([ + possible_document_stats_types = add_url_to_choices([ ("authors", "Number of authors"), ("pages", "Pages"), ("words", "Words"), @@ -96,8 +96,18 @@ def document_stats(request, stats_type=None): ("formlang", "Formal languages"), ], lambda slug: build_document_stats_url(stats_type_override=slug)) + # statistics type - one of the tables or the chart + possible_author_stats_types = add_url_to_choices([ + ("author/documents", "Number of documents"), + ("author/affiliation", "Affiliation"), + ("author/country", "Country"), + ("author/continent", "Continent"), + ("author/citation", "Citations"), + ], lambda slug: build_document_stats_url(stats_type_override=slug)) + + if not stats_type: - return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_stats_types[0][0])) + return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_document_stats_types[0][0])) possible_document_types = add_url_to_choices([ @@ -124,196 +134,248 @@ def document_stats(request, stats_type=None): except ValueError: pass - def generate_canonical_names(docalias_qs): - for doc_id, ts in itertools.groupby(docalias_qs.order_by("document"), lambda t: t[0]): - chosen = None - for t in ts: - if chosen is None: - chosen = t - else: - if t[0].startswith("rfc"): - chosen = t - elif t[0].startswith("draft") and not chosen[0].startswith("rfc"): - chosen = t - - yield chosen - - # filter documents - docalias_qs = DocAlias.objects.filter(document__type="draft") - - if document_type == "rfc": - docalias_qs = docalias_qs.filter(document__states__type="draft", document__states__slug="rfc") - elif document_type == "draft": - docalias_qs = docalias_qs.exclude(document__states__type="draft", document__states__slug="rfc") - - if from_time: - # this is actually faster than joining in the database, - # despite the round-trip back and forth - docs_within_time_constraint = list(Document.objects.filter( - type="draft", - docevent__time__gte=from_time, - docevent__type__in=["published_rfc", "new_revision"], - ).values_list("pk")) - - docalias_qs = docalias_qs.filter(document__in=docs_within_time_constraint) - chart_data = [] table_data = [] - - if document_type == "rfc": - doc_label = "RFC" - elif document_type == "draft": - doc_label = "draft" - else: - doc_label = "document" - stats_title = "" bin_size = 1 - total_docs = docalias_qs.count() - if stats_type == "authors": - stats_title = "Number of authors for each {}".format(doc_label) + if any(stats_type == t[0] for t in possible_document_stats_types): + # filter documents + docalias_qs = DocAlias.objects.filter(document__type="draft") - bins = defaultdict(list) + if document_type == "rfc": + docalias_qs = docalias_qs.filter(document__states__type="draft", document__states__slug="rfc") + elif document_type == "draft": + docalias_qs = docalias_qs.exclude(document__states__type="draft", document__states__slug="rfc") - for name, author_count in generate_canonical_names(docalias_qs.values_list("name").annotate(Count("document__documentauthor"))): - bins[author_count].append(name) + if from_time: + # this is actually faster than joining in the database, + # despite the round-trip back and forth + docs_within_time_constraint = list(Document.objects.filter( + type="draft", + docevent__time__gte=from_time, + docevent__type__in=["published_rfc", "new_revision"], + ).values_list("pk")) - series_data = [] - for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / total_docs - series_data.append((author_count, percentage)) - table_data.append((author_count, percentage, names)) + docalias_qs = docalias_qs.filter(document__in=docs_within_time_constraint) - chart_data.append({ - "data": series_data, - "animation": False, - }) + if document_type == "rfc": + doc_label = "RFC" + elif document_type == "draft": + doc_label = "draft" + else: + doc_label = "document" - elif stats_type == "pages": - stats_title = "Number of pages for each {}".format(doc_label) + total_docs = docalias_qs.count() - bins = defaultdict(list) + def generate_canonical_names(docalias_qs): + for doc_id, ts in itertools.groupby(docalias_qs.order_by("document"), lambda t: t[0]): + chosen = None + for t in ts: + if chosen is None: + chosen = t + else: + if t[0].startswith("rfc"): + chosen = t + elif t[0].startswith("draft") and not chosen[0].startswith("rfc"): + chosen = t - for name, pages in generate_canonical_names(docalias_qs.values_list("name", "document__pages")): - bins[pages].append(name) + yield chosen - series_data = [] - for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / total_docs - if pages is not None: - series_data.append((pages, len(names))) - table_data.append((pages, percentage, names)) + if stats_type == "authors": + stats_title = "Number of authors for each {}".format(doc_label) - chart_data.append({ - "data": series_data, - "animation": False, - }) + bins = defaultdict(list) - elif stats_type == "words": - stats_title = "Number of words for each {}".format(doc_label) + for name, author_count in generate_canonical_names(docalias_qs.values_list("name").annotate(Count("document__documentauthor"))): + bins[author_count].append(name) - bin_size = 500 + series_data = [] + for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / total_docs + series_data.append((author_count, percentage)) + table_data.append((author_count, percentage, names)) - bins = defaultdict(list) + chart_data.append({ + "data": series_data, + "animation": False, + }) - for name, words in generate_canonical_names(docalias_qs.values_list("name", "document__words")): - bins[put_into_bin(words, bin_size)].append(name) + elif stats_type == "pages": + stats_title = "Number of pages for each {}".format(doc_label) - series_data = [] - for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]): - percentage = len(names) * 100.0 / total_docs - if words is not None: - series_data.append((value, len(names))) + bins = defaultdict(list) - table_data.append((words, percentage, names)) + for name, pages in generate_canonical_names(docalias_qs.values_list("name", "document__pages")): + bins[pages].append(name) - chart_data.append({ - "data": series_data, - "animation": False, - }) + series_data = [] + for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / total_docs + if pages is not None: + series_data.append((pages, len(names))) + table_data.append((pages, percentage, names)) - elif stats_type == "format": - stats_title = "Submission formats for each {}".format(doc_label) + chart_data.append({ + "data": series_data, + "animation": False, + }) - bins = defaultdict(list) + elif stats_type == "words": + stats_title = "Number of words for each {}".format(doc_label) - # on new documents, we should have a Submission row with the file types - submission_types = {} + bin_size = 500 - for doc_name, file_types in Submission.objects.values_list("draft", "file_types").order_by("submission_date", "id"): - submission_types[doc_name] = file_types + bins = defaultdict(list) - doc_names_with_missing_types = {} - for canonical_name, rev, doc_name in generate_canonical_names(docalias_qs.values_list("name", "document__rev", "document__name")): - types = submission_types.get(doc_name) - if types: - for dot_ext in types.split(","): - bins[dot_ext.lstrip(".").upper()].append(canonical_name) + for name, words in generate_canonical_names(docalias_qs.values_list("name", "document__words")): + bins[put_into_bin(words, bin_size)].append(name) - else: + series_data = [] + for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]): + percentage = len(names) * 100.0 / total_docs + if words is not None: + series_data.append((value, len(names))) + + table_data.append((words, percentage, names)) + + chart_data.append({ + "data": series_data, + "animation": False, + }) + + elif stats_type == "format": + stats_title = "Submission formats for each {}".format(doc_label) + + bins = defaultdict(list) + + # on new documents, we should have a Submission row with the file types + submission_types = {} + + for doc_name, file_types in Submission.objects.values_list("draft", "file_types").order_by("submission_date", "id"): + submission_types[doc_name] = file_types + + doc_names_with_missing_types = {} + for canonical_name, rev, doc_name in generate_canonical_names(docalias_qs.values_list("name", "document__rev", "document__name")): + types = submission_types.get(doc_name) + if types: + for dot_ext in types.split(","): + bins[dot_ext.lstrip(".").upper()].append(canonical_name) - if canonical_name.startswith("rfc"): - filename = canonical_name else: - filename = canonical_name + "-" + rev - doc_names_with_missing_types[filename] = canonical_name + if canonical_name.startswith("rfc"): + filename = canonical_name + else: + filename = canonical_name + "-" + rev - # look up the remaining documents on disk - for filename in itertools.chain(os.listdir(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR), os.listdir(settings.RFC_PATH)): - t = filename.split(".", 1) - if len(t) != 2: - continue + doc_names_with_missing_types[filename] = canonical_name - basename, ext = t - if any(ext.lower().endswith(blacklisted_ext.lower()) for blacklisted_ext in settings.DOCUMENT_FORMAT_BLACKLIST): - continue + # look up the remaining documents on disk + for filename in itertools.chain(os.listdir(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR), os.listdir(settings.RFC_PATH)): + t = filename.split(".", 1) + if len(t) != 2: + continue - canonical_name = doc_names_with_missing_types.get(basename) + basename, ext = t + if any(ext.lower().endswith(blacklisted_ext.lower()) for blacklisted_ext in settings.DOCUMENT_FORMAT_BLACKLIST): + continue - if canonical_name: - bins[ext.upper()].append(canonical_name) + canonical_name = doc_names_with_missing_types.get(basename) - series_data = [] - for fmt, names in sorted(bins.iteritems(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / total_docs - series_data.append((fmt, len(names))) + if canonical_name: + bins[ext.upper()].append(canonical_name) - table_data.append((fmt, percentage, names)) + series_data = [] + for fmt, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / total_docs + series_data.append((fmt, len(names))) - chart_data.append({ - "data": series_data, - "animation": False, - }) + table_data.append((fmt, percentage, names)) - elif stats_type == "formlang": - stats_title = "Formal languages used for each {}".format(doc_label) + chart_data.append({ + "data": series_data, + "animation": False, + }) - bins = defaultdict(list) + elif stats_type == "formlang": + stats_title = "Formal languages used for each {}".format(doc_label) - for name, formal_language_name in generate_canonical_names(docalias_qs.values_list("name", "document__formal_languages__name")): - bins[formal_language_name].append(name) + bins = defaultdict(list) - series_data = [] - for formal_language, names in sorted(bins.iteritems(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / total_docs - if formal_language is not None: - series_data.append((formal_language, len(names))) - table_data.append((formal_language, percentage, names)) + for name, formal_language_name in generate_canonical_names(docalias_qs.values_list("name", "document__formal_languages__name")): + bins[formal_language_name].append(name) + + series_data = [] + for formal_language, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / total_docs + if formal_language is not None: + series_data.append((formal_language, len(names))) + table_data.append((formal_language, percentage, names)) + + chart_data.append({ + "data": series_data, + "animation": False, + }) + + elif any(stats_type == t[0] for t in possible_author_stats_types): + person_filters = Q(documentauthor__document__type="draft") + + # filter persons + if document_type == "rfc": + person_filters &= Q(documentauthor__document__states__type="draft", documentauthor__document__states__slug="rfc") + elif document_type == "draft": + person_filters &= ~Q(documentauthor__document__states__type="draft", documentauthor__document__states__slug="rfc") + + if from_time: + # this is actually faster than joining in the database, + # despite the round-trip back and forth + docs_within_time_constraint = list(Document.objects.filter( + type="draft", + docevent__time__gte=from_time, + docevent__type__in=["published_rfc", "new_revision"], + ).values_list("pk")) + + person_filters &= Q(documentauthor__document__in=docs_within_time_constraint) + + person_qs = Person.objects.filter(person_filters) + + if document_type == "rfc": + doc_label = "RFC" + elif document_type == "draft": + doc_label = "draft" + else: + doc_label = "document" + + total_persons = person_qs.count() + + if stats_type == "author/documents": + stats_title = "Number of {}s for each author".format(doc_label) + + bins = defaultdict(list) + + for name, document_count in person_qs.values_list("name").annotate(Count("documentauthor")): + bins[document_count].append(name) + + series_data = [] + for document_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / total_persons + series_data.append((document_count, percentage)) + table_data.append((document_count, percentage, names)) + + chart_data.append({ + "data": series_data, + "animation": False, + }) - chart_data.append({ - "data": series_data, - "animation": False, - }) - return render(request, "stats/document_stats.html", { "chart_data": mark_safe(json.dumps(chart_data)), "table_data": table_data, "stats_title": stats_title, - "possible_stats_types": possible_stats_types, + "possible_document_stats_types": possible_document_stats_types, + "possible_author_stats_types": possible_author_stats_types, "stats_type": stats_type, "possible_document_types": possible_document_types, "document_type": document_type, @@ -321,9 +383,10 @@ def document_stats(request, stats_type=None): "time_choice": time_choice, "doc_label": doc_label, "bin_size": bin_size, - "content_template": "stats/document_stats_{}.html".format(stats_type), + "content_template": "stats/document_stats_{}.html".format(stats_type.replace("/", "_")), }) + @login_required def review_stats(request, stats_type=None, acronym=None): # This view is a bit complex because we want to show a bunch of diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html index bac17e587..2e4bda401 100644 --- a/ietf/templates/stats/document_stats.html +++ b/ietf/templates/stats/document_stats.html @@ -13,28 +13,39 @@ {% block content %} {% origin %} -

Document statistics

+

Draft/RFC statistics

- Show: + Documents: +
- {% for slug, label, url in possible_stats_types %} + {% for slug, label, url in possible_document_stats_types %} {{ label }} {% endfor %}
- Document type: + Authors: +
- {% for slug, label, url in possible_document_types %} - {{ label }} + {% for slug, label, url in possible_author_stats_types %} + {{ label }} {% endfor %}
+
Options
+
+ Document type: +
+ {% for slug, label, url in possible_document_types %} + {{ label }} + {% endfor %} +
+ Time:
{% for slug, label, url in possible_time_choices %} diff --git a/ietf/templates/stats/document_stats_author_documents.html b/ietf/templates/stats/document_stats_author_documents.html new file mode 100644 index 000000000..0d21b41d2 --- /dev/null +++ b/ietf/templates/stats/document_stats_author_documents.html @@ -0,0 +1,65 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for document_count, percentage, names in table_data %} + + + + + + {% endfor %} + +
DocumentsPercentage of authorsAuthors
{{ document_count }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
diff --git a/ietf/templates/stats/document_stats_authors.html b/ietf/templates/stats/document_stats_authors.html index 143da1114..70fe249fb 100644 --- a/ietf/templates/stats/document_stats_authors.html +++ b/ietf/templates/stats/document_stats_authors.html @@ -49,7 +49,7 @@ Authors Percentage of {{ doc_label }}s - {{ doc_label }}s + {{ doc_label|capfirst }}s @@ -57,7 +57,7 @@ {{ author_count }} {{ percentage|floatformat:2 }}% - {% include "stats/includes/docnames_cell.html" %} + {% include "stats/includes/number_with_details_cell.html" %} {% endfor %} diff --git a/ietf/templates/stats/document_stats_format.html b/ietf/templates/stats/document_stats_format.html index 7e701343f..ce1512a09 100644 --- a/ietf/templates/stats/document_stats_format.html +++ b/ietf/templates/stats/document_stats_format.html @@ -53,7 +53,7 @@ {{ pages }} {{ percentage|floatformat:2 }}% - {% include "stats/includes/docnames_cell.html" %} + {% include "stats/includes/number_with_details_cell.html" %} {% endfor %} diff --git a/ietf/templates/stats/document_stats_formlang.html b/ietf/templates/stats/document_stats_formlang.html index 248a45b82..e4b586d95 100644 --- a/ietf/templates/stats/document_stats_formlang.html +++ b/ietf/templates/stats/document_stats_formlang.html @@ -53,7 +53,7 @@ {{ formal_language }} {{ percentage|floatformat:2 }}% - {% include "stats/includes/docnames_cell.html" %} + {% include "stats/includes/number_with_details_cell.html" %} {% endfor %} diff --git a/ietf/templates/stats/document_stats_pages.html b/ietf/templates/stats/document_stats_pages.html index f4c930e46..dca167b1c 100644 --- a/ietf/templates/stats/document_stats_pages.html +++ b/ietf/templates/stats/document_stats_pages.html @@ -51,7 +51,7 @@ {{ pages }} {{ percentage|floatformat:2 }}% - {% include "stats/includes/docnames_cell.html" %} + {% include "stats/includes/number_with_details_cell.html" %} {% endfor %} diff --git a/ietf/templates/stats/document_stats_words.html b/ietf/templates/stats/document_stats_words.html index d5983f1d6..956e49ea7 100644 --- a/ietf/templates/stats/document_stats_words.html +++ b/ietf/templates/stats/document_stats_words.html @@ -51,7 +51,7 @@ {{ pages }} {{ percentage|floatformat:2 }}% - {% include "stats/includes/docnames_cell.html" %} + {% include "stats/includes/number_with_details_cell.html" %} {% endfor %} diff --git a/ietf/templates/stats/includes/docnames_cell.html b/ietf/templates/stats/includes/docnames_cell.html deleted file mode 100644 index fecdbe3a3..000000000 --- a/ietf/templates/stats/includes/docnames_cell.html +++ /dev/null @@ -1 +0,0 @@ -{{ names|length }} diff --git a/ietf/templates/stats/includes/number_with_details_cell.html b/ietf/templates/stats/includes/number_with_details_cell.html new file mode 100644 index 000000000..cdadc287a --- /dev/null +++ b/ietf/templates/stats/includes/number_with_details_cell.html @@ -0,0 +1 @@ +{{ names|length }} diff --git a/ietf/templates/stats/index.html b/ietf/templates/stats/index.html index 77b8b7925..9e8cc2e1f 100644 --- a/ietf/templates/stats/index.html +++ b/ietf/templates/stats/index.html @@ -9,9 +9,11 @@

{% block title %}Statistics{% endblock %}

+

Statistics on...

+ {% endblock %} From 3954dc047d281a06855fcf0f44b20a7ebae95ad2 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Tue, 31 Jan 2017 16:39:19 +0000 Subject: [PATCH 25/49] Remove extra t - Legacy-Id: 12767 --- ietf/static/ietf/js/document-stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/static/ietf/js/document-stats.js b/ietf/static/ietf/js/document-stats.js index 7d97d868e..fdfbfa36b 100644 --- a/ietf/static/ietf/js/document-stats.js +++ b/ietf/static/ietf/js/document-stats.js @@ -19,7 +19,7 @@ $(document).ready(function () { var stdNameRegExp = new RegExp("^(rfc|bcp|fyi|std)[0-9]+$", 'i'); var draftRegExp = new RegExp("^draft-", 'i'); - var html = [];t + var html = []; $.each(($(this).data("elements") || "").split("|"), function (i, element) { if (!$.trim(element)) return; From ef251c6bc7aa1ca19a2d9eb3120be3af0aa1845b Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 3 Feb 2017 18:49:43 +0000 Subject: [PATCH 26/49] Add author affiliation chart. Also add a model for registering an alias for an affiliation so that we can group affiliations that are considered the same for statistical purposes, and a model for registering unimportant endings like Inc. and GmbH. Affiliation grouping is done through three means: stripping uninteresting endings, merging entries that only differ in case and aliases that map from case-insensitive alias to name. Stripping endings and merging based on case seem to reduce the number of needed manually maintained aliases greatly. - Legacy-Id: 12785 --- ietf/doc/admin.py | 4 +- ietf/person/admin.py | 12 ++- ...filiationalias_affiliationignoredending.py | 29 ++++++ .../migrations/0016_auto_20170203_1030.py | 29 ++++++ ietf/person/models.py | 23 +++++ ietf/person/utils.py | 91 ++++++++++++++++++- ietf/static/ietf/css/ietf.css | 2 +- ietf/static/ietf/js/document-stats.js | 5 +- ietf/stats/tests.py | 2 +- ietf/stats/views.py | 35 ++++++- ietf/templates/stats/document_stats.html | 4 +- .../document_stats_author_affiliation.html | 59 ++++++++++++ 12 files changed, 284 insertions(+), 11 deletions(-) create mode 100644 ietf/person/migrations/0015_affiliationalias_affiliationignoredending.py create mode 100644 ietf/person/migrations/0016_auto_20170203_1030.py create mode 100644 ietf/templates/stats/document_stats_author_affiliation.html diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index c5db20e37..288c4a635 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -174,7 +174,7 @@ class BallotPositionDocEventAdmin(DocEventAdmin): admin.site.register(BallotPositionDocEvent, BallotPositionDocEventAdmin) class DocumentAuthorAdmin(admin.ModelAdmin): - list_display = ['id', 'document', 'person', 'email', 'order'] - search_fields = [ 'document__name', 'person__name', 'email__address', ] + list_display = ['id', 'document', 'person', 'email', 'affiliation', 'order'] + search_fields = [ 'document__name', 'person__name', 'email__address', 'affiliation'] admin.site.register(DocumentAuthor, DocumentAuthorAdmin) diff --git a/ietf/person/admin.py b/ietf/person/admin.py index 8c5ce62c0..563d212e3 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin -from ietf.person.models import Email, Alias, Person +from ietf.person.models import Email, Alias, Person, AffiliationAlias, AffiliationIgnoredEnding from ietf.person.name import name_parts class EmailAdmin(admin.ModelAdmin): @@ -33,3 +33,13 @@ class PersonAdmin(admin.ModelAdmin): # actions = None admin.site.register(Person, PersonAdmin) +class AffiliationAliasAdmin(admin.ModelAdmin): + list_filter = ["name"] + list_display = ["alias", "name"] + search_fields = ["alias", "name"] +admin.site.register(AffiliationAlias, AffiliationAliasAdmin) + +class AffiliationIgnoredEndingAdmin(admin.ModelAdmin): + list_display = ["ending"] + search_fields = ["ending"] +admin.site.register(AffiliationIgnoredEnding, AffiliationIgnoredEndingAdmin) diff --git a/ietf/person/migrations/0015_affiliationalias_affiliationignoredending.py b/ietf/person/migrations/0015_affiliationalias_affiliationignoredending.py new file mode 100644 index 000000000..1747fd224 --- /dev/null +++ b/ietf/person/migrations/0015_affiliationalias_affiliationignoredending.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('person', '0014_auto_20160613_0751'), + ] + + operations = [ + migrations.CreateModel( + name='AffiliationAlias', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('alias', models.CharField(help_text=b'Note that aliases are matched without regarding case.', max_length=255)), + ('name', models.CharField(max_length=255)), + ], + ), + migrations.CreateModel( + name='AffiliationIgnoredEnding', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('ending', models.CharField(max_length=255)), + ], + ), + ] diff --git a/ietf/person/migrations/0016_auto_20170203_1030.py b/ietf/person/migrations/0016_auto_20170203_1030.py new file mode 100644 index 000000000..d5f4fd950 --- /dev/null +++ b/ietf/person/migrations/0016_auto_20170203_1030.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def add_affiliation_info(apps, schema_editor): + AffiliationAlias = apps.get_model("person", "AffiliationAlias") + + AffiliationAlias.objects.get_or_create(alias="cisco", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco system", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco systems (india) private limited", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco systems india pvt", name="Cisco Systems") + + AffiliationIgnoredEnding = apps.get_model("person", "AffiliationIgnoredEnding") + AffiliationIgnoredEnding.objects.get_or_create(ending="LLC\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="Ltd\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="Inc\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="GmbH\.?") + + +class Migration(migrations.Migration): + + dependencies = [ + ('person', '0015_affiliationalias_affiliationignoredending'), + ] + + operations = [ + migrations.RunPython(add_affiliation_info, migrations.RunPython.noop) + ] diff --git a/ietf/person/models.py b/ietf/person/models.py index 61fa6b2c0..9b2392d4c 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -241,3 +241,26 @@ class Email(models.Model): return return self.address + +class AffiliationAlias(models.Model): + """Records that alias should be treated as name for statistical + purposes.""" + + alias = models.CharField(max_length=255, help_text="Note that aliases are matched without regarding case.") + name = models.CharField(max_length=255) + + def __unicode__(self): + return u"{} -> {}".format(self.alias, self.name) + + def save(self, *args, **kwargs): + self.alias = self.alias.lower() + super(AffiliationAlias, self).save(*args, **kwargs) + +class AffiliationIgnoredEnding(models.Model): + """Records that ending should be stripped from the affiliation for statistical purposes.""" + + ending = models.CharField(max_length=255, help_text="Regexp with ending, e.g. 'Inc\\.?' - remember to escape .!") + + def __unicode__(self): + return self.ending + diff --git a/ietf/person/utils.py b/ietf/person/utils.py index 55e7a6929..0dcf08ee6 100755 --- a/ietf/person/utils.py +++ b/ietf/person/utils.py @@ -1,8 +1,10 @@ -import pprint +import pprint +import re +from collections import defaultdict from django.contrib import admin from django.contrib.auth.models import User -from ietf.person.models import Person +from ietf.person.models import Person, AffiliationAlias, AffiliationIgnoredEnding def merge_persons(source,target,stream): @@ -86,3 +88,88 @@ def merge_persons(source,target,stream): else: print >>stream, "Deleting Person: {}({})".format(source.ascii,source.pk) source.delete() + + +def compile_affiliation_ending_stripping_regexp(): + parts = [] + for ending_re in AffiliationIgnoredEnding.objects.values_list("ending", flat=True): + try: + re.compile(ending_re) + except re.error: + pass + + parts.append(ending_re) + + re_str = ",? *({}) *$".format("|".join(parts)) + + return re.compile(re_str, re.IGNORECASE) + + +def get_aliased_affiliations(affiliations): + """Given non-unique sequence of affiliations, returns dictionary with + aliases needed. + + We employ the following strategies, interleaved: + + - Stripping company endings like Inc., GmbH etc. from database + + - Looking up aliases stored directly in the database, like + "Examplar International" -> "Examplar" + + - Case-folding so Examplar and EXAMPLAR is merged with the + winner being the one with most occurrences (so input should not + be made unique) or most upper case letters in case of ties. + Case folding can be overridden by the aliases in the database.""" + + res = {} + + ending_re = compile_affiliation_ending_stripping_regexp() + + known_aliases = { alias.lower(): name for alias, name in AffiliationAlias.objects.values_list("alias", "name") } + + affiliations_with_case_spellings = defaultdict(set) + case_spelling_count = defaultdict(int) + for affiliation in affiliations: + original_affiliation = affiliation + + # check aliases from DB + alias = known_aliases.get(affiliation.lower()) + if alias is not None: + affiliation = alias + res[original_affiliation] = affiliation + + # strip ending + alias = ending_re.sub("", affiliation) + if alias != affiliation: + affiliation = alias + res[original_affiliation] = affiliation + + # check aliases from DB + alias = known_aliases.get(affiliation.lower()) + if alias is not None: + affiliation = alias + res[original_affiliation] = affiliation + + affiliations_with_case_spellings[affiliation.lower()].add(original_affiliation) + case_spelling_count[affiliation] += 1 + + def affiliation_sort_key(affiliation): + count = case_spelling_count[affiliation] + uppercase_letters = sum(1 for c in affiliation if c.isupper()) + return (count, uppercase_letters) + + # now we just need to pick the most popular uppercase/lowercase + # spelling for each affiliation with more than one + for similar_affiliations in affiliations_with_case_spellings.itervalues(): + if len(similar_affiliations) > 1: + most_popular = sorted(similar_affiliations, key=affiliation_sort_key, reverse=True)[0] + print similar_affiliations, most_popular + for affiliation in similar_affiliations: + if affiliation != most_popular: + res[affiliation] = most_popular + print affiliation, "->", most_popular + + return res + + + diff --git a/ietf/static/ietf/css/ietf.css b/ietf/static/ietf/css/ietf.css index 76b391d58..f095e9eb7 100644 --- a/ietf/static/ietf/css/ietf.css +++ b/ietf/static/ietf/css/ietf.css @@ -568,7 +568,7 @@ table.simple-table td:last-child { width: 7em; } -.popover .docname { +.document-stats .popover .element { padding-left: 1em; text-indent: -1em; } diff --git a/ietf/static/ietf/js/document-stats.js b/ietf/static/ietf/js/document-stats.js index fdfbfa36b..25dfcc785 100644 --- a/ietf/static/ietf/js/document-stats.js +++ b/ietf/static/ietf/js/document-stats.js @@ -30,10 +30,10 @@ $(document).ready(function () { if (stdNameRegExp.test(element)) displayName = element.slice(0, 3).toUpperCase() + " " + element.slice(3); - html.push('
' + displayName + '
'); + html.push('
' + displayName + '
'); } else { - html.push('
' + element + '
'); + html.push('
' + element + '
'); } }); @@ -44,6 +44,7 @@ $(document).ready(function () { trigger: "focus", template: '', content: html.join(""), + placement: "top", html: true }).on("click", function (e) { e.preventDefault(); diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index fa973544c..026a49b96 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -25,7 +25,7 @@ class StatisticsTests(TestCase): self.assertTrue(authors_url in r["Location"]) # check various stats types - for stats_type in ["authors", "pages", "words", "format", "formlang", "author/documents"]: + for stats_type in ["authors", "pages", "words", "format", "formlang", "author/documents", "author/affiliation"]: for document_type in ["", "rfc", "draft"]: for time_choice in ["", "5y"]: url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type }) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index c151888b5..0dadfdaf7 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -25,6 +25,7 @@ from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewRequestStateName, ReviewResultName from ietf.doc.models import DocAlias, Document +from ietf.person.utils import get_aliased_affiliations from ietf.ietfauth.utils import has_role def stats_index(request): @@ -351,7 +352,7 @@ def document_stats(request, stats_type=None): total_persons = person_qs.count() if stats_type == "author/documents": - stats_title = "Number of {}s for each author".format(doc_label) + stats_title = "Number of {}s per author".format(doc_label) bins = defaultdict(list) @@ -369,6 +370,38 @@ def document_stats(request, stats_type=None): "animation": False, }) + elif stats_type == "author/affiliation": + stats_title = "Number of {} authors per affiliation".format(doc_label) + + bins = defaultdict(list) + + # Since people don't write the affiliation names in the + # same way, and we don't want to go back and edit them + # either, we transform them here. + + name_affiliation_set = set((name, affiliation) + for name, affiliation in person_qs.values_list("name", "documentauthor__affiliation")) + + aliases = get_aliased_affiliations(affiliation for _, affiliation in name_affiliation_set) + + for name, affiliation in name_affiliation_set: + bins[aliases.get(affiliation, affiliation)].append(name) + + series_data = [] + for affiliation, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / total_persons + if affiliation: + series_data.append((affiliation, len(names))) + table_data.append((affiliation, percentage, names)) + + series_data.sort(key=lambda t: t[1], reverse=True) + series_data = series_data[:30] + + chart_data.append({ + "data": series_data, + "animation": False, + }) + return render(request, "stats/document_stats.html", { "chart_data": mark_safe(json.dumps(chart_data)), diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html index 2e4bda401..8ff53471d 100644 --- a/ietf/templates/stats/document_stats.html +++ b/ietf/templates/stats/document_stats.html @@ -55,7 +55,9 @@
- {% include content_template %} +
+ {% include content_template %} +
{% endblock %} {% block js %} diff --git a/ietf/templates/stats/document_stats_author_affiliation.html b/ietf/templates/stats/document_stats_author_affiliation.html new file mode 100644 index 000000000..acca4ff4d --- /dev/null +++ b/ietf/templates/stats/document_stats_author_affiliation.html @@ -0,0 +1,59 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for affiliation, percentage, names in table_data %} + + + + + + {% endfor %} + +
AffiliationPercentage of authorsAuthors
{{ affiliation|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
From ea93001c6309dac3db8ea0dcb1f15a18e81e8d6d Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 3 Feb 2017 18:54:23 +0000 Subject: [PATCH 27/49] Remove debug output - Legacy-Id: 12786 --- ietf/person/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ietf/person/utils.py b/ietf/person/utils.py index 0dcf08ee6..024b91a53 100755 --- a/ietf/person/utils.py +++ b/ietf/person/utils.py @@ -163,11 +163,9 @@ def get_aliased_affiliations(affiliations): for similar_affiliations in affiliations_with_case_spellings.itervalues(): if len(similar_affiliations) > 1: most_popular = sorted(similar_affiliations, key=affiliation_sort_key, reverse=True)[0] - print similar_affiliations, most_popular for affiliation in similar_affiliations: if affiliation != most_popular: res[affiliation] = most_popular - print affiliation, "->", most_popular return res From 0d5bc332839fd4010dc99fa508ecc1b610a7e260 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 6 Feb 2017 11:35:28 +0000 Subject: [PATCH 28/49] Fix some test errors - Legacy-Id: 12787 --- ietf/person/resources.py | 28 +++++++++++++++++++++++++++- ietf/settings.py | 1 + 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/ietf/person/resources.py b/ietf/person/resources.py index 045b5ece4..4b6fc3cda 100644 --- a/ietf/person/resources.py +++ b/ietf/person/resources.py @@ -6,7 +6,8 @@ from tastypie.cache import SimpleCache from ietf import api -from ietf.person.models import Person, Email, Alias, PersonHistory +from ietf.person.models import (Person, Email, Alias, PersonHistory, + AffiliationAlias, AffiliationIgnoredEnding) from ietf.utils.resources import UserResource @@ -82,3 +83,28 @@ class PersonHistoryResource(ModelResource): } api.person.register(PersonHistoryResource()) +class AffiliationIgnoredEndingResource(ModelResource): + class Meta: + queryset = AffiliationIgnoredEnding.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'affiliationignoredending' + filtering = { + "id": ALL, + "ending": ALL, + } +api.person.register(AffiliationIgnoredEndingResource()) + +class AffiliationAliasResource(ModelResource): + class Meta: + queryset = AffiliationAlias.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'affiliationalias' + filtering = { + "id": ALL, + "alias": ALL, + "name": ALL, + } +api.person.register(AffiliationAliasResource()) + diff --git a/ietf/settings.py b/ietf/settings.py index 70ba8dda5..ddc77c8df 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -408,6 +408,7 @@ TEST_CODE_COVERAGE_EXCLUDE = [ "ietf/utils/test_runner.py", "name/generate_fixtures.py", "review/import_from_review_tool.py", + "stats/backfill_data.py", ] # These are filename globs. They are used by test_parse_templates() and From 882579bab34d693c2c712f56906a781fb1744b0c Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Wed, 15 Feb 2017 18:39:35 +0000 Subject: [PATCH 29/49] Don't check url reverses on the form admin:blahblah for the time being - Legacy-Id: 12845 --- ietf/utils/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 886b75043..3fcb5b9d3 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -167,7 +167,7 @@ class TemplateChecksTestCase(TestCase): Check that an URLNode's callback is in callbacks. """ cb = node.view_name.token.strip("\"'") - if cb in callbacks: + if cb in callbacks or cb.startswith("admin:"): return [] else: return [ (origin, cb), ] From b2ff10b0f2fb28570e062723fa8b2caca420aac0 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Wed, 15 Feb 2017 18:43:57 +0000 Subject: [PATCH 30/49] Add support for extracting the country line from the author addresses to the draft parser (incorporating patch from trunk), store the extracted country instead of trying to turn it into an ISO country code, add country and continent name models and add initial data for those, add helper function for cleaning the countries, add author country and continent charts, move the affiliation models to stats/models.py, fix a bunch of bugs. - Legacy-Id: 12846 --- ietf/doc/admin.py | 5 +- .../doc/migrations/0020_auto_20170112_0753.py | 5 +- ietf/doc/models.py | 4 +- ietf/name/admin.py | 17 +- .../0019_continentname_countryname.py | 44 +++ .../0020_add_country_continent_names.py | 275 ++++++++++++++++++ ietf/name/models.py | 6 + ietf/name/resources.py | 37 ++- ietf/person/admin.py | 13 +- ...filiationalias_affiliationignoredending.py | 29 -- .../migrations/0016_auto_20170203_1030.py | 29 -- ietf/person/models.py | 23 -- ietf/person/resources.py | 29 +- ietf/person/utils.py | 87 +----- ietf/secr/drafts/forms.py | 4 +- ietf/settings.py | 2 +- ietf/stats/admin.py | 22 ++ ietf/stats/backfill_data.py | 30 +- ietf/stats/migrations/0001_initial.py | 37 +++ .../migrations/0002_add_initial_aliases.py | 87 ++++++ ietf/stats/migrations/__init__.py | 0 ietf/stats/models.py | 41 +++ ietf/stats/resources.py | 52 ++++ ietf/stats/tests.py | 3 +- ietf/stats/utils.py | 198 +++++++++++++ ietf/stats/views.py | 94 +++++- ietf/submit/forms.py | 15 +- ietf/submit/views.py | 4 +- .../document_stats_author_affiliation.html | 41 +++ .../document_stats_author_continent.html | 65 +++++ .../stats/document_stats_author_country.html | 124 ++++++++ ietf/templates/submit/submission_status.html | 6 +- ietf/utils/draft.py | 73 +++-- ietf/utils/templatetags/country.py | 14 - 34 files changed, 1234 insertions(+), 281 deletions(-) create mode 100644 ietf/name/migrations/0019_continentname_countryname.py create mode 100644 ietf/name/migrations/0020_add_country_continent_names.py delete mode 100644 ietf/person/migrations/0015_affiliationalias_affiliationignoredending.py delete mode 100644 ietf/person/migrations/0016_auto_20170203_1030.py create mode 100644 ietf/stats/admin.py create mode 100644 ietf/stats/migrations/0001_initial.py create mode 100644 ietf/stats/migrations/0002_add_initial_aliases.py create mode 100644 ietf/stats/migrations/__init__.py create mode 100644 ietf/stats/models.py create mode 100644 ietf/stats/resources.py create mode 100644 ietf/stats/utils.py create mode 100644 ietf/templates/stats/document_stats_author_continent.html create mode 100644 ietf/templates/stats/document_stats_author_country.html delete mode 100644 ietf/utils/templatetags/country.py diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index 288c4a635..63109f45f 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -174,7 +174,8 @@ class BallotPositionDocEventAdmin(DocEventAdmin): admin.site.register(BallotPositionDocEvent, BallotPositionDocEventAdmin) class DocumentAuthorAdmin(admin.ModelAdmin): - list_display = ['id', 'document', 'person', 'email', 'affiliation', 'order'] - search_fields = [ 'document__name', 'person__name', 'email__address', 'affiliation'] + list_display = ['id', 'document', 'person', 'email', 'affiliation', 'country', 'order'] + search_fields = ['document__docalias__name', 'person__name', 'email__address', 'affiliation', 'country'] + raw_id_fields = ["document", "person", "email"] admin.site.register(DocumentAuthor, DocumentAuthorAdmin) diff --git a/ietf/doc/migrations/0020_auto_20170112_0753.py b/ietf/doc/migrations/0020_auto_20170112_0753.py index 9404b8aab..7335533b3 100644 --- a/ietf/doc/migrations/0020_auto_20170112_0753.py +++ b/ietf/doc/migrations/0020_auto_20170112_0753.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals from django.db import migrations, models -import django_countries.fields class Migration(migrations.Migration): @@ -49,7 +48,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='dochistoryauthor', name='country', - field=django_countries.fields.CountryField(blank=True, help_text=b'Country used by author for submission', max_length=2), + field=models.CharField(blank=True, help_text=b'Country used by author for submission', max_length=255), ), migrations.RenameField( model_name='dochistoryauthor', @@ -74,7 +73,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='documentauthor', name='country', - field=django_countries.fields.CountryField(blank=True, help_text=b'Country used by author for submission', max_length=2), + field=models.CharField(blank=True, help_text=b'Country used by author for submission', max_length=255), ), migrations.RenameField( model_name='documentauthor', diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 3592b4578..b7c3371a5 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -11,8 +11,6 @@ from django.contrib.contenttypes.models import ContentType from django.conf import settings from django.utils.html import mark_safe -from django_countries.fields import CountryField - import debug # pyflakes:ignore from ietf.group.models import Group @@ -406,7 +404,7 @@ class DocumentAuthorInfo(models.Model): # email should only be null for some historic documents email = models.ForeignKey(Email, help_text="Email address used by author for submission", blank=True, null=True) affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission") - country = CountryField(blank=True, help_text="Country used by author for submission") + country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission") order = models.IntegerField(default=1) def formatted_email(self): diff --git a/ietf/name/admin.py b/ietf/name/admin.py index 642d36aff..c4d2caa35 100644 --- a/ietf/name/admin.py +++ b/ietf/name/admin.py @@ -1,7 +1,8 @@ from django.contrib import admin from ietf.name.models import ( - BallotPositionName, ConstraintName, DBTemplateTypeName, DocRelationshipName, + BallotPositionName, ConstraintName, ContinentName, CountryName, + DBTemplateTypeName, DocRelationshipName, DocReminderTypeName, DocTagName, DocTypeName, DraftSubmissionStateName, FeedbackTypeName, FormalLanguageName, GroupMilestoneStateName, GroupStateName, GroupTypeName, IntendedStdLevelName, IprDisclosureStateName, IprEventTypeName, IprLicenseTypeName, @@ -10,8 +11,11 @@ from ietf.name.models import ( ReviewRequestStateName, ReviewResultName, ReviewTypeName, RoleName, RoomResourceName, SessionStatusName, StdLevelName, StreamName, TimeSlotTypeName, ) +from ietf.stats.models import CountryAlias + class NameAdmin(admin.ModelAdmin): list_display = ["slug", "name", "desc", "used"] + search_fields = ["slug", "name"] prepopulate_from = { "slug": ("name",) } class DocRelationshipNameAdmin(NameAdmin): @@ -26,8 +30,19 @@ class GroupTypeNameAdmin(NameAdmin): list_display = ["slug", "name", "verbose_name", "desc", "used"] admin.site.register(GroupTypeName, GroupTypeNameAdmin) +class CountryAliasInline(admin.TabularInline): + model = CountryAlias + extra = 1 + +class CountryNameAdmin(NameAdmin): + list_display = ["slug", "name", "continent", "in_eu"] + list_filter = ["continent", "in_eu"] + inlines = [CountryAliasInline] +admin.site.register(CountryName, CountryNameAdmin) + admin.site.register(BallotPositionName, NameAdmin) admin.site.register(ConstraintName, NameAdmin) +admin.site.register(ContinentName, NameAdmin) admin.site.register(DBTemplateTypeName, NameAdmin) admin.site.register(DocReminderTypeName, NameAdmin) admin.site.register(DocTagName, NameAdmin) diff --git a/ietf/name/migrations/0019_continentname_countryname.py b/ietf/name/migrations/0019_continentname_countryname.py new file mode 100644 index 000000000..3239276e6 --- /dev/null +++ b/ietf/name/migrations/0019_continentname_countryname.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0018_add_formlang_names'), + ] + + operations = [ + migrations.CreateModel( + name='ContinentName', + fields=[ + ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('desc', models.TextField(blank=True)), + ('used', models.BooleanField(default=True)), + ('order', models.IntegerField(default=0)), + ], + options={ + 'ordering': ['order', 'name'], + 'abstract': False, + }, + ), + migrations.CreateModel( + name='CountryName', + fields=[ + ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('desc', models.TextField(blank=True)), + ('used', models.BooleanField(default=True)), + ('order', models.IntegerField(default=0)), + ('in_eu', models.BooleanField(default=False, verbose_name='In EU')), + ('continent', models.ForeignKey(to='name.ContinentName')), + ], + options={ + 'ordering': ['order', 'name'], + 'abstract': False, + }, + ), + ] diff --git a/ietf/name/migrations/0020_add_country_continent_names.py b/ietf/name/migrations/0020_add_country_continent_names.py new file mode 100644 index 000000000..5adc748b7 --- /dev/null +++ b/ietf/name/migrations/0020_add_country_continent_names.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def insert_initial_country_continent_names(apps, schema_editor): + ContinentName = apps.get_model("name", "ContinentName") + africa, _ = ContinentName.objects.get_or_create(slug="africa", name="Africa") + antarctica, _ = ContinentName.objects.get_or_create(slug="antarctica", name="Antarctica") + asia, _ = ContinentName.objects.get_or_create(slug="asia", name="Asia") + europe, _ = ContinentName.objects.get_or_create(slug="europe", name="Europe") + north_america, _ = ContinentName.objects.get_or_create(slug="north-america", name="North America") + oceania, _ = ContinentName.objects.get_or_create(slug="oceania", name="Oceania") + south_america, _ = ContinentName.objects.get_or_create(slug="south-america", name="South America") + + CountryName = apps.get_model("name", "CountryName") + CountryName.objects.get_or_create(slug="AD", name=u"Andorra", continent=europe) + CountryName.objects.get_or_create(slug="AE", name=u"United Arab Emirates", continent=asia) + CountryName.objects.get_or_create(slug="AF", name=u"Afghanistan", continent=asia) + CountryName.objects.get_or_create(slug="AG", name=u"Antigua and Barbuda", continent=north_america) + CountryName.objects.get_or_create(slug="AI", name=u"Anguilla", continent=north_america) + CountryName.objects.get_or_create(slug="AL", name=u"Albania", continent=europe) + CountryName.objects.get_or_create(slug="AM", name=u"Armenia", continent=asia) + CountryName.objects.get_or_create(slug="AO", name=u"Angola", continent=africa) + CountryName.objects.get_or_create(slug="AQ", name=u"Antarctica", continent=antarctica) + CountryName.objects.get_or_create(slug="AR", name=u"Argentina", continent=south_america) + CountryName.objects.get_or_create(slug="AS", name=u"American Samoa", continent=oceania) + CountryName.objects.get_or_create(slug="AT", name=u"Austria", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="AU", name=u"Australia", continent=oceania) + CountryName.objects.get_or_create(slug="AW", name=u"Aruba", continent=north_america) + CountryName.objects.get_or_create(slug="AX", name=u"Åland Islands", continent=europe) + CountryName.objects.get_or_create(slug="AZ", name=u"Azerbaijan", continent=asia) + CountryName.objects.get_or_create(slug="BA", name=u"Bosnia and Herzegovina", continent=europe) + CountryName.objects.get_or_create(slug="BB", name=u"Barbados", continent=north_america) + CountryName.objects.get_or_create(slug="BD", name=u"Bangladesh", continent=asia) + CountryName.objects.get_or_create(slug="BE", name=u"Belgium", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="BF", name=u"Burkina Faso", continent=africa) + CountryName.objects.get_or_create(slug="BG", name=u"Bulgaria", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="BH", name=u"Bahrain", continent=asia) + CountryName.objects.get_or_create(slug="BI", name=u"Burundi", continent=africa) + CountryName.objects.get_or_create(slug="BJ", name=u"Benin", continent=africa) + CountryName.objects.get_or_create(slug="BL", name=u"Saint Barthélemy", continent=north_america) + CountryName.objects.get_or_create(slug="BM", name=u"Bermuda", continent=north_america) + CountryName.objects.get_or_create(slug="BN", name=u"Brunei", continent=asia) + CountryName.objects.get_or_create(slug="BO", name=u"Bolivia", continent=south_america) + CountryName.objects.get_or_create(slug="BQ", name=u"Bonaire, Sint Eustatius and Saba", continent=north_america) + CountryName.objects.get_or_create(slug="BR", name=u"Brazil", continent=south_america) + CountryName.objects.get_or_create(slug="BS", name=u"Bahamas", continent=north_america) + CountryName.objects.get_or_create(slug="BT", name=u"Bhutan", continent=asia) + CountryName.objects.get_or_create(slug="BV", name=u"Bouvet Island", continent=antarctica) + CountryName.objects.get_or_create(slug="BW", name=u"Botswana", continent=africa) + CountryName.objects.get_or_create(slug="BY", name=u"Belarus", continent=europe) + CountryName.objects.get_or_create(slug="BZ", name=u"Belize", continent=north_america) + CountryName.objects.get_or_create(slug="CA", name=u"Canada", continent=north_america) + CountryName.objects.get_or_create(slug="CC", name=u"Cocos (Keeling) Islands", continent=asia) + CountryName.objects.get_or_create(slug="CD", name=u"Congo (the Democratic Republic of the)", continent=africa) + CountryName.objects.get_or_create(slug="CF", name=u"Central African Republic", continent=africa) + CountryName.objects.get_or_create(slug="CG", name=u"Congo", continent=africa) + CountryName.objects.get_or_create(slug="CH", name=u"Switzerland", continent=europe) + CountryName.objects.get_or_create(slug="CI", name=u"Côte d'Ivoire", continent=africa) + CountryName.objects.get_or_create(slug="CK", name=u"Cook Islands", continent=oceania) + CountryName.objects.get_or_create(slug="CL", name=u"Chile", continent=south_america) + CountryName.objects.get_or_create(slug="CM", name=u"Cameroon", continent=africa) + CountryName.objects.get_or_create(slug="CN", name=u"China", continent=asia) + CountryName.objects.get_or_create(slug="CO", name=u"Colombia", continent=south_america) + CountryName.objects.get_or_create(slug="CR", name=u"Costa Rica", continent=north_america) + CountryName.objects.get_or_create(slug="CU", name=u"Cuba", continent=north_america) + CountryName.objects.get_or_create(slug="CV", name=u"Cabo Verde", continent=africa) + CountryName.objects.get_or_create(slug="CW", name=u"Curaçao", continent=north_america) + CountryName.objects.get_or_create(slug="CX", name=u"Christmas Island", continent=asia) + CountryName.objects.get_or_create(slug="CY", name=u"Cyprus", continent=asia, in_eu=True) + CountryName.objects.get_or_create(slug="CZ", name=u"Czech Republic", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="DE", name=u"Germany", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="DJ", name=u"Djibouti", continent=africa) + CountryName.objects.get_or_create(slug="DK", name=u"Denmark", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="DM", name=u"Dominica", continent=north_america) + CountryName.objects.get_or_create(slug="DO", name=u"Dominican Republic", continent=north_america) + CountryName.objects.get_or_create(slug="DZ", name=u"Algeria", continent=africa) + CountryName.objects.get_or_create(slug="EC", name=u"Ecuador", continent=south_america) + CountryName.objects.get_or_create(slug="EE", name=u"Estonia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="EG", name=u"Egypt", continent=africa) + CountryName.objects.get_or_create(slug="EH", name=u"Western Sahara", continent=africa) + CountryName.objects.get_or_create(slug="ER", name=u"Eritrea", continent=africa) + CountryName.objects.get_or_create(slug="ES", name=u"Spain", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="ET", name=u"Ethiopia", continent=africa) + CountryName.objects.get_or_create(slug="FI", name=u"Finland", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="FJ", name=u"Fiji", continent=oceania) + CountryName.objects.get_or_create(slug="FK", name=u"Falkland Islands [Malvinas]", continent=south_america) + CountryName.objects.get_or_create(slug="FM", name=u"Micronesia (Federated States of)", continent=oceania) + CountryName.objects.get_or_create(slug="FO", name=u"Faroe Islands", continent=europe) + CountryName.objects.get_or_create(slug="FR", name=u"France", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="GA", name=u"Gabon", continent=africa) + CountryName.objects.get_or_create(slug="GB", name=u"United Kingdom", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="GD", name=u"Grenada", continent=north_america) + CountryName.objects.get_or_create(slug="GE", name=u"Georgia", continent=asia) + CountryName.objects.get_or_create(slug="GF", name=u"French Guiana", continent=south_america) + CountryName.objects.get_or_create(slug="GG", name=u"Guernsey", continent=europe) + CountryName.objects.get_or_create(slug="GH", name=u"Ghana", continent=africa) + CountryName.objects.get_or_create(slug="GI", name=u"Gibraltar", continent=europe) + CountryName.objects.get_or_create(slug="GL", name=u"Greenland", continent=north_america) + CountryName.objects.get_or_create(slug="GM", name=u"Gambia", continent=africa) + CountryName.objects.get_or_create(slug="GN", name=u"Guinea", continent=africa) + CountryName.objects.get_or_create(slug="GP", name=u"Guadeloupe", continent=north_america) + CountryName.objects.get_or_create(slug="GQ", name=u"Equatorial Guinea", continent=africa) + CountryName.objects.get_or_create(slug="GR", name=u"Greece", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="GS", name=u"South Georgia and the South Sandwich Islands", continent=antarctica) + CountryName.objects.get_or_create(slug="GT", name=u"Guatemala", continent=north_america) + CountryName.objects.get_or_create(slug="GU", name=u"Guam", continent=oceania) + CountryName.objects.get_or_create(slug="GW", name=u"Guinea-Bissau", continent=africa) + CountryName.objects.get_or_create(slug="GY", name=u"Guyana", continent=south_america) + CountryName.objects.get_or_create(slug="HK", name=u"Hong Kong", continent=asia) + CountryName.objects.get_or_create(slug="HM", name=u"Heard Island and McDonald Islands", continent=antarctica) + CountryName.objects.get_or_create(slug="HN", name=u"Honduras", continent=north_america) + CountryName.objects.get_or_create(slug="HR", name=u"Croatia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="HT", name=u"Haiti", continent=north_america) + CountryName.objects.get_or_create(slug="HU", name=u"Hungary", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="ID", name=u"Indonesia", continent=asia) + CountryName.objects.get_or_create(slug="IE", name=u"Ireland", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="IL", name=u"Israel", continent=asia) + CountryName.objects.get_or_create(slug="IM", name=u"Isle of Man", continent=europe) + CountryName.objects.get_or_create(slug="IN", name=u"India", continent=asia) + CountryName.objects.get_or_create(slug="IO", name=u"British Indian Ocean Territory", continent=asia) + CountryName.objects.get_or_create(slug="IQ", name=u"Iraq", continent=asia) + CountryName.objects.get_or_create(slug="IR", name=u"Iran", continent=asia) + CountryName.objects.get_or_create(slug="IS", name=u"Iceland", continent=europe) + CountryName.objects.get_or_create(slug="IT", name=u"Italy", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="JE", name=u"Jersey", continent=europe) + CountryName.objects.get_or_create(slug="JM", name=u"Jamaica", continent=north_america) + CountryName.objects.get_or_create(slug="JO", name=u"Jordan", continent=asia) + CountryName.objects.get_or_create(slug="JP", name=u"Japan", continent=asia) + CountryName.objects.get_or_create(slug="KE", name=u"Kenya", continent=africa) + CountryName.objects.get_or_create(slug="KG", name=u"Kyrgyzstan", continent=asia) + CountryName.objects.get_or_create(slug="KH", name=u"Cambodia", continent=asia) + CountryName.objects.get_or_create(slug="KI", name=u"Kiribati", continent=oceania) + CountryName.objects.get_or_create(slug="KM", name=u"Comoros", continent=africa) + CountryName.objects.get_or_create(slug="KN", name=u"Saint Kitts and Nevis", continent=north_america) + CountryName.objects.get_or_create(slug="KP", name=u"North Korea", continent=asia) + CountryName.objects.get_or_create(slug="KR", name=u"South Korea", continent=asia) + CountryName.objects.get_or_create(slug="KW", name=u"Kuwait", continent=asia) + CountryName.objects.get_or_create(slug="KY", name=u"Cayman Islands", continent=north_america) + CountryName.objects.get_or_create(slug="KZ", name=u"Kazakhstan", continent=asia) + CountryName.objects.get_or_create(slug="LA", name=u"Laos", continent=asia) + CountryName.objects.get_or_create(slug="LB", name=u"Lebanon", continent=asia) + CountryName.objects.get_or_create(slug="LC", name=u"Saint Lucia", continent=north_america) + CountryName.objects.get_or_create(slug="LI", name=u"Liechtenstein", continent=europe) + CountryName.objects.get_or_create(slug="LK", name=u"Sri Lanka", continent=asia) + CountryName.objects.get_or_create(slug="LR", name=u"Liberia", continent=africa) + CountryName.objects.get_or_create(slug="LS", name=u"Lesotho", continent=africa) + CountryName.objects.get_or_create(slug="LT", name=u"Lithuania", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="LU", name=u"Luxembourg", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="LV", name=u"Latvia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="LY", name=u"Libya", continent=africa) + CountryName.objects.get_or_create(slug="MA", name=u"Morocco", continent=africa) + CountryName.objects.get_or_create(slug="MC", name=u"Monaco", continent=europe) + CountryName.objects.get_or_create(slug="MD", name=u"Moldova", continent=europe) + CountryName.objects.get_or_create(slug="ME", name=u"Montenegro", continent=europe) + CountryName.objects.get_or_create(slug="MF", name=u"Saint Martin (French part)", continent=north_america) + CountryName.objects.get_or_create(slug="MG", name=u"Madagascar", continent=africa) + CountryName.objects.get_or_create(slug="MH", name=u"Marshall Islands", continent=oceania) + CountryName.objects.get_or_create(slug="MK", name=u"Macedonia", continent=europe) + CountryName.objects.get_or_create(slug="ML", name=u"Mali", continent=africa) + CountryName.objects.get_or_create(slug="MM", name=u"Myanmar", continent=asia) + CountryName.objects.get_or_create(slug="MN", name=u"Mongolia", continent=asia) + CountryName.objects.get_or_create(slug="MO", name=u"Macao", continent=asia) + CountryName.objects.get_or_create(slug="MP", name=u"Northern Mariana Islands", continent=oceania) + CountryName.objects.get_or_create(slug="MQ", name=u"Martinique", continent=north_america) + CountryName.objects.get_or_create(slug="MR", name=u"Mauritania", continent=africa) + CountryName.objects.get_or_create(slug="MS", name=u"Montserrat", continent=north_america) + CountryName.objects.get_or_create(slug="MT", name=u"Malta", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="MU", name=u"Mauritius", continent=africa) + CountryName.objects.get_or_create(slug="MV", name=u"Maldives", continent=asia) + CountryName.objects.get_or_create(slug="MW", name=u"Malawi", continent=africa) + CountryName.objects.get_or_create(slug="MX", name=u"Mexico", continent=north_america) + CountryName.objects.get_or_create(slug="MY", name=u"Malaysia", continent=asia) + CountryName.objects.get_or_create(slug="MZ", name=u"Mozambique", continent=africa) + CountryName.objects.get_or_create(slug="NA", name=u"Namibia", continent=africa) + CountryName.objects.get_or_create(slug="NC", name=u"New Caledonia", continent=oceania) + CountryName.objects.get_or_create(slug="NE", name=u"Niger", continent=africa) + CountryName.objects.get_or_create(slug="NF", name=u"Norfolk Island", continent=oceania) + CountryName.objects.get_or_create(slug="NG", name=u"Nigeria", continent=africa) + CountryName.objects.get_or_create(slug="NI", name=u"Nicaragua", continent=north_america) + CountryName.objects.get_or_create(slug="NL", name=u"Netherlands", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="NO", name=u"Norway", continent=europe) + CountryName.objects.get_or_create(slug="NP", name=u"Nepal", continent=asia) + CountryName.objects.get_or_create(slug="NR", name=u"Nauru", continent=oceania) + CountryName.objects.get_or_create(slug="NU", name=u"Niue", continent=oceania) + CountryName.objects.get_or_create(slug="NZ", name=u"New Zealand", continent=oceania) + CountryName.objects.get_or_create(slug="OM", name=u"Oman", continent=asia) + CountryName.objects.get_or_create(slug="PA", name=u"Panama", continent=north_america) + CountryName.objects.get_or_create(slug="PE", name=u"Peru", continent=south_america) + CountryName.objects.get_or_create(slug="PF", name=u"French Polynesia", continent=oceania) + CountryName.objects.get_or_create(slug="PG", name=u"Papua New Guinea", continent=oceania) + CountryName.objects.get_or_create(slug="PH", name=u"Philippines", continent=asia) + CountryName.objects.get_or_create(slug="PK", name=u"Pakistan", continent=asia) + CountryName.objects.get_or_create(slug="PL", name=u"Poland", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="PM", name=u"Saint Pierre and Miquelon", continent=north_america) + CountryName.objects.get_or_create(slug="PN", name=u"Pitcairn", continent=oceania) + CountryName.objects.get_or_create(slug="PR", name=u"Puerto Rico", continent=north_america) + CountryName.objects.get_or_create(slug="PS", name=u"Palestine, State of", continent=asia) + CountryName.objects.get_or_create(slug="PT", name=u"Portugal", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="PW", name=u"Palau", continent=oceania) + CountryName.objects.get_or_create(slug="PY", name=u"Paraguay", continent=south_america) + CountryName.objects.get_or_create(slug="QA", name=u"Qatar", continent=asia) + CountryName.objects.get_or_create(slug="RE", name=u"Réunion", continent=africa) + CountryName.objects.get_or_create(slug="RO", name=u"Romania", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="RS", name=u"Serbia", continent=europe) + CountryName.objects.get_or_create(slug="RU", name=u"Russia", continent=europe) + CountryName.objects.get_or_create(slug="RW", name=u"Rwanda", continent=africa) + CountryName.objects.get_or_create(slug="SA", name=u"Saudi Arabia", continent=asia) + CountryName.objects.get_or_create(slug="SB", name=u"Solomon Islands", continent=oceania) + CountryName.objects.get_or_create(slug="SC", name=u"Seychelles", continent=africa) + CountryName.objects.get_or_create(slug="SD", name=u"Sudan", continent=africa) + CountryName.objects.get_or_create(slug="SE", name=u"Sweden", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="SG", name=u"Singapore", continent=asia) + CountryName.objects.get_or_create(slug="SH", name=u"Saint Helena, Ascension and Tristan da Cunha", continent=africa) + CountryName.objects.get_or_create(slug="SI", name=u"Slovenia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="SJ", name=u"Svalbard and Jan Mayen", continent=europe) + CountryName.objects.get_or_create(slug="SK", name=u"Slovakia", continent=europe, in_eu=True) + CountryName.objects.get_or_create(slug="SL", name=u"Sierra Leone", continent=africa) + CountryName.objects.get_or_create(slug="SM", name=u"San Marino", continent=europe) + CountryName.objects.get_or_create(slug="SN", name=u"Senegal", continent=africa) + CountryName.objects.get_or_create(slug="SO", name=u"Somalia", continent=africa) + CountryName.objects.get_or_create(slug="SR", name=u"Suriname", continent=south_america) + CountryName.objects.get_or_create(slug="SS", name=u"South Sudan", continent=africa) + CountryName.objects.get_or_create(slug="ST", name=u"Sao Tome and Principe", continent=africa) + CountryName.objects.get_or_create(slug="SV", name=u"El Salvador", continent=north_america) + CountryName.objects.get_or_create(slug="SX", name=u"Sint Maarten (Dutch part)", continent=north_america) + CountryName.objects.get_or_create(slug="SY", name=u"Syria", continent=asia) + CountryName.objects.get_or_create(slug="SZ", name=u"Swaziland", continent=africa) + CountryName.objects.get_or_create(slug="TC", name=u"Turks and Caicos Islands", continent=north_america) + CountryName.objects.get_or_create(slug="TD", name=u"Chad", continent=africa) + CountryName.objects.get_or_create(slug="TF", name=u"French Southern Territories", continent=antarctica) + CountryName.objects.get_or_create(slug="TG", name=u"Togo", continent=africa) + CountryName.objects.get_or_create(slug="TH", name=u"Thailand", continent=asia) + CountryName.objects.get_or_create(slug="TJ", name=u"Tajikistan", continent=asia) + CountryName.objects.get_or_create(slug="TK", name=u"Tokelau", continent=oceania) + CountryName.objects.get_or_create(slug="TL", name=u"Timor-Leste", continent=asia) + CountryName.objects.get_or_create(slug="TM", name=u"Turkmenistan", continent=asia) + CountryName.objects.get_or_create(slug="TN", name=u"Tunisia", continent=africa) + CountryName.objects.get_or_create(slug="TO", name=u"Tonga", continent=oceania) + CountryName.objects.get_or_create(slug="TR", name=u"Turkey", continent=europe) + CountryName.objects.get_or_create(slug="TT", name=u"Trinidad and Tobago", continent=north_america) + CountryName.objects.get_or_create(slug="TV", name=u"Tuvalu", continent=oceania) + CountryName.objects.get_or_create(slug="TW", name=u"Taiwan", continent=asia) + CountryName.objects.get_or_create(slug="TZ", name=u"Tanzania", continent=africa) + CountryName.objects.get_or_create(slug="UA", name=u"Ukraine", continent=europe) + CountryName.objects.get_or_create(slug="UG", name=u"Uganda", continent=africa) + CountryName.objects.get_or_create(slug="UM", name=u"United States Minor Outlying Islands", continent=oceania) + CountryName.objects.get_or_create(slug="US", name=u"United States of America", continent=north_america) + CountryName.objects.get_or_create(slug="UY", name=u"Uruguay", continent=south_america) + CountryName.objects.get_or_create(slug="UZ", name=u"Uzbekistan", continent=asia) + CountryName.objects.get_or_create(slug="VA", name=u"Holy See", continent=europe) + CountryName.objects.get_or_create(slug="VC", name=u"Saint Vincent and the Grenadines", continent=north_america) + CountryName.objects.get_or_create(slug="VE", name=u"Venezuela", continent=south_america) + CountryName.objects.get_or_create(slug="VG", name=u"Virgin Islands (British)", continent=north_america) + CountryName.objects.get_or_create(slug="VI", name=u"Virgin Islands (U.S.)", continent=north_america) + CountryName.objects.get_or_create(slug="VN", name=u"Vietnam", continent=asia) + CountryName.objects.get_or_create(slug="VU", name=u"Vanuatu", continent=oceania) + CountryName.objects.get_or_create(slug="WF", name=u"Wallis and Futuna", continent=oceania) + CountryName.objects.get_or_create(slug="WS", name=u"Samoa", continent=oceania) + CountryName.objects.get_or_create(slug="YE", name=u"Yemen", continent=asia) + CountryName.objects.get_or_create(slug="YT", name=u"Mayotte", continent=africa) + CountryName.objects.get_or_create(slug="ZA", name=u"South Africa", continent=africa) + CountryName.objects.get_or_create(slug="ZM", name=u"Zambia", continent=africa) + CountryName.objects.get_or_create(slug="ZW", name=u"Zimbabwe", continent=africa) + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0019_continentname_countryname'), + ] + + operations = [ + migrations.RunPython(insert_initial_country_continent_names, migrations.RunPython.noop) + ] diff --git a/ietf/name/models.py b/ietf/name/models.py index d208a867b..15161f331 100644 --- a/ietf/name/models.py +++ b/ietf/name/models.py @@ -99,4 +99,10 @@ class ReviewResultName(NameModel): """Almost ready, Has issues, Has nits, Not Ready, On the right track, Ready, Ready with issues, Ready with nits, Serious Issues""" +class ContinentName(NameModel): + "Africa, Antarctica, Asia, ..." +class CountryName(NameModel): + "Afghanistan, Aaland Islands, Albania, ..." + continent = models.ForeignKey(ContinentName) + in_eu = models.BooleanField(verbose_name="In EU", default=False) diff --git a/ietf/name/resources.py b/ietf/name/resources.py index f6a74387d..287963395 100644 --- a/ietf/name/resources.py +++ b/ietf/name/resources.py @@ -15,7 +15,7 @@ from ietf.name.models import (TimeSlotTypeName, GroupStateName, DocTagName, Inte LiaisonStatementTagName, FeedbackTypeName, LiaisonStatementState, StreamName, BallotPositionName, DBTemplateTypeName, NomineePositionStateName, ReviewRequestStateName, ReviewTypeName, ReviewResultName, - FormalLanguageName) + FormalLanguageName, ContinentName, CountryName) class TimeSlotTypeNameResource(ModelResource): @@ -474,3 +474,38 @@ class FormalLanguageNameResource(ModelResource): } api.name.register(FormalLanguageNameResource()) + + +class ContinentNameResource(ModelResource): + class Meta: + queryset = ContinentName.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'continentname' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(ContinentNameResource()) + +class CountryNameResource(ModelResource): + continent = ToOneField(ContinentNameResource, 'continent') + class Meta: + queryset = CountryName.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'countryname' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + "in_eu": ALL, + "continent": ALL_WITH_RELATIONS, + } +api.name.register(CountryNameResource()) + diff --git a/ietf/person/admin.py b/ietf/person/admin.py index 563d212e3..e51427afa 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin -from ietf.person.models import Email, Alias, Person, AffiliationAlias, AffiliationIgnoredEnding +from ietf.person.models import Email, Alias, Person from ietf.person.name import name_parts class EmailAdmin(admin.ModelAdmin): @@ -32,14 +32,3 @@ class PersonAdmin(admin.ModelAdmin): inlines = [ EmailInline, AliasInline, ] # actions = None admin.site.register(Person, PersonAdmin) - -class AffiliationAliasAdmin(admin.ModelAdmin): - list_filter = ["name"] - list_display = ["alias", "name"] - search_fields = ["alias", "name"] -admin.site.register(AffiliationAlias, AffiliationAliasAdmin) - -class AffiliationIgnoredEndingAdmin(admin.ModelAdmin): - list_display = ["ending"] - search_fields = ["ending"] -admin.site.register(AffiliationIgnoredEnding, AffiliationIgnoredEndingAdmin) diff --git a/ietf/person/migrations/0015_affiliationalias_affiliationignoredending.py b/ietf/person/migrations/0015_affiliationalias_affiliationignoredending.py deleted file mode 100644 index 1747fd224..000000000 --- a/ietf/person/migrations/0015_affiliationalias_affiliationignoredending.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('person', '0014_auto_20160613_0751'), - ] - - operations = [ - migrations.CreateModel( - name='AffiliationAlias', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('alias', models.CharField(help_text=b'Note that aliases are matched without regarding case.', max_length=255)), - ('name', models.CharField(max_length=255)), - ], - ), - migrations.CreateModel( - name='AffiliationIgnoredEnding', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('ending', models.CharField(max_length=255)), - ], - ), - ] diff --git a/ietf/person/migrations/0016_auto_20170203_1030.py b/ietf/person/migrations/0016_auto_20170203_1030.py deleted file mode 100644 index d5f4fd950..000000000 --- a/ietf/person/migrations/0016_auto_20170203_1030.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import migrations - -def add_affiliation_info(apps, schema_editor): - AffiliationAlias = apps.get_model("person", "AffiliationAlias") - - AffiliationAlias.objects.get_or_create(alias="cisco", name="Cisco Systems") - AffiliationAlias.objects.get_or_create(alias="cisco system", name="Cisco Systems") - AffiliationAlias.objects.get_or_create(alias="cisco systems (india) private limited", name="Cisco Systems") - AffiliationAlias.objects.get_or_create(alias="cisco systems india pvt", name="Cisco Systems") - - AffiliationIgnoredEnding = apps.get_model("person", "AffiliationIgnoredEnding") - AffiliationIgnoredEnding.objects.get_or_create(ending="LLC\.?") - AffiliationIgnoredEnding.objects.get_or_create(ending="Ltd\.?") - AffiliationIgnoredEnding.objects.get_or_create(ending="Inc\.?") - AffiliationIgnoredEnding.objects.get_or_create(ending="GmbH\.?") - - -class Migration(migrations.Migration): - - dependencies = [ - ('person', '0015_affiliationalias_affiliationignoredending'), - ] - - operations = [ - migrations.RunPython(add_affiliation_info, migrations.RunPython.noop) - ] diff --git a/ietf/person/models.py b/ietf/person/models.py index 9b2392d4c..61fa6b2c0 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -241,26 +241,3 @@ class Email(models.Model): return return self.address - -class AffiliationAlias(models.Model): - """Records that alias should be treated as name for statistical - purposes.""" - - alias = models.CharField(max_length=255, help_text="Note that aliases are matched without regarding case.") - name = models.CharField(max_length=255) - - def __unicode__(self): - return u"{} -> {}".format(self.alias, self.name) - - def save(self, *args, **kwargs): - self.alias = self.alias.lower() - super(AffiliationAlias, self).save(*args, **kwargs) - -class AffiliationIgnoredEnding(models.Model): - """Records that ending should be stripped from the affiliation for statistical purposes.""" - - ending = models.CharField(max_length=255, help_text="Regexp with ending, e.g. 'Inc\\.?' - remember to escape .!") - - def __unicode__(self): - return self.ending - diff --git a/ietf/person/resources.py b/ietf/person/resources.py index 4b6fc3cda..ff7351cc8 100644 --- a/ietf/person/resources.py +++ b/ietf/person/resources.py @@ -6,8 +6,7 @@ from tastypie.cache import SimpleCache from ietf import api -from ietf.person.models import (Person, Email, Alias, PersonHistory, - AffiliationAlias, AffiliationIgnoredEnding) +from ietf.person.models import (Person, Email, Alias, PersonHistory) from ietf.utils.resources import UserResource @@ -82,29 +81,3 @@ class PersonHistoryResource(ModelResource): "user": ALL_WITH_RELATIONS, } api.person.register(PersonHistoryResource()) - -class AffiliationIgnoredEndingResource(ModelResource): - class Meta: - queryset = AffiliationIgnoredEnding.objects.all() - serializer = api.Serializer() - cache = SimpleCache() - #resource_name = 'affiliationignoredending' - filtering = { - "id": ALL, - "ending": ALL, - } -api.person.register(AffiliationIgnoredEndingResource()) - -class AffiliationAliasResource(ModelResource): - class Meta: - queryset = AffiliationAlias.objects.all() - serializer = api.Serializer() - cache = SimpleCache() - #resource_name = 'affiliationalias' - filtering = { - "id": ALL, - "alias": ALL, - "name": ALL, - } -api.person.register(AffiliationAliasResource()) - diff --git a/ietf/person/utils.py b/ietf/person/utils.py index 024b91a53..9c033462d 100755 --- a/ietf/person/utils.py +++ b/ietf/person/utils.py @@ -1,10 +1,8 @@ import pprint -import re -from collections import defaultdict from django.contrib import admin from django.contrib.auth.models import User -from ietf.person.models import Person, AffiliationAlias, AffiliationIgnoredEnding +from ietf.person.models import Person def merge_persons(source,target,stream): @@ -88,86 +86,3 @@ def merge_persons(source,target,stream): else: print >>stream, "Deleting Person: {}({})".format(source.ascii,source.pk) source.delete() - - -def compile_affiliation_ending_stripping_regexp(): - parts = [] - for ending_re in AffiliationIgnoredEnding.objects.values_list("ending", flat=True): - try: - re.compile(ending_re) - except re.error: - pass - - parts.append(ending_re) - - re_str = ",? *({}) *$".format("|".join(parts)) - - return re.compile(re_str, re.IGNORECASE) - - -def get_aliased_affiliations(affiliations): - """Given non-unique sequence of affiliations, returns dictionary with - aliases needed. - - We employ the following strategies, interleaved: - - - Stripping company endings like Inc., GmbH etc. from database - - - Looking up aliases stored directly in the database, like - "Examplar International" -> "Examplar" - - - Case-folding so Examplar and EXAMPLAR is merged with the - winner being the one with most occurrences (so input should not - be made unique) or most upper case letters in case of ties. - Case folding can be overridden by the aliases in the database.""" - - res = {} - - ending_re = compile_affiliation_ending_stripping_regexp() - - known_aliases = { alias.lower(): name for alias, name in AffiliationAlias.objects.values_list("alias", "name") } - - affiliations_with_case_spellings = defaultdict(set) - case_spelling_count = defaultdict(int) - for affiliation in affiliations: - original_affiliation = affiliation - - # check aliases from DB - alias = known_aliases.get(affiliation.lower()) - if alias is not None: - affiliation = alias - res[original_affiliation] = affiliation - - # strip ending - alias = ending_re.sub("", affiliation) - if alias != affiliation: - affiliation = alias - res[original_affiliation] = affiliation - - # check aliases from DB - alias = known_aliases.get(affiliation.lower()) - if alias is not None: - affiliation = alias - res[original_affiliation] = affiliation - - affiliations_with_case_spellings[affiliation.lower()].add(original_affiliation) - case_spelling_count[affiliation] += 1 - - def affiliation_sort_key(affiliation): - count = case_spelling_count[affiliation] - uppercase_letters = sum(1 for c in affiliation if c.isupper()) - return (count, uppercase_letters) - - # now we just need to pick the most popular uppercase/lowercase - # spelling for each affiliation with more than one - for similar_affiliations in affiliations_with_case_spellings.itervalues(): - if len(similar_affiliations) > 1: - most_popular = sorted(similar_affiliations, key=affiliation_sort_key, reverse=True)[0] - for affiliation in similar_affiliations: - if affiliation != most_popular: - res[affiliation] = most_popular - - return res - - - diff --git a/ietf/secr/drafts/forms.py b/ietf/secr/drafts/forms.py index 956c87109..59b12718b 100644 --- a/ietf/secr/drafts/forms.py +++ b/ietf/secr/drafts/forms.py @@ -4,8 +4,6 @@ import os from django import forms -from django_countries.fields import countries - from ietf.doc.models import Document, DocAlias, State from ietf.name.models import IntendedStdLevelName, DocRelationshipName from ietf.group.models import Group @@ -107,7 +105,7 @@ class AuthorForm(forms.Form): person = forms.CharField(max_length=50,widget=forms.TextInput(attrs={'class':'name-autocomplete'}),help_text="To see a list of people type the first name, or last name, or both.") email = forms.CharField(widget=forms.Select(),help_text="Select an email.") affiliation = forms.CharField(max_length=100, required=False, help_text="Affiliation") - country = forms.ChoiceField(choices=[('', "(Not specified)")] + list(countries), required=False, help_text="Country") + country = forms.CharField(max_length=255, required=False, help_text="Country") # check for id within parenthesis to ensure name was selected from the list def clean_person(self): diff --git a/ietf/settings.py b/ietf/settings.py index ddc77c8df..6f815617f 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -293,7 +293,6 @@ INSTALLED_APPS = ( 'tastypie', 'widget_tweaks', 'django_markup', - 'django_countries', # IETF apps 'ietf.api', 'ietf.community', @@ -315,6 +314,7 @@ INSTALLED_APPS = ( 'ietf.redirects', 'ietf.release', 'ietf.review', + 'ietf.stats', 'ietf.submit', 'ietf.sync', 'ietf.utils', diff --git a/ietf/stats/admin.py b/ietf/stats/admin.py new file mode 100644 index 000000000..57f489746 --- /dev/null +++ b/ietf/stats/admin.py @@ -0,0 +1,22 @@ +from django.contrib import admin + +from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias + + +class AffiliationAliasAdmin(admin.ModelAdmin): + list_filter = ["name"] + list_display = ["alias", "name"] + search_fields = ["alias", "name"] +admin.site.register(AffiliationAlias, AffiliationAliasAdmin) + +class AffiliationIgnoredEndingAdmin(admin.ModelAdmin): + list_display = ["ending"] + search_fields = ["ending"] +admin.site.register(AffiliationIgnoredEnding, AffiliationIgnoredEndingAdmin) + +class CountryAliasAdmin(admin.ModelAdmin): + list_filter = ["country"] + list_display = ["alias", "country"] + search_fields = ["alias", "country__name"] +admin.site.register(CountryAlias, CountryAliasAdmin) + diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py index c62edfd30..cf4d7ed28 100644 --- a/ietf/stats/backfill_data.py +++ b/ietf/stats/backfill_data.py @@ -26,7 +26,6 @@ args = parser.parse_args() formal_language_dict = { l.pk: l for l in FormalLanguageName.objects.all() } - docs_qs = Document.objects.filter(type="draft") if args.document: @@ -80,11 +79,20 @@ for doc in docs_qs.prefetch_related("docalias_set", "formal_languages", "documen for author in old_authors: for alias in author.person.alias_set.all(): old_authors_by_name[alias.name] = author + old_authors_by_name[author.person.plain_name()] = author if author.email_id: old_authors_by_email[author.email_id] = author - for full, _, _, _, _, email, company in d.get_author_list(): + # the draft parser sometimes has a problem if affiliation + # isn't in the second line, then it will report an extra + # author - skip those + seen = set() + for full, _, _, _, _, email, country, company in d.get_author_list(): + if email in seen: + continue + seen.add(email) + old_author = None if email: old_author = old_authors_by_email.get(email) @@ -92,15 +100,29 @@ for doc in docs_qs.prefetch_related("docalias_set", "formal_languages", "documen old_author = old_authors_by_name.get(full) if not old_author: - print "UNKNOWN AUTHOR", doc.name, full, email, company + print "UNKNOWN AUTHOR", doc.name, full, email, country, company continue if old_author.affiliation != company: - print "new affiliation", old_author.affiliation, company + print "new affiliation", canonical_name, "[", full, "]", old_author.affiliation, "->", company old_author.affiliation = company old_author.save(update_fields=["affiliation"]) updated = True + if country is None: + country = "" + + try: + country = country.decode("utf-8") + except UnicodeDecodeError: + country = country.decode("latin-1") + + if old_author.country != country: + print "new country", canonical_name ,"[", full, "]", old_author.country.encode("utf-8"), "->", country.encode("utf-8") + old_author.country = country + old_author.save(update_fields=["country"]) + updated = True + if updates: Document.objects.filter(pk=doc.pk).update(**updates) diff --git a/ietf/stats/migrations/0001_initial.py b/ietf/stats/migrations/0001_initial.py new file mode 100644 index 000000000..89297340c --- /dev/null +++ b/ietf/stats/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0020_add_country_continent_names'), + ] + + operations = [ + migrations.CreateModel( + name='AffiliationAlias', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('alias', models.CharField(help_text=b"Note that aliases will be matched case-insensitive and both before and after some clean-up.", max_length=255, unique=True)), + ('name', models.CharField(max_length=255)), + ], + ), + migrations.CreateModel( + name='AffiliationIgnoredEnding', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('ending', models.CharField(help_text=b"Regexp with ending, e.g. 'Inc\\.?' - remember to escape .!", max_length=255)), + ], + ), + migrations.CreateModel( + name='CountryAlias', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('alias', models.CharField(help_text=b"Note that aliases are matched case-insensitive if the length is > 2.", max_length=255)), + ('country', models.ForeignKey(to='name.CountryName', max_length=255)), + ], + ), + ] diff --git a/ietf/stats/migrations/0002_add_initial_aliases.py b/ietf/stats/migrations/0002_add_initial_aliases.py new file mode 100644 index 000000000..b25cc152c --- /dev/null +++ b/ietf/stats/migrations/0002_add_initial_aliases.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def add_affiliation_info(apps, schema_editor): + AffiliationAlias = apps.get_model("stats", "AffiliationAlias") + + AffiliationAlias.objects.get_or_create(alias="cisco", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco system", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco systems (india) private limited", name="Cisco Systems") + AffiliationAlias.objects.get_or_create(alias="cisco systems india pvt", name="Cisco Systems") + + AffiliationIgnoredEnding = apps.get_model("stats", "AffiliationIgnoredEnding") + AffiliationIgnoredEnding.objects.get_or_create(ending="LLC\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="Ltd\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="Inc\.?") + AffiliationIgnoredEnding.objects.get_or_create(ending="GmbH\.?") + + CountryAlias = apps.get_model("stats", "CountryAlias") + CountryAlias.objects.get_or_create(alias="russian federation", country_id="RU") + CountryAlias.objects.get_or_create(alias="p. r. china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r. china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r.china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r china", country_id="CN") + CountryAlias.objects.get_or_create(alias="p.r. of china", country_id="CN") + CountryAlias.objects.get_or_create(alias="PRC", country_id="CN") + CountryAlias.objects.get_or_create(alias="P.R.C", country_id="CN") + CountryAlias.objects.get_or_create(alias="P.R.C.", country_id="CN") + CountryAlias.objects.get_or_create(alias="beijing", country_id="CN") + CountryAlias.objects.get_or_create(alias="shenzhen", country_id="CN") + CountryAlias.objects.get_or_create(alias="R.O.C.", country_id="TW") + CountryAlias.objects.get_or_create(alias="usa", country_id="US") + CountryAlias.objects.get_or_create(alias="UAS", country_id="US") + CountryAlias.objects.get_or_create(alias="USA.", country_id="US") + CountryAlias.objects.get_or_create(alias="u.s.a.", country_id="US") + CountryAlias.objects.get_or_create(alias="u. s. a.", country_id="US") + CountryAlias.objects.get_or_create(alias="u.s.a", country_id="US") + CountryAlias.objects.get_or_create(alias="u.s.", country_id="US") + CountryAlias.objects.get_or_create(alias="U.S", country_id="GB") + CountryAlias.objects.get_or_create(alias="US of A", country_id="US") + CountryAlias.objects.get_or_create(alias="united sates", country_id="US") + CountryAlias.objects.get_or_create(alias="united state", country_id="US") + CountryAlias.objects.get_or_create(alias="united states", country_id="US") + CountryAlias.objects.get_or_create(alias="unites states", country_id="US") + CountryAlias.objects.get_or_create(alias="texas", country_id="US") + CountryAlias.objects.get_or_create(alias="UK", country_id="GB") + CountryAlias.objects.get_or_create(alias="united kingcom", country_id="GB") + CountryAlias.objects.get_or_create(alias="great britain", country_id="GB") + CountryAlias.objects.get_or_create(alias="england", country_id="GB") + CountryAlias.objects.get_or_create(alias="U.K.", country_id="GB") + CountryAlias.objects.get_or_create(alias="U.K", country_id="GB") + CountryAlias.objects.get_or_create(alias="Uk", country_id="GB") + CountryAlias.objects.get_or_create(alias="scotland", country_id="GB") + CountryAlias.objects.get_or_create(alias="republic of korea", country_id="KR") + CountryAlias.objects.get_or_create(alias="korea", country_id="KR") + CountryAlias.objects.get_or_create(alias="korea rep", country_id="KR") + CountryAlias.objects.get_or_create(alias="korea (the republic of)", country_id="KR") + CountryAlias.objects.get_or_create(alias="the netherlands", country_id="NL") + CountryAlias.objects.get_or_create(alias="netherland", country_id="NL") + CountryAlias.objects.get_or_create(alias="danmark", country_id="DK") + CountryAlias.objects.get_or_create(alias="sweeden", country_id="SE") + CountryAlias.objects.get_or_create(alias="swede", country_id="SE") + CountryAlias.objects.get_or_create(alias="belgique", country_id="BE") + CountryAlias.objects.get_or_create(alias="madrid", country_id="ES") + CountryAlias.objects.get_or_create(alias="espana", country_id="ES") + CountryAlias.objects.get_or_create(alias="hellas", country_id="GR") + CountryAlias.objects.get_or_create(alias="gemany", country_id="DE") + CountryAlias.objects.get_or_create(alias="deutschland", country_id="DE") + CountryAlias.objects.get_or_create(alias="italia", country_id="IT") + CountryAlias.objects.get_or_create(alias="isreal", country_id="IL") + CountryAlias.objects.get_or_create(alias="tel aviv", country_id="IL") + CountryAlias.objects.get_or_create(alias="UAE", country_id="AE") + CountryAlias.objects.get_or_create(alias="grand-duchy of luxembourg", country_id="LU") + CountryAlias.objects.get_or_create(alias="brasil", country_id="BR") + + + +class Migration(migrations.Migration): + + dependencies = [ + ('stats', '0001_initial'), + ] + + operations = [ + migrations.RunPython(add_affiliation_info, migrations.RunPython.noop) + ] diff --git a/ietf/stats/migrations/__init__.py b/ietf/stats/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ietf/stats/models.py b/ietf/stats/models.py new file mode 100644 index 000000000..875a81afa --- /dev/null +++ b/ietf/stats/models.py @@ -0,0 +1,41 @@ +from django.db import models +from ietf.name.models import CountryName + +class AffiliationAlias(models.Model): + """Records that alias should be treated as name for statistical + purposes.""" + + alias = models.CharField(max_length=255, help_text="Note that aliases will be matched case-insensitive and both before and after some clean-up.", unique=True) + name = models.CharField(max_length=255) + + def __unicode__(self): + return u"{} -> {}".format(self.alias, self.name) + + def save(self, *args, **kwargs): + self.alias = self.alias.lower() + super(AffiliationAlias, self).save(*args, **kwargs) + + class Meta: + verbose_name_plural = "affiliation aliases" + +class AffiliationIgnoredEnding(models.Model): + """Records that ending should be stripped from the affiliation for statistical purposes.""" + + ending = models.CharField(max_length=255, help_text="Regexp with ending, e.g. 'Inc\\.?' - remember to escape .!") + + def __unicode__(self): + return self.ending + +class CountryAlias(models.Model): + """Records that alias should be treated as country for statistical + purposes.""" + + alias = models.CharField(max_length=255, help_text="Note that lower-case aliases are matched case-insensitive while aliases with at least one uppercase letter is matched case-sensitive.") + country = models.ForeignKey(CountryName, max_length=255) + + def __unicode__(self): + return u"{} -> {}".format(self.alias, self.country.name) + + class Meta: + verbose_name_plural = "country aliases" + diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py new file mode 100644 index 000000000..7b7d354e5 --- /dev/null +++ b/ietf/stats/resources.py @@ -0,0 +1,52 @@ +# Autogenerated by the makeresources management command 2017-02-15 10:10 PST +from tastypie.resources import ModelResource +from tastypie.fields import ToManyField # pyflakes:ignore +from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore +from tastypie.cache import SimpleCache + +from ietf import api +from ietf.api import ToOneField # pyflakes:ignore + +from ietf.stats.models import CountryAlias, AffiliationIgnoredEnding, AffiliationAlias + + +from ietf.name.resources import CountryNameResource +class CountryAliasResource(ModelResource): + country = ToOneField(CountryNameResource, 'country') + class Meta: + queryset = CountryAlias.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'countryalias' + filtering = { + "id": ALL, + "alias": ALL, + "country": ALL_WITH_RELATIONS, + } +api.stats.register(CountryAliasResource()) + +class AffiliationIgnoredEndingResource(ModelResource): + class Meta: + queryset = AffiliationIgnoredEnding.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'affiliationignoredending' + filtering = { + "id": ALL, + "ending": ALL, + } +api.stats.register(AffiliationIgnoredEndingResource()) + +class AffiliationAliasResource(ModelResource): + class Meta: + queryset = AffiliationAlias.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'affiliationalias' + filtering = { + "id": ALL, + "alias": ALL, + "name": ALL, + } +api.stats.register(AffiliationAliasResource()) + diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 026a49b96..1d0e1ef47 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -25,7 +25,8 @@ class StatisticsTests(TestCase): self.assertTrue(authors_url in r["Location"]) # check various stats types - for stats_type in ["authors", "pages", "words", "format", "formlang", "author/documents", "author/affiliation"]: + for stats_type in ["authors", "pages", "words", "format", "formlang", + "author/documents", "author/affiliation", "author/country", "author/continent"]: for document_type in ["", "rfc", "draft"]: for time_choice in ["", "5y"]: url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type }) diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py new file mode 100644 index 000000000..d2e05fffe --- /dev/null +++ b/ietf/stats/utils.py @@ -0,0 +1,198 @@ +import re +from collections import defaultdict + +from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias +from ietf.name.models import CountryName + +def compile_affiliation_ending_stripping_regexp(): + parts = [] + for ending_re in AffiliationIgnoredEnding.objects.values_list("ending", flat=True): + try: + re.compile(ending_re) + except re.error: + pass + + parts.append(ending_re) + + re_str = ",? *({}) *$".format("|".join(parts)) + + return re.compile(re_str, re.IGNORECASE) + + +def get_aliased_affiliations(affiliations): + """Given non-unique sequence of affiliations, returns dictionary with + aliases needed. + + We employ the following strategies, interleaved: + + - Stripping company endings like Inc., GmbH etc. from database + + - Looking up aliases stored directly in the database, like + "Examplar International" -> "Examplar" + + - Case-folding so Examplar and EXAMPLAR is merged with the + winner being the one with most occurrences (so input should not + be made unique) or most upper case letters in case of ties. + Case folding can be overridden by the aliases in the database.""" + + res = {} + + ending_re = compile_affiliation_ending_stripping_regexp() + + known_aliases = { alias.lower(): name for alias, name in AffiliationAlias.objects.values_list("alias", "name") } + + affiliations_with_case_spellings = defaultdict(set) + case_spelling_count = defaultdict(int) + for affiliation in affiliations: + original_affiliation = affiliation + + # check aliases from DB + name = known_aliases.get(affiliation.lower()) + if name is not None: + affiliation = name + res[original_affiliation] = affiliation + + # strip ending + name = ending_re.sub("", affiliation) + if name != affiliation: + affiliation = name + res[original_affiliation] = affiliation + + # check aliases from DB + name = known_aliases.get(affiliation.lower()) + if name is not None: + affiliation = name + res[original_affiliation] = affiliation + + affiliations_with_case_spellings[affiliation.lower()].add(original_affiliation) + case_spelling_count[affiliation] += 1 + + def affiliation_sort_key(affiliation): + count = case_spelling_count[affiliation] + uppercase_letters = sum(1 for c in affiliation if c.isupper()) + return (count, uppercase_letters) + + # now we just need to pick the most popular uppercase/lowercase + # spelling for each affiliation with more than one + for similar_affiliations in affiliations_with_case_spellings.itervalues(): + if len(similar_affiliations) > 1: + most_popular = sorted(similar_affiliations, key=affiliation_sort_key, reverse=True)[0] + for affiliation in similar_affiliations: + if affiliation != most_popular: + res[affiliation] = most_popular + + return res + + + + +def get_aliased_countries(countries): + known_aliases = dict(CountryAlias.objects.values_list("alias", "country__name")) + + iso_code_aliases = {} + + # add aliases for known countries + for slug, name in CountryName.objects.values_list("slug", "name"): + if len(name) > 2: + known_aliases[name.lower()] = name + + if len(slug) == 2 and slug[0].isupper() and slug[1].isupper(): + iso_code_aliases[slug] = name # add ISO code + + def lookup_alias(possible_alias): + name = known_aliases.get(possible_alias) + if name is not None: + return name + + name = known_aliases.get(possible_alias.lower()) + if name is not None: + return name + + return possible_alias + + known_re_aliases = { + re.compile(u"\\b{}\\b".format(re.escape(alias))): name + for alias, name in known_aliases.iteritems() + } + + # specific hack: check for zip codes from the US since in the + # early days, the addresses often didn't include the country + us_zipcode_re = re.compile(r"\b(AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY|AS|GU|MP|PR|VI|UM|FM|MH|PW|Ca|Cal.|California|CALIFORNIA|Colorado|Georgia|Illinois|Ill|Maryland|Ma|Ma.|Mass|Massachuss?etts|Michigan|Minnesota|New Jersey|New York|Ny|N.Y.|North Carolina|NORTH CAROLINA|Ohio|Oregon|Pennsylvania|Tx|Texas|Tennessee|Utah|Vermont|Virginia|Va.|Washington)[., -]*[0-9]{5}\b") + + us_country_name = CountryName.objects.get(slug="US").name + + def last_text_part_stripped(split): + for t in reversed(split): + t = t.strip() + if t: + return t + return u"" + + known_countries = set(CountryName.objects.values_list("name", flat=True)) + + res = {} + + for country in countries: + if country in res or country in known_countries: + continue + + original_country = country + + # aliased name + country = lookup_alias(country) + if country in known_countries: + res[original_country] = country + continue + + # contains US zipcode + if us_zipcode_re.search(country): + res[original_country] = us_country_name + continue + + # do a little bit of cleanup + if len(country) > 1 and country[-1] == "." and not country[-2].isupper(): + country = country.rstrip(".") + + country = country.strip("-,").strip() + + # aliased name + country = lookup_alias(country) + if country in known_countries: + res[original_country] = country + continue + + # country name at end, separated by comma + last_part = lookup_alias(last_text_part_stripped(country.split(","))) + if last_part in known_countries: + res[original_country] = last_part + continue + + # country name at end, separated by whitespace + last_part = lookup_alias(last_text_part_stripped(country.split())) + if last_part in known_countries: + res[original_country] = last_part + continue + + # country name anywhere + country_lower = country.lower() + found = False + for alias_re, name in known_re_aliases.iteritems(): + if alias_re.search(country) or alias_re.search(country_lower): + res[original_country] = name + found = True + break + + if found: + continue + + # if everything else has failed, try ISO code + country = iso_code_aliases.get(country, country) + if country in known_countries: + res[original_country] = country + continue + + # unknown country + res[original_country] = "" + + return res + diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 0dadfdaf7..68e3f29d0 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -23,9 +23,9 @@ from ietf.review.utils import (extract_review_request_data, from ietf.submit.models import Submission from ietf.group.models import Role, Group from ietf.person.models import Person -from ietf.name.models import ReviewRequestStateName, ReviewResultName +from ietf.name.models import ReviewRequestStateName, ReviewResultName, CountryName from ietf.doc.models import DocAlias, Document -from ietf.person.utils import get_aliased_affiliations +from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries from ietf.ietfauth.utils import has_role def stats_index(request): @@ -139,6 +139,8 @@ def document_stats(request, stats_type=None): table_data = [] stats_title = "" bin_size = 1 + alias_data = [] + eu_countries = None if any(stats_type == t[0] for t in possible_document_stats_types): @@ -332,7 +334,7 @@ def document_stats(request, stats_type=None): if from_time: # this is actually faster than joining in the database, # despite the round-trip back and forth - docs_within_time_constraint = list(Document.objects.filter( + docs_within_time_constraint = set(Document.objects.filter( type="draft", docevent__time__gte=from_time, docevent__type__in=["published_rfc", "new_revision"], @@ -349,7 +351,7 @@ def document_stats(request, stats_type=None): else: doc_label = "document" - total_persons = person_qs.count() + total_persons = person_qs.distinct().count() if stats_type == "author/documents": stats_title = "Number of {}s per author".format(doc_label) @@ -402,6 +404,86 @@ def document_stats(request, stats_type=None): "animation": False, }) + for alias, name in sorted(aliases.iteritems(), key=lambda t: t[1]): + alias_data.append((name, alias)) + + elif stats_type == "author/country": + stats_title = "Number of {} authors per country".format(doc_label) + + bins = defaultdict(list) + + # Since people don't write the country names in the + # same way, and we don't want to go back and edit them + # either, we transform them here. + + name_country_set = set((name, country) + for name, country in person_qs.values_list("name", "documentauthor__country")) + + aliases = get_aliased_countries(country for _, country in name_country_set) + + countries = { c.name: c for c in CountryName.objects.all() } + eu_name = "EU" + eu_countries = set(c for c in countries.itervalues() if c.in_eu) + + for name, country in name_country_set: + country_name = aliases.get(country, country) + bins[country_name].append(name) + + c = countries.get(country_name) + if c and c.in_eu: + bins[eu_name].append(name) + + series_data = [] + for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / total_persons + if country: + series_data.append((country, len(names))) + table_data.append((country, percentage, names)) + + series_data.sort(key=lambda t: t[1], reverse=True) + series_data = series_data[:30] + + chart_data.append({ + "data": series_data, + "animation": False, + }) + + for alias, country_name in aliases.iteritems(): + alias_data.append((country_name, alias, countries.get(country_name))) + + alias_data.sort() + + elif stats_type == "author/continent": + stats_title = "Number of {} authors per continent".format(doc_label) + + bins = defaultdict(list) + + name_country_set = set((name, country) + for name, country in person_qs.values_list("name", "documentauthor__country")) + + aliases = get_aliased_countries(country for _, country in name_country_set) + + country_to_continent = dict(CountryName.objects.values_list("name", "continent__name")) + + for name, country in name_country_set: + country_name = aliases.get(country, country) + continent_name = country_to_continent.get(country_name, "") + bins[continent_name].append(name) + + series_data = [] + for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + percentage = len(names) * 100.0 / total_persons + if continent: + series_data.append((continent, len(names))) + table_data.append((continent, percentage, names)) + + series_data.sort(key=lambda t: t[1], reverse=True) + + chart_data.append({ + "data": series_data, + "animation": False, + }) + return render(request, "stats/document_stats.html", { "chart_data": mark_safe(json.dumps(chart_data)), @@ -416,6 +498,10 @@ def document_stats(request, stats_type=None): "time_choice": time_choice, "doc_label": doc_label, "bin_size": bin_size, + "show_aliases_url": build_document_stats_url(get_overrides={ "showaliases": "1" }), + "hide_aliases_url": build_document_stats_url(get_overrides={ "showaliases": None }), + "alias_data": alias_data, + "eu_countries": sorted(eu_countries or [], key=lambda c: c.name), "content_template": "stats/document_stats_{}.html".format(stats_type.replace("/", "_")), }) diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index 6af82a6c3..dbef225af 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -12,8 +12,6 @@ from django.conf import settings from django.utils.html import mark_safe from django.core.urlresolvers import reverse as urlreverse -from django_countries.fields import countries - import debug # pyflakes:ignore from ietf.doc.models import Document @@ -32,15 +30,6 @@ from ietf.submit.parsers.ps_parser import PSParser from ietf.submit.parsers.xml_parser import XMLParser from ietf.utils.draft import Draft -def clean_country(country): - country = country.upper() - for code, name in countries: - if country == code: - return code - if country == name.upper(): - return code - return "" # unknown - class SubmissionUploadForm(forms.Form): txt = forms.FileField(label=u'.txt format', required=False) xml = forms.FileField(label=u'.xml format', required=False) @@ -194,7 +183,7 @@ class SubmissionUploadForm(forms.Form): "name": author.attrib.get('fullname'), "email": author.findtext('address/email'), "affiliation": author.findtext('organization'), - "country": clean_country(author.findtext('address/postal/country')), + "country": author.findtext('address/postal/country'), }) except forms.ValidationError: raise @@ -348,7 +337,7 @@ class NameEmailForm(forms.Form): class AuthorForm(NameEmailForm): affiliation = forms.CharField(max_length=100, required=False) - country = forms.ChoiceField(choices=[('', "(Not specified)")] + list(countries), required=False) + country = forms.CharField(max_length=255, required=False) def __init__(self, *args, **kwargs): super(AuthorForm, self).__init__(*args, **kwargs) diff --git a/ietf/submit/views.py b/ietf/submit/views.py index bd2305779..ed49fc836 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -81,7 +81,7 @@ def upload_submission(request): # If we don't have an xml file, try to extract the # relevant information from the text file for author in form.parsed_draft.get_author_list(): - full_name, first_name, middle_initial, last_name, name_suffix, email, company = author + full_name, first_name, middle_initial, last_name, name_suffix, email, country, company = author name = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() @@ -114,7 +114,7 @@ def upload_submission(request): "name": name, "email": email, "affiliation": company, - # FIXME: missing country + "country": country }) if form.abstract: diff --git a/ietf/templates/stats/document_stats_author_affiliation.html b/ietf/templates/stats/document_stats_author_affiliation.html index acca4ff4d..e5bcb23c7 100644 --- a/ietf/templates/stats/document_stats_author_affiliation.html +++ b/ietf/templates/stats/document_stats_author_affiliation.html @@ -57,3 +57,44 @@ {% endfor %} + +

Some authors are authors of multiple documents with different + affiliation information associated, so the sum of multiple rows in the + table can be more than 100%.

+ + + +

Affiliation Aliases

+ +

In generating the above statistics, some heuristics have been applied to determine the affiliation of each author.

+ +{% if request.GET.showaliases %} +

Hide generated aliases

+ + {% if request.user.is_staff %} +

Note: since you're an admin, you can add an extra known alias or see the existing known aliases and generally ignored endings.

+ {% endif %} + + {% if alias_data %} + + + + + + + {% for name, alias in alias_data %} + + + + + {% endfor %} +
AffiliationAlias
+ {% ifchanged %} + {{ name|default:"(unknown)" }} + {% endifchanged %} + {{ alias }}
+ {% endif %} + +{% else %} +

Show generated aliases

+{% endif %} diff --git a/ietf/templates/stats/document_stats_author_continent.html b/ietf/templates/stats/document_stats_author_continent.html new file mode 100644 index 000000000..d0327bf3a --- /dev/null +++ b/ietf/templates/stats/document_stats_author_continent.html @@ -0,0 +1,65 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for continent, percentage, names in table_data %} + + + + + + {% endfor %} + +
ContinentPercentage of authorsAuthors
{{ continent|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
+ +

The country information for an author can vary between documents, + so the sum of the rows in the table can be more than 100%. This + is especially true for the row with unknown continent information - + many authors may have one or more author entries with an + unrecognized country.

diff --git a/ietf/templates/stats/document_stats_author_country.html b/ietf/templates/stats/document_stats_author_country.html new file mode 100644 index 000000000..1b8911c4c --- /dev/null +++ b/ietf/templates/stats/document_stats_author_country.html @@ -0,0 +1,124 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for country, percentage, names in table_data %} + + + + + + {% endfor %} + +
CountryPercentage of authorsAuthors
{{ country|default:"(unknown)" }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}
+ +

The country information for an author can vary between documents, + so the sum of multiple rows in the table can be more than 100%. This + is especially true for the row with unknown country information - + many authors may have one or more author entries with an + unrecognized country.

+ +

An author is counted in EU if the country is a member of the EU + now, even if that was not the case at publication. + EU members: + {% for c in eu_countries %}{{ c.name }}{% if not forloop.last %}, {% endif %}{% endfor %}.

+ +

Country Aliases

+ +

In generating the above statistics, some heuristics have been + applied to figure out which country each author is from.

+ +{% if request.GET.showaliases %} +

Hide generated aliases

+ + {% if request.user.is_staff %} +

Note: since you're an admin, some extra links are visible. You + can either correct a document author entry directly in case the + information is obviously missing or add an alias if an unknown + country name + is being used. +

+ + {% endif %} + + {% if alias_data %} + + + + + + + + {% for name, alias, country in alias_data %} + + + + + + {% endfor %} +
CountryAlias
+ {% ifchanged %} + {% if country and request.user.is_staff %} + + {% endif %} + {{ name|default:"(unknown)" }} + {% if country and request.user.is_staff %} + + {% endif %} + {% endifchanged %} + {{ alias }} + {% if request.user.is_staff and name != "EU" %} + Matching authors + {% endif %} +
+ {% endif %} + +{% else %} +

Show generated aliases

+{% endif %} diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index 3bf65c096..39612397d 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -2,7 +2,7 @@ {# Copyright The IETF Trust 2015, All Rights Reserved #} {% load origin %} {% load staticfiles %} -{% load ietf_filters submit_tags country %} +{% load ietf_filters submit_tags %} {% block title %}Submission status of {{ submission.name }}-{{ submission.rev }}{% endblock %} @@ -207,8 +207,8 @@ Author {{ forloop.counter }} {{ author.name }} {% if author.email %}<{{ author.email }}>{% endif %} - {% if author.affiliation %}- {{ author.affiliation }}{% endif %} - {% if author.country %}- {{ author.country|country_name }}{% endif %} + - {% if author.affiliation %}{{ author.affiliation }}{% else %}unknown affiliation{% endif %} + - {% if author.country %}{{ author.country }}{% else %}unknown country{% endif %} {% endfor %} diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index b7e9c4231..4bc0365d0 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -196,6 +196,7 @@ class Draft(): line = "" newpage = False sentence = False + shortline = False blankcount = 0 linecount = 0 # two functions with side effects @@ -262,7 +263,7 @@ class Draft(): sentence = True if re.search("[^ \t]", line): if newpage: - if sentence: + if sentence or shortline: stripped += [""] else: if blankcount: @@ -270,6 +271,7 @@ class Draft(): blankcount = 0 sentence = False newpage = False + shortline = len(line.strip()) < 18 if re.search("[.:]$", line): sentence = True if re.search("^[ \t]*$", line): @@ -847,7 +849,8 @@ class Draft(): nonblank_count = 0 blanklines = 0 email = None - for line in self.lines[start+1:]: + country = None + for line_offset, line in enumerate(self.lines[start+1:]): _debug( " " + line.strip()) # Break on the second blank line if not line: @@ -887,15 +890,18 @@ class Draft(): else: pass - try: - column = line[beg:end].strip() - except: - column = line - column = re.sub(" *\(at\) *", "@", column) - column = re.sub(" *\(dot\) *", ".", column) - column = re.sub(" +at +", "@", column) - column = re.sub(" +dot +", ".", column) - column = re.sub("&cisco.com", "@cisco.com", column) + def columnify(l): + try: + column = l.replace('\t', 8 * ' ')[max(0, beg - 1):end].strip() + except: + column = l + column = re.sub(" *(?:\(at\)| | at ) *", "@", column) + column = re.sub(" *(?:\(dot\)| | dot ) *", ".", column) + column = re.sub("&cisco.com", "@cisco.com", column) + column = column.replace("\xa0", " ") + return column + + column = columnify(line) # if re.search("^\w+: \w+", column): # keyword = True @@ -906,13 +912,42 @@ class Draft(): # break #_debug( " Column text :: " + column) + if nonblank_count >= 2 and blanklines == 0: + # Usually, the contact info lines will look + # like this: "Email: someone@example.com" or + # "Tel: +1 (412)-2390 23123", but sometimes + # the : is left out. That's okay for things we + # can't misinterpret, but "tel" may match "Tel + # Aviv 69710, Israel" so match + # - misc contact info + # - tel/fax [number] + # - [phone number] + # - [email] + + other_contact_info_regex = re.compile(r'^(((contact )?e|\(e|e-|m|electronic )?mail|email_id|mailto|e-main|(tele)?phone|voice|mobile|work|uri|url|tel:)\b|^((ph|tel\.?|telefax|fax) *[:.]? *\(?( ?\+ ?)?[0-9]+)|^(\++[0-9]+|\(\+*[0-9]+\)|\(dsn\)|[0-9]+)([ -.]*\b|\b[ -.]*)(([0-9]{2,}|\([0-9]{2,}\)|(\([0-9]\)|[0-9])[ -][0-9]{2,}|\([0-9]\)[0-9]+)([ -.]+([0-9]+|\([0-9]+\)))+|([0-9]{7,}|\([0-9]{7,}\)))|^(?|^https?://|^www\.') + next_line_index = start + 1 + line_offset + 1 + + if (not country + and not other_contact_info_regex.search(column.lower()) + and next_line_index < len(self.lines)): + + next_line_lower = columnify(self.lines[next_line_index]).lower().strip() + + if not next_line_lower or other_contact_info_regex.search(next_line_lower): + # country should be here, as the last + # part of the address, right before an + # empty line or other contact info + country = column.strip() or None + _debug(" Country: %s" % country) + _debug("3: authors[%s]: %s" % (i, authors[i])) emailmatch = re.search("[-A-Za-z0-9_.+]+@[-A-Za-z0-9_.]+", column) if emailmatch and not "@" in author: email = emailmatch.group(0).lower() break - authors[i] = authors[i] + ( email, ) + + authors[i] = authors[i] + ( email, country) else: if not author in ignore: companies[i] = authors[i] @@ -938,8 +973,8 @@ class Draft(): _debug(" * Final company list: %s" % (companies,)) _debug(" * Final companies_seen: %s" % (companies_seen,)) self._author_info = authors - self._authors_with_firm = [ "%s <%s> (%s)"%(full,email,company) for full,first,middle,last,suffix,email,company in authors ] # pyflakes:ignore - self._authors = [ "%s <%s>"%(full,email) if email else full for full,first,middle,last,suffix,email,company in authors ] + self._authors_with_firm = [ "%s <%s> (%s)"%(full,email,company) for full,first,middle,last,suffix,email,country,company in authors ] # pyflakes:ignore + self._authors = [ "%s <%s>"%(full,email) if email else full for full,first,middle,last,suffix,email,country,company in authors ] self._authors.sort() _debug(" * Final author list: " + ", ".join(self._authors)) _debug("-"*72) @@ -1159,10 +1194,10 @@ def getmeta(fn): def _output(docname, fields, outfile=sys.stdout): global company_domain if opt_getauthors: - # Output an (incomplete!) getauthors-compatible format. Country - # information is always UNKNOWN, and information about security and - # iana sections presence is missing. - for full,first,middle,last,suffix,email,company in fields["_authorlist"]: + # Output an (incomplete!) getauthors-compatible format. + # Information about security and iana sections presence is + # missing. + for full,first,middle,last,suffix,email,country,company in fields["_authorlist"]: if company in company_domain: company = company_domain[company] else: @@ -1173,7 +1208,7 @@ def _output(docname, fields, outfile=sys.stdout): fields["name"] = full fields["email"] = email fields["company"] = company - fields["country"] = "UNKNOWN" + fields["country"] = country or "UNKNOWN" try: year, month, day = fields["doccreationdate"].split("-") except ValueError: diff --git a/ietf/utils/templatetags/country.py b/ietf/utils/templatetags/country.py deleted file mode 100644 index 7d730d2f9..000000000 --- a/ietf/utils/templatetags/country.py +++ /dev/null @@ -1,14 +0,0 @@ -from django.template.base import Library -from django.template.defaultfilters import stringfilter - -from django_countries import countries - -register = Library() - -@register.filter(is_safe=True) -@stringfilter -def country_name(value): - """ - Converts country code to country name - """ - return dict(countries).get(value, "") From a20d4f50ef8f856cb20262d45f4183ab16484477 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Wed, 15 Feb 2017 18:48:14 +0000 Subject: [PATCH 31/49] Update the fixtures with the country information - Legacy-Id: 12847 --- ietf/name/fixtures/names.json | 3134 ++++++++++++++++++++++++++++++++- 1 file changed, 3126 insertions(+), 8 deletions(-) diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index e8eaed56f..f876c41b7 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -120,6 +120,3064 @@ "model": "name.constraintname", "pk": "bethere" }, +{ + "fields": { + "order": 0, + "used": true, + "name": "Africa", + "desc": "" + }, + "model": "name.continentname", + "pk": "africa" +}, +{ + "fields": { + "order": 0, + "used": true, + "name": "Antarctica", + "desc": "" + }, + "model": "name.continentname", + "pk": "antarctica" +}, +{ + "fields": { + "order": 0, + "used": true, + "name": "Asia", + "desc": "" + }, + "model": "name.continentname", + "pk": "asia" +}, +{ + "fields": { + "order": 0, + "used": true, + "name": "Europe", + "desc": "" + }, + "model": "name.continentname", + "pk": "europe" +}, +{ + "fields": { + "order": 0, + "used": true, + "name": "North America", + "desc": "" + }, + "model": "name.continentname", + "pk": "north-america" +}, +{ + "fields": { + "order": 0, + "used": true, + "name": "Oceania", + "desc": "" + }, + "model": "name.continentname", + "pk": "oceania" +}, +{ + "fields": { + "order": 0, + "used": true, + "name": "South America", + "desc": "" + }, + "model": "name.continentname", + "pk": "south-america" +}, +{ + "fields": { + "used": true, + "name": "Afghanistan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AF" +}, +{ + "fields": { + "used": true, + "name": "\u00c5land Islands", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AX" +}, +{ + "fields": { + "used": true, + "name": "Albania", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AL" +}, +{ + "fields": { + "used": true, + "name": "Algeria", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "DZ" +}, +{ + "fields": { + "used": true, + "name": "American Samoa", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AS" +}, +{ + "fields": { + "used": true, + "name": "Andorra", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AD" +}, +{ + "fields": { + "used": true, + "name": "Angola", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AO" +}, +{ + "fields": { + "used": true, + "name": "Anguilla", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AI" +}, +{ + "fields": { + "used": true, + "name": "Antarctica", + "continent": "antarctica", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AQ" +}, +{ + "fields": { + "used": true, + "name": "Antigua and Barbuda", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AG" +}, +{ + "fields": { + "used": true, + "name": "Argentina", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AR" +}, +{ + "fields": { + "used": true, + "name": "Armenia", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AM" +}, +{ + "fields": { + "used": true, + "name": "Aruba", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AW" +}, +{ + "fields": { + "used": true, + "name": "Australia", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AU" +}, +{ + "fields": { + "used": true, + "name": "Austria", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "AT" +}, +{ + "fields": { + "used": true, + "name": "Azerbaijan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AZ" +}, +{ + "fields": { + "used": true, + "name": "Bahamas", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BS" +}, +{ + "fields": { + "used": true, + "name": "Bahrain", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BH" +}, +{ + "fields": { + "used": true, + "name": "Bangladesh", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BD" +}, +{ + "fields": { + "used": true, + "name": "Barbados", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BB" +}, +{ + "fields": { + "used": true, + "name": "Belarus", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BY" +}, +{ + "fields": { + "used": true, + "name": "Belgium", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "BE" +}, +{ + "fields": { + "used": true, + "name": "Belize", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BZ" +}, +{ + "fields": { + "used": true, + "name": "Benin", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BJ" +}, +{ + "fields": { + "used": true, + "name": "Bermuda", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BM" +}, +{ + "fields": { + "used": true, + "name": "Bhutan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BT" +}, +{ + "fields": { + "used": true, + "name": "Bolivia", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BO" +}, +{ + "fields": { + "used": true, + "name": "Bonaire, Sint Eustatius and Saba", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BQ" +}, +{ + "fields": { + "used": true, + "name": "Bosnia and Herzegovina", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BA" +}, +{ + "fields": { + "used": true, + "name": "Botswana", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BW" +}, +{ + "fields": { + "used": true, + "name": "Bouvet Island", + "continent": "antarctica", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BV" +}, +{ + "fields": { + "used": true, + "name": "Brazil", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BR" +}, +{ + "fields": { + "used": true, + "name": "British Indian Ocean Territory", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "IO" +}, +{ + "fields": { + "used": true, + "name": "Brunei", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BN" +}, +{ + "fields": { + "used": true, + "name": "Bulgaria", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "BG" +}, +{ + "fields": { + "used": true, + "name": "Burkina Faso", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BF" +}, +{ + "fields": { + "used": true, + "name": "Burundi", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BI" +}, +{ + "fields": { + "used": true, + "name": "Cabo Verde", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CV" +}, +{ + "fields": { + "used": true, + "name": "Cambodia", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KH" +}, +{ + "fields": { + "used": true, + "name": "Cameroon", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CM" +}, +{ + "fields": { + "used": true, + "name": "Canada", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CA" +}, +{ + "fields": { + "used": true, + "name": "Cayman Islands", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KY" +}, +{ + "fields": { + "used": true, + "name": "Central African Republic", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CF" +}, +{ + "fields": { + "used": true, + "name": "Chad", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TD" +}, +{ + "fields": { + "used": true, + "name": "Chile", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CL" +}, +{ + "fields": { + "used": true, + "name": "China", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CN" +}, +{ + "fields": { + "used": true, + "name": "Christmas Island", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CX" +}, +{ + "fields": { + "used": true, + "name": "Cocos (Keeling) Islands", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CC" +}, +{ + "fields": { + "used": true, + "name": "Colombia", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CO" +}, +{ + "fields": { + "used": true, + "name": "Comoros", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KM" +}, +{ + "fields": { + "used": true, + "name": "Congo", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CG" +}, +{ + "fields": { + "used": true, + "name": "Congo (the Democratic Republic of the)", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CD" +}, +{ + "fields": { + "used": true, + "name": "Cook Islands", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CK" +}, +{ + "fields": { + "used": true, + "name": "Costa Rica", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CR" +}, +{ + "fields": { + "used": true, + "name": "C\u00f4te d'Ivoire", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CI" +}, +{ + "fields": { + "used": true, + "name": "Croatia", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "HR" +}, +{ + "fields": { + "used": true, + "name": "Cuba", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CU" +}, +{ + "fields": { + "used": true, + "name": "Cura\u00e7ao", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CW" +}, +{ + "fields": { + "used": true, + "name": "Cyprus", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "CY" +}, +{ + "fields": { + "used": true, + "name": "Czech Republic", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "CZ" +}, +{ + "fields": { + "used": true, + "name": "Denmark", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "DK" +}, +{ + "fields": { + "used": true, + "name": "Djibouti", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "DJ" +}, +{ + "fields": { + "used": true, + "name": "Dominica", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "DM" +}, +{ + "fields": { + "used": true, + "name": "Dominican Republic", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "DO" +}, +{ + "fields": { + "used": true, + "name": "Ecuador", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "EC" +}, +{ + "fields": { + "used": true, + "name": "Egypt", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "EG" +}, +{ + "fields": { + "used": true, + "name": "El Salvador", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SV" +}, +{ + "fields": { + "used": true, + "name": "Equatorial Guinea", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GQ" +}, +{ + "fields": { + "used": true, + "name": "Eritrea", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "ER" +}, +{ + "fields": { + "used": true, + "name": "Estonia", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "EE" +}, +{ + "fields": { + "used": true, + "name": "Ethiopia", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "ET" +}, +{ + "fields": { + "used": true, + "name": "Falkland Islands [Malvinas]", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "FK" +}, +{ + "fields": { + "used": true, + "name": "Faroe Islands", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "FO" +}, +{ + "fields": { + "used": true, + "name": "Fiji", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "FJ" +}, +{ + "fields": { + "used": true, + "name": "Finland", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "FI" +}, +{ + "fields": { + "used": true, + "name": "France", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "FR" +}, +{ + "fields": { + "used": true, + "name": "French Guiana", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GF" +}, +{ + "fields": { + "used": true, + "name": "French Polynesia", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PF" +}, +{ + "fields": { + "used": true, + "name": "French Southern Territories", + "continent": "antarctica", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TF" +}, +{ + "fields": { + "used": true, + "name": "Gabon", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GA" +}, +{ + "fields": { + "used": true, + "name": "Gambia", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GM" +}, +{ + "fields": { + "used": true, + "name": "Georgia", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GE" +}, +{ + "fields": { + "used": true, + "name": "Germany", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "DE" +}, +{ + "fields": { + "used": true, + "name": "Ghana", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GH" +}, +{ + "fields": { + "used": true, + "name": "Gibraltar", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GI" +}, +{ + "fields": { + "used": true, + "name": "Greece", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "GR" +}, +{ + "fields": { + "used": true, + "name": "Greenland", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GL" +}, +{ + "fields": { + "used": true, + "name": "Grenada", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GD" +}, +{ + "fields": { + "used": true, + "name": "Guadeloupe", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GP" +}, +{ + "fields": { + "used": true, + "name": "Guam", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GU" +}, +{ + "fields": { + "used": true, + "name": "Guatemala", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GT" +}, +{ + "fields": { + "used": true, + "name": "Guernsey", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GG" +}, +{ + "fields": { + "used": true, + "name": "Guinea", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GN" +}, +{ + "fields": { + "used": true, + "name": "Guinea-Bissau", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GW" +}, +{ + "fields": { + "used": true, + "name": "Guyana", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GY" +}, +{ + "fields": { + "used": true, + "name": "Haiti", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "HT" +}, +{ + "fields": { + "used": true, + "name": "Heard Island and McDonald Islands", + "continent": "antarctica", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "HM" +}, +{ + "fields": { + "used": true, + "name": "Holy See", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "VA" +}, +{ + "fields": { + "used": true, + "name": "Honduras", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "HN" +}, +{ + "fields": { + "used": true, + "name": "Hong Kong", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "HK" +}, +{ + "fields": { + "used": true, + "name": "Hungary", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "HU" +}, +{ + "fields": { + "used": true, + "name": "Iceland", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "IS" +}, +{ + "fields": { + "used": true, + "name": "India", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "IN" +}, +{ + "fields": { + "used": true, + "name": "Indonesia", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "ID" +}, +{ + "fields": { + "used": true, + "name": "Iran", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "IR" +}, +{ + "fields": { + "used": true, + "name": "Iraq", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "IQ" +}, +{ + "fields": { + "used": true, + "name": "Ireland", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "IE" +}, +{ + "fields": { + "used": true, + "name": "Isle of Man", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "IM" +}, +{ + "fields": { + "used": true, + "name": "Israel", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "IL" +}, +{ + "fields": { + "used": true, + "name": "Italy", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "IT" +}, +{ + "fields": { + "used": true, + "name": "Jamaica", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "JM" +}, +{ + "fields": { + "used": true, + "name": "Japan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "JP" +}, +{ + "fields": { + "used": true, + "name": "Jersey", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "JE" +}, +{ + "fields": { + "used": true, + "name": "Jordan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "JO" +}, +{ + "fields": { + "used": true, + "name": "Kazakhstan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KZ" +}, +{ + "fields": { + "used": true, + "name": "Kenya", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KE" +}, +{ + "fields": { + "used": true, + "name": "Kiribati", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KI" +}, +{ + "fields": { + "used": true, + "name": "Kuwait", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KW" +}, +{ + "fields": { + "used": true, + "name": "Kyrgyzstan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KG" +}, +{ + "fields": { + "used": true, + "name": "Laos", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "LA" +}, +{ + "fields": { + "used": true, + "name": "Latvia", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "LV" +}, +{ + "fields": { + "used": true, + "name": "Lebanon", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "LB" +}, +{ + "fields": { + "used": true, + "name": "Lesotho", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "LS" +}, +{ + "fields": { + "used": true, + "name": "Liberia", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "LR" +}, +{ + "fields": { + "used": true, + "name": "Libya", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "LY" +}, +{ + "fields": { + "used": true, + "name": "Liechtenstein", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "LI" +}, +{ + "fields": { + "used": true, + "name": "Lithuania", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "LT" +}, +{ + "fields": { + "used": true, + "name": "Luxembourg", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "LU" +}, +{ + "fields": { + "used": true, + "name": "Macao", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MO" +}, +{ + "fields": { + "used": true, + "name": "Macedonia", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MK" +}, +{ + "fields": { + "used": true, + "name": "Madagascar", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MG" +}, +{ + "fields": { + "used": true, + "name": "Malawi", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MW" +}, +{ + "fields": { + "used": true, + "name": "Malaysia", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MY" +}, +{ + "fields": { + "used": true, + "name": "Maldives", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MV" +}, +{ + "fields": { + "used": true, + "name": "Mali", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "ML" +}, +{ + "fields": { + "used": true, + "name": "Malta", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "MT" +}, +{ + "fields": { + "used": true, + "name": "Marshall Islands", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MH" +}, +{ + "fields": { + "used": true, + "name": "Martinique", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MQ" +}, +{ + "fields": { + "used": true, + "name": "Mauritania", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MR" +}, +{ + "fields": { + "used": true, + "name": "Mauritius", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MU" +}, +{ + "fields": { + "used": true, + "name": "Mayotte", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "YT" +}, +{ + "fields": { + "used": true, + "name": "Mexico", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MX" +}, +{ + "fields": { + "used": true, + "name": "Micronesia (Federated States of)", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "FM" +}, +{ + "fields": { + "used": true, + "name": "Moldova", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MD" +}, +{ + "fields": { + "used": true, + "name": "Monaco", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MC" +}, +{ + "fields": { + "used": true, + "name": "Mongolia", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MN" +}, +{ + "fields": { + "used": true, + "name": "Montenegro", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "ME" +}, +{ + "fields": { + "used": true, + "name": "Montserrat", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MS" +}, +{ + "fields": { + "used": true, + "name": "Morocco", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MA" +}, +{ + "fields": { + "used": true, + "name": "Mozambique", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MZ" +}, +{ + "fields": { + "used": true, + "name": "Myanmar", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MM" +}, +{ + "fields": { + "used": true, + "name": "Namibia", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NA" +}, +{ + "fields": { + "used": true, + "name": "Nauru", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NR" +}, +{ + "fields": { + "used": true, + "name": "Nepal", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NP" +}, +{ + "fields": { + "used": true, + "name": "Netherlands", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "NL" +}, +{ + "fields": { + "used": true, + "name": "New Caledonia", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NC" +}, +{ + "fields": { + "used": true, + "name": "New Zealand", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NZ" +}, +{ + "fields": { + "used": true, + "name": "Nicaragua", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NI" +}, +{ + "fields": { + "used": true, + "name": "Niger", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NE" +}, +{ + "fields": { + "used": true, + "name": "Nigeria", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NG" +}, +{ + "fields": { + "used": true, + "name": "Niue", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NU" +}, +{ + "fields": { + "used": true, + "name": "Norfolk Island", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NF" +}, +{ + "fields": { + "used": true, + "name": "North Korea", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KP" +}, +{ + "fields": { + "used": true, + "name": "Northern Mariana Islands", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MP" +}, +{ + "fields": { + "used": true, + "name": "Norway", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "NO" +}, +{ + "fields": { + "used": true, + "name": "Oman", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "OM" +}, +{ + "fields": { + "used": true, + "name": "Pakistan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PK" +}, +{ + "fields": { + "used": true, + "name": "Palau", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PW" +}, +{ + "fields": { + "used": true, + "name": "Palestine, State of", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PS" +}, +{ + "fields": { + "used": true, + "name": "Panama", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PA" +}, +{ + "fields": { + "used": true, + "name": "Papua New Guinea", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PG" +}, +{ + "fields": { + "used": true, + "name": "Paraguay", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PY" +}, +{ + "fields": { + "used": true, + "name": "Peru", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PE" +}, +{ + "fields": { + "used": true, + "name": "Philippines", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PH" +}, +{ + "fields": { + "used": true, + "name": "Pitcairn", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PN" +}, +{ + "fields": { + "used": true, + "name": "Poland", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "PL" +}, +{ + "fields": { + "used": true, + "name": "Portugal", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "PT" +}, +{ + "fields": { + "used": true, + "name": "Puerto Rico", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PR" +}, +{ + "fields": { + "used": true, + "name": "Qatar", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "QA" +}, +{ + "fields": { + "used": true, + "name": "R\u00e9union", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "RE" +}, +{ + "fields": { + "used": true, + "name": "Romania", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "RO" +}, +{ + "fields": { + "used": true, + "name": "Russia", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "RU" +}, +{ + "fields": { + "used": true, + "name": "Rwanda", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "RW" +}, +{ + "fields": { + "used": true, + "name": "Saint Barth\u00e9lemy", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "BL" +}, +{ + "fields": { + "used": true, + "name": "Saint Helena, Ascension and Tristan da Cunha", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SH" +}, +{ + "fields": { + "used": true, + "name": "Saint Kitts and Nevis", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KN" +}, +{ + "fields": { + "used": true, + "name": "Saint Lucia", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "LC" +}, +{ + "fields": { + "used": true, + "name": "Saint Martin (French part)", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "MF" +}, +{ + "fields": { + "used": true, + "name": "Saint Pierre and Miquelon", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "PM" +}, +{ + "fields": { + "used": true, + "name": "Saint Vincent and the Grenadines", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "VC" +}, +{ + "fields": { + "used": true, + "name": "Samoa", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "WS" +}, +{ + "fields": { + "used": true, + "name": "San Marino", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SM" +}, +{ + "fields": { + "used": true, + "name": "Sao Tome and Principe", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "ST" +}, +{ + "fields": { + "used": true, + "name": "Saudi Arabia", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SA" +}, +{ + "fields": { + "used": true, + "name": "Senegal", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SN" +}, +{ + "fields": { + "used": true, + "name": "Serbia", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "RS" +}, +{ + "fields": { + "used": true, + "name": "Seychelles", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SC" +}, +{ + "fields": { + "used": true, + "name": "Sierra Leone", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SL" +}, +{ + "fields": { + "used": true, + "name": "Singapore", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SG" +}, +{ + "fields": { + "used": true, + "name": "Sint Maarten (Dutch part)", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SX" +}, +{ + "fields": { + "used": true, + "name": "Slovakia", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "SK" +}, +{ + "fields": { + "used": true, + "name": "Slovenia", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "SI" +}, +{ + "fields": { + "used": true, + "name": "Solomon Islands", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SB" +}, +{ + "fields": { + "used": true, + "name": "Somalia", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SO" +}, +{ + "fields": { + "used": true, + "name": "South Africa", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "ZA" +}, +{ + "fields": { + "used": true, + "name": "South Georgia and the South Sandwich Islands", + "continent": "antarctica", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "GS" +}, +{ + "fields": { + "used": true, + "name": "South Korea", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "KR" +}, +{ + "fields": { + "used": true, + "name": "South Sudan", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SS" +}, +{ + "fields": { + "used": true, + "name": "Spain", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "ES" +}, +{ + "fields": { + "used": true, + "name": "Sri Lanka", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "LK" +}, +{ + "fields": { + "used": true, + "name": "Sudan", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SD" +}, +{ + "fields": { + "used": true, + "name": "Suriname", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SR" +}, +{ + "fields": { + "used": true, + "name": "Svalbard and Jan Mayen", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SJ" +}, +{ + "fields": { + "used": true, + "name": "Swaziland", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SZ" +}, +{ + "fields": { + "used": true, + "name": "Sweden", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "SE" +}, +{ + "fields": { + "used": true, + "name": "Switzerland", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "CH" +}, +{ + "fields": { + "used": true, + "name": "Syria", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "SY" +}, +{ + "fields": { + "used": true, + "name": "Taiwan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TW" +}, +{ + "fields": { + "used": true, + "name": "Tajikistan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TJ" +}, +{ + "fields": { + "used": true, + "name": "Tanzania", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TZ" +}, +{ + "fields": { + "used": true, + "name": "Thailand", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TH" +}, +{ + "fields": { + "used": true, + "name": "Timor-Leste", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TL" +}, +{ + "fields": { + "used": true, + "name": "Togo", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TG" +}, +{ + "fields": { + "used": true, + "name": "Tokelau", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TK" +}, +{ + "fields": { + "used": true, + "name": "Tonga", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TO" +}, +{ + "fields": { + "used": true, + "name": "Trinidad and Tobago", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TT" +}, +{ + "fields": { + "used": true, + "name": "Tunisia", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TN" +}, +{ + "fields": { + "used": true, + "name": "Turkey", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TR" +}, +{ + "fields": { + "used": true, + "name": "Turkmenistan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TM" +}, +{ + "fields": { + "used": true, + "name": "Turks and Caicos Islands", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TC" +}, +{ + "fields": { + "used": true, + "name": "Tuvalu", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "TV" +}, +{ + "fields": { + "used": true, + "name": "Uganda", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "UG" +}, +{ + "fields": { + "used": true, + "name": "Ukraine", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "UA" +}, +{ + "fields": { + "used": true, + "name": "United Arab Emirates", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "AE" +}, +{ + "fields": { + "used": true, + "name": "United Kingdom", + "continent": "europe", + "desc": "", + "order": 0, + "in_eu": true + }, + "model": "name.countryname", + "pk": "GB" +}, +{ + "fields": { + "used": true, + "name": "United States Minor Outlying Islands", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "UM" +}, +{ + "fields": { + "used": true, + "name": "United States of America", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "US" +}, +{ + "fields": { + "used": true, + "name": "Uruguay", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "UY" +}, +{ + "fields": { + "used": true, + "name": "Uzbekistan", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "UZ" +}, +{ + "fields": { + "used": true, + "name": "Vanuatu", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "VU" +}, +{ + "fields": { + "used": true, + "name": "Venezuela", + "continent": "south-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "VE" +}, +{ + "fields": { + "used": true, + "name": "Vietnam", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "VN" +}, +{ + "fields": { + "used": true, + "name": "Virgin Islands (British)", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "VG" +}, +{ + "fields": { + "used": true, + "name": "Virgin Islands (U.S.)", + "continent": "north-america", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "VI" +}, +{ + "fields": { + "used": true, + "name": "Wallis and Futuna", + "continent": "oceania", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "WF" +}, +{ + "fields": { + "used": true, + "name": "Western Sahara", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "EH" +}, +{ + "fields": { + "used": true, + "name": "Yemen", + "continent": "asia", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "YE" +}, +{ + "fields": { + "used": true, + "name": "Zambia", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "ZM" +}, +{ + "fields": { + "used": true, + "name": "Zimbabwe", + "continent": "africa", + "desc": "", + "order": 0, + "in_eu": false + }, + "model": "name.countryname", + "pk": "ZW" +}, { "fields": { "order": 0, @@ -937,6 +3995,66 @@ "model": "name.feedbacktypename", "pk": "questio" }, +{ + "fields": { + "order": 1, + "used": true, + "name": "ABNF", + "desc": "Augmented Backus-Naur Form" + }, + "model": "name.formallanguagename", + "pk": "abnf" +}, +{ + "fields": { + "order": 2, + "used": true, + "name": "ASN.1", + "desc": "Abstract Syntax Notation One" + }, + "model": "name.formallanguagename", + "pk": "asn1" +}, +{ + "fields": { + "order": 3, + "used": true, + "name": "CBOR", + "desc": "Concise Binary Object Representation" + }, + "model": "name.formallanguagename", + "pk": "cbor" +}, +{ + "fields": { + "order": 4, + "used": true, + "name": "C Code", + "desc": "Code in the C Programming Language" + }, + "model": "name.formallanguagename", + "pk": "ccode" +}, +{ + "fields": { + "order": 5, + "used": true, + "name": "JSON", + "desc": "Javascript Object Notation" + }, + "model": "name.formallanguagename", + "pk": "json" +}, +{ + "fields": { + "order": 6, + "used": true, + "name": "XML", + "desc": "Extensible Markup Language" + }, + "model": "name.formallanguagename", + "pk": "xml" +}, { "fields": { "order": 1, @@ -2750,18 +5868,18 @@ }, { "fields": { - "label": "Shepherd's writeup state" - }, - "model": "doc.statetype", - "pk": "shepwrit" -}, -{ - "fields": { - "label": "Liaison state" + "label": "Liason Statement State" }, "model": "doc.statetype", "pk": "liaison" }, +{ + "fields": { + "label": "Shepherd's Writeup State" + }, + "model": "doc.statetype", + "pk": "shepwrit" +}, { "fields": { "used": true, From d2e85a3aa36b991b2bebaeb6466e0617b421975a Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Wed, 15 Feb 2017 19:10:59 +0000 Subject: [PATCH 32/49] Apply draft parser patch from Henrik to improve the patch on trunk to combine paragraphs across page splits - this makes the country part of the parser find more countries - Legacy-Id: 12848 --- ietf/utils/draft.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index 4bc0365d0..603a63afa 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -196,7 +196,7 @@ class Draft(): line = "" newpage = False sentence = False - shortline = False + shortprev = False blankcount = 0 linecount = 0 # two functions with side effects @@ -238,7 +238,7 @@ class Draft(): if re.search("\f", line, re.I): pages, page, newpage = begpage(pages, page, newpage) continue - if re.search("^ *Internet.Draft.+ .+[12][0-9][0-9][0-9] *$", line, re.I): + if re.search("^ *Internet.Draft.+ .+[12][0-9][0-9][0-9] *$", line, re.I): pages, page, newpage = begpage(pages, page, newpage, line) continue # if re.search("^ *Internet.Draft +", line, re.I): @@ -263,7 +263,9 @@ class Draft(): sentence = True if re.search("[^ \t]", line): if newpage: - if sentence or shortline: + # 36 is a somewhat arbitrary count for a 'short' line + shortthis = len(line.strip()) < 36 # 36 is a somewhat arbitrary count for a 'short' line + if sentence or (shortprev and not shortthis): stripped += [""] else: if blankcount: @@ -271,7 +273,7 @@ class Draft(): blankcount = 0 sentence = False newpage = False - shortline = len(line.strip()) < 18 + shortprev = len(line.strip()) < 36 # 36 is a somewhat arbitrary count for a 'short' line if re.search("[.:]$", line): sentence = True if re.search("^[ \t]*$", line): From 1a0e4599c530c197a73dabb919859d8f37d60b7a Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Thu, 16 Feb 2017 12:23:45 +0000 Subject: [PATCH 33/49] Adjust the way authors with unknown countries are counted and improve the explanation of how the numbers have come to be - Legacy-Id: 12858 --- ietf/stats/views.py | 16 ++++++++++++++++ .../stats/document_stats_author_continent.html | 9 ++++----- .../stats/document_stats_author_country.html | 12 +++++++----- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 68e3f29d0..3466e329a 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -433,6 +433,14 @@ def document_stats(request, stats_type=None): if c and c.in_eu: bins[eu_name].append(name) + # remove from the unknown bin all authors with a known country + all_known = set(n for b, names in bins.iteritems() if b for n in names) + unknown = [] + for name in bins[""]: + if name not in all_known: + unknown.append(name) + bins[""] = unknown + series_data = [] for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): percentage = len(names) * 100.0 / total_persons @@ -470,6 +478,14 @@ def document_stats(request, stats_type=None): continent_name = country_to_continent.get(country_name, "") bins[continent_name].append(name) + # remove from the unknown bin all authors with a known continent + all_known = set(n for b, names in bins.iteritems() if b for n in names) + unknown = [] + for name in bins[""]: + if name not in all_known: + unknown.append(name) + bins[""] = unknown + series_data = [] for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): percentage = len(names) * 100.0 / total_persons diff --git a/ietf/templates/stats/document_stats_author_continent.html b/ietf/templates/stats/document_stats_author_continent.html index d0327bf3a..b17d49552 100644 --- a/ietf/templates/stats/document_stats_author_continent.html +++ b/ietf/templates/stats/document_stats_author_continent.html @@ -58,8 +58,7 @@ -

The country information for an author can vary between documents, - so the sum of the rows in the table can be more than 100%. This - is especially true for the row with unknown continent information - - many authors may have one or more author entries with an - unrecognized country.

+

The statistics are based entirely on the author addresses provided + in each draft. Since this varies across documents, a travelling + author may be counted in more than country, making the total sum + more than 100%.

diff --git a/ietf/templates/stats/document_stats_author_country.html b/ietf/templates/stats/document_stats_author_country.html index 1b8911c4c..99c6d0152 100644 --- a/ietf/templates/stats/document_stats_author_country.html +++ b/ietf/templates/stats/document_stats_author_country.html @@ -58,11 +58,13 @@ -

The country information for an author can vary between documents, - so the sum of multiple rows in the table can be more than 100%. This - is especially true for the row with unknown country information - - many authors may have one or more author entries with an - unrecognized country.

+

The statistics are based entirely on the author addresses provided + in each draft. Since this varies across documents, a travelling + author may be counted in more than country, making the total sum + more than 100%.

+ +

In case no country information is found for an author in the time + period, the author is counted as (unknown).

An author is counted in EU if the country is a member of the EU now, even if that was not the case at publication. From d06f56fc0d9eb7a3e635b6990b4bff75fa8e1781 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Thu, 16 Feb 2017 12:32:03 +0000 Subject: [PATCH 34/49] Adjust unknown bin in the affiliation statistics too, adjust wording - Legacy-Id: 12859 --- ietf/stats/views.py | 29 ++++++++++--------- .../document_stats_author_affiliation.html | 10 ++++--- .../document_stats_author_continent.html | 2 +- .../stats/document_stats_author_country.html | 2 +- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 3466e329a..b55362162 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -353,6 +353,17 @@ def document_stats(request, stats_type=None): total_persons = person_qs.distinct().count() + def prune_unknown_bin_with_known(bins): + # remove from the unknown bin all authors within the + # named/known bins + all_known = set(n for b, names in bins.iteritems() if b for n in names) + unknown = [] + for name in bins[""]: + if name not in all_known: + unknown.append(name) + bins[""] = unknown + + if stats_type == "author/documents": stats_title = "Number of {}s per author".format(doc_label) @@ -389,6 +400,8 @@ def document_stats(request, stats_type=None): for name, affiliation in name_affiliation_set: bins[aliases.get(affiliation, affiliation)].append(name) + prune_unknown_bin_with_known(bins) + series_data = [] for affiliation, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): percentage = len(names) * 100.0 / total_persons @@ -433,13 +446,7 @@ def document_stats(request, stats_type=None): if c and c.in_eu: bins[eu_name].append(name) - # remove from the unknown bin all authors with a known country - all_known = set(n for b, names in bins.iteritems() if b for n in names) - unknown = [] - for name in bins[""]: - if name not in all_known: - unknown.append(name) - bins[""] = unknown + prune_unknown_bin_with_known(bins) series_data = [] for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): @@ -478,13 +485,7 @@ def document_stats(request, stats_type=None): continent_name = country_to_continent.get(country_name, "") bins[continent_name].append(name) - # remove from the unknown bin all authors with a known continent - all_known = set(n for b, names in bins.iteritems() if b for n in names) - unknown = [] - for name in bins[""]: - if name not in all_known: - unknown.append(name) - bins[""] = unknown + prune_unknown_bin_with_known(bins) series_data = [] for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): diff --git a/ietf/templates/stats/document_stats_author_affiliation.html b/ietf/templates/stats/document_stats_author_affiliation.html index e5bcb23c7..6bec8d3c8 100644 --- a/ietf/templates/stats/document_stats_author_affiliation.html +++ b/ietf/templates/stats/document_stats_author_affiliation.html @@ -58,15 +58,17 @@ -

Some authors are authors of multiple documents with different - affiliation information associated, so the sum of multiple rows in the - table can be more than 100%.

+

The statistics are based entirely on the author affiliation + provided with each draft. Since this may vary across documents, an + author may be counted with more than one affiliation, making the + total sum more than 100%.

Affiliation Aliases

-

In generating the above statistics, some heuristics have been applied to determine the affiliation of each author.

+

In generating the above statistics, some heuristics have been + applied to determine the affiliations of each author.

{% if request.GET.showaliases %}

Hide generated aliases

diff --git a/ietf/templates/stats/document_stats_author_continent.html b/ietf/templates/stats/document_stats_author_continent.html index b17d49552..0e0a1f849 100644 --- a/ietf/templates/stats/document_stats_author_continent.html +++ b/ietf/templates/stats/document_stats_author_continent.html @@ -59,6 +59,6 @@

The statistics are based entirely on the author addresses provided - in each draft. Since this varies across documents, a travelling + with each draft. Since this varies across documents, a travelling author may be counted in more than country, making the total sum more than 100%.

diff --git a/ietf/templates/stats/document_stats_author_country.html b/ietf/templates/stats/document_stats_author_country.html index 99c6d0152..93914abcb 100644 --- a/ietf/templates/stats/document_stats_author_country.html +++ b/ietf/templates/stats/document_stats_author_country.html @@ -59,7 +59,7 @@

The statistics are based entirely on the author addresses provided - in each draft. Since this varies across documents, a travelling + with each draft. Since this varies across documents, a travelling author may be counted in more than country, making the total sum more than 100%.

From 9492e1ee85a75709b2656e3b9e8257fc88b377a1 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Thu, 16 Feb 2017 15:24:31 +0000 Subject: [PATCH 35/49] Improve EU explanation - Legacy-Id: 12860 --- ietf/templates/stats/document_stats_author_country.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ietf/templates/stats/document_stats_author_country.html b/ietf/templates/stats/document_stats_author_country.html index 93914abcb..7819e6c2c 100644 --- a/ietf/templates/stats/document_stats_author_country.html +++ b/ietf/templates/stats/document_stats_author_country.html @@ -66,9 +66,8 @@

In case no country information is found for an author in the time period, the author is counted as (unknown).

-

An author is counted in EU if the country is a member of the EU - now, even if that was not the case at publication. - EU members: +

EU (European Union) is not a country, but has been added for reference, as the sum of + all current EU member countries: {% for c in eu_countries %}{{ c.name }}{% if not forloop.last %}, {% endif %}{% endfor %}.

Country Aliases

From 4501cd65d7d23ea295d2db027206bad5832ffb18 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Thu, 16 Feb 2017 15:26:29 +0000 Subject: [PATCH 36/49] Speed up a couple of the stats queries - Legacy-Id: 12861 --- ietf/stats/views.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index b55362162..1275fbd65 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -24,7 +24,7 @@ from ietf.submit.models import Submission from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewRequestStateName, ReviewResultName, CountryName -from ietf.doc.models import DocAlias, Document +from ietf.doc.models import DocAlias, Document, State from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries from ietf.ietfauth.utils import has_role @@ -147,10 +147,11 @@ def document_stats(request, stats_type=None): # filter documents docalias_qs = DocAlias.objects.filter(document__type="draft") + rfc_state = State.objects.get(type="draft", slug="rfc") if document_type == "rfc": - docalias_qs = docalias_qs.filter(document__states__type="draft", document__states__slug="rfc") + docalias_qs = docalias_qs.filter(document__states=rfc_state) elif document_type == "draft": - docalias_qs = docalias_qs.exclude(document__states__type="draft", document__states__slug="rfc") + docalias_qs = docalias_qs.exclude(document__states=rfc_state) if from_time: # this is actually faster than joining in the database, @@ -326,10 +327,11 @@ def document_stats(request, stats_type=None): person_filters = Q(documentauthor__document__type="draft") # filter persons + rfc_state = State.objects.get(type="draft", slug="rfc") if document_type == "rfc": - person_filters &= Q(documentauthor__document__states__type="draft", documentauthor__document__states__slug="rfc") + person_filters &= Q(documentauthor__document__states=rfc_state) elif document_type == "draft": - person_filters &= ~Q(documentauthor__document__states__type="draft", documentauthor__document__states__slug="rfc") + person_filters &= ~Q(documentauthor__document__states=rfc_state) if from_time: # this is actually faster than joining in the database, @@ -351,18 +353,14 @@ def document_stats(request, stats_type=None): else: doc_label = "document" - total_persons = person_qs.distinct().count() - def prune_unknown_bin_with_known(bins): # remove from the unknown bin all authors within the # named/known bins all_known = set(n for b, names in bins.iteritems() if b for n in names) - unknown = [] - for name in bins[""]: - if name not in all_known: - unknown.append(name) - bins[""] = unknown + bins[""] = [name for name in bins[""] if name not in all_known] + def count_bins(bins): + return len(set(n for b, names in bins.iteritems() if b for n in names)) if stats_type == "author/documents": stats_title = "Number of {}s per author".format(doc_label) @@ -372,6 +370,8 @@ def document_stats(request, stats_type=None): for name, document_count in person_qs.values_list("name").annotate(Count("documentauthor")): bins[document_count].append(name) + total_persons = count_bins(bins) + series_data = [] for document_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): percentage = len(names) * 100.0 / total_persons @@ -401,6 +401,7 @@ def document_stats(request, stats_type=None): bins[aliases.get(affiliation, affiliation)].append(name) prune_unknown_bin_with_known(bins) + total_persons = count_bins(bins) series_data = [] for affiliation, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): @@ -447,6 +448,7 @@ def document_stats(request, stats_type=None): bins[eu_name].append(name) prune_unknown_bin_with_known(bins) + total_persons = count_bins(bins) series_data = [] for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): @@ -486,6 +488,7 @@ def document_stats(request, stats_type=None): bins[continent_name].append(name) prune_unknown_bin_with_known(bins) + total_persons = count_bins(bins) series_data = [] for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): From 9d8874d8b1f2d03f7a630e2fbc55c7e3eda335df Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Thu, 16 Feb 2017 17:56:23 +0000 Subject: [PATCH 37/49] Add a suggestion to fix the country name when submitting a draft in case we can't parse it and it's not empty, add page with list of countries (to be able to refer people to it), add ISO codes as country aliases - the country alias code is now more intelligent with respect to case so it's easier to keep these aliases explicitly - Legacy-Id: 12862 --- .../migrations/0002_add_initial_aliases.py | 21 +++++++++++- ietf/stats/tests.py | 12 ++++++- ietf/stats/urls.py | 1 + ietf/stats/utils.py | 24 +++++--------- ietf/stats/views.py | 12 +++++++ ietf/submit/views.py | 4 +++ .../templates/stats/known_countries_list.html | 33 +++++++++++++++++++ ietf/templates/submit/submission_status.html | 27 ++++++++++++--- 8 files changed, 112 insertions(+), 22 deletions(-) create mode 100644 ietf/templates/stats/known_countries_list.html diff --git a/ietf/stats/migrations/0002_add_initial_aliases.py b/ietf/stats/migrations/0002_add_initial_aliases.py index b25cc152c..d317267d8 100644 --- a/ietf/stats/migrations/0002_add_initial_aliases.py +++ b/ietf/stats/migrations/0002_add_initial_aliases.py @@ -18,6 +18,26 @@ def add_affiliation_info(apps, schema_editor): AffiliationIgnoredEnding.objects.get_or_create(ending="GmbH\.?") CountryAlias = apps.get_model("stats", "CountryAlias") + for iso_country_code in ['AD', 'AE', 'AF', 'AG', 'AI', 'AL', 'AM', 'AO', 'AQ', 'AR', 'AS', 'AT', 'AU', 'AW', + 'AX', 'AZ', 'BA', 'BB', 'BD', 'BE', 'BF', 'BG', 'BH', 'BI', 'BJ', 'BL', 'BM', 'BN', + 'BO', 'BQ', 'BR', 'BS', 'BT', 'BV', 'BW', 'BY', 'BZ', 'CA', 'CC', 'CD', 'CF', 'CG', + 'CH', 'CI', 'CK', 'CL', 'CM', 'CN', 'CO', 'CR', 'CU', 'CV', 'CW', 'CX', 'CY', 'CZ', + 'DE', 'DJ', 'DK', 'DM', 'DO', 'DZ', 'EC', 'EE', 'EG', 'EH', 'ER', 'ES', 'ET', 'FI', + 'FJ', 'FK', 'FM', 'FO', 'FR', 'GA', 'GB', 'GD', 'GE', 'GF', 'GG', 'GH', 'GI', 'GL', + 'GM', 'GN', 'GP', 'GQ', 'GR', 'GS', 'GT', 'GU', 'GW', 'GY', 'HK', 'HM', 'HN', 'HR', + 'HT', 'HU', 'ID', 'IE', 'IL', 'IM', 'IN', 'IO', 'IQ', 'IR', 'IS', 'IT', 'JE', 'JM', + 'JO', 'JP', 'KE', 'KG', 'KH', 'KI', 'KM', 'KN', 'KP', 'KR', 'KW', 'KY', 'KZ', 'LA', + 'LB', 'LC', 'LI', 'LK', 'LR', 'LS', 'LT', 'LU', 'LV', 'LY', 'MA', 'MC', 'MD', 'ME', + 'MF', 'MG', 'MH', 'MK', 'ML', 'MM', 'MN', 'MO', 'MP', 'MQ', 'MR', 'MS', 'MT', 'MU', + 'MV', 'MW', 'MX', 'MY', 'MZ', 'NA', 'NC', 'NE', 'NF', 'NG', 'NI', 'NL', 'NO', 'NP', + 'NR', 'NU', 'NZ', 'OM', 'PA', 'PE', 'PF', 'PG', 'PH', 'PK', 'PL', 'PM', 'PN', 'PR', + 'PS', 'PT', 'PW', 'PY', 'QA', 'RE', 'RO', 'RS', 'RU', 'RW', 'SA', 'SB', 'SC', 'SD', + 'SE', 'SG', 'SH', 'SI', 'SJ', 'SK', 'SL', 'SM', 'SN', 'SO', 'SR', 'SS', 'ST', 'SV', + 'SX', 'SY', 'SZ', 'TC', 'TD', 'TF', 'TG', 'TH', 'TJ', 'TK', 'TL', 'TM', 'TN', 'TO', + 'TR', 'TT', 'TV', 'TW', 'TZ', 'UA', 'UG', 'UM', 'US', 'UY', 'UZ', 'VA', 'VC', 'VE', + 'VG', 'VI', 'VN', 'VU', 'WF', 'WS', 'YE', 'YT', 'ZA', 'ZM', 'ZW']: + CountryAlias.objects.get_or_create(alias=iso_country_code, country_id=iso_country_code) + CountryAlias.objects.get_or_create(alias="russian federation", country_id="RU") CountryAlias.objects.get_or_create(alias="p. r. china", country_id="CN") CountryAlias.objects.get_or_create(alias="p.r. china", country_id="CN") @@ -74,7 +94,6 @@ def add_affiliation_info(apps, schema_editor): CountryAlias.objects.get_or_create(alias="grand-duchy of luxembourg", country_id="LU") CountryAlias.objects.get_or_create(alias="brasil", country_id="BR") - class Migration(migrations.Migration): diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 1d0e1ef47..042d976a7 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -3,7 +3,7 @@ from pyquery import PyQuery from django.core.urlresolvers import reverse as urlreverse from ietf.utils.test_data import make_test_data, make_review_data -from ietf.utils.test_utils import login_testing_unauthorized, TestCase +from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent import ietf.stats.views class StatisticsTests(TestCase): @@ -39,6 +39,16 @@ class StatisticsTests(TestCase): self.assertTrue(q('#chart')) self.assertTrue(q('table.stats-data')) + def test_known_country_list(self): + make_test_data() + + # check redirect + url = urlreverse(ietf.stats.views.known_country_list) + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertTrue("United States" in unicontent(r)) + def test_review_stats(self): doc = make_test_data() review_req = make_review_data(doc) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index ee4fb4018..cb626c401 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -6,5 +6,6 @@ import ietf.stats.views urlpatterns = patterns('', url("^$", ietf.stats.views.stats_index), url("^document/(?:(?Pauthors|pages|words|format|formlang|author/documents|author/affiliation|author/country|author/continent|author/citation)/)?$", ietf.stats.views.document_stats), + url("^knowncountries/$", ietf.stats.views.known_countries_list), url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, ietf.stats.views.review_stats), ) diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py index d2e05fffe..78468dff5 100644 --- a/ietf/stats/utils.py +++ b/ietf/stats/utils.py @@ -84,20 +84,12 @@ def get_aliased_affiliations(affiliations): return res - - def get_aliased_countries(countries): known_aliases = dict(CountryAlias.objects.values_list("alias", "country__name")) - iso_code_aliases = {} - # add aliases for known countries for slug, name in CountryName.objects.values_list("slug", "name"): - if len(name) > 2: - known_aliases[name.lower()] = name - - if len(slug) == 2 and slug[0].isupper() and slug[1].isupper(): - iso_code_aliases[slug] = name # add ISO code + known_aliases[name.lower()] = name def lookup_alias(possible_alias): name = known_aliases.get(possible_alias) @@ -185,14 +177,16 @@ def get_aliased_countries(countries): if found: continue - # if everything else has failed, try ISO code - country = iso_code_aliases.get(country, country) - if country in known_countries: - res[original_country] = country - continue - # unknown country res[original_country] = "" return res + +def clean_country_name(country_name): + if country_name: + country_name = get_aliased_countries([country_name]).get(country_name, country_name) + if country_name and CountryName.objects.filter(name=country_name).exists(): + return country_name + + return "" diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 1275fbd65..3a0b0c61f 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -526,6 +526,18 @@ def document_stats(request, stats_type=None): }) +def known_countries_list(request, stats_type=None, acronym=None): + countries = CountryName.objects.prefetch_related("countryalias_set") + for c in countries: + # the sorting is a bit of a hack - it puts the ISO code first + # since it was added in a migration + c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) + + return render(request, "stats/known_countries_list.html", { + "countries": countries + }) + + @login_required def review_stats(request, stats_type=None, acronym=None): # This view is a bit complex because we want to show a bunch of diff --git a/ietf/submit/views.py b/ietf/submit/views.py index ed49fc836..368a0cbb8 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -31,6 +31,7 @@ from ietf.submit.utils import ( approvable_submissions_for_user, preapprovals_fo recently_approved_by_user, validate_submission, create_submission_event, docevent_from_submission, post_submission, cancel_submission, rename_submission_files, get_person_from_name_email ) +from ietf.stats.utils import clean_country_name from ietf.utils.accesstoken import generate_random_key, generate_access_token from ietf.utils.draft import Draft from ietf.utils.log import log @@ -401,6 +402,9 @@ def submission_status(request, submission_id, access_token=None): # something went wrong, turn this into a GET and let the user deal with it return HttpResponseRedirect("") + for author in submission.authors: + author["cleaned_country"] = clean_country_name(author.get("country")) + return render(request, 'submit/submission_status.html', { 'selected': 'status', 'submission': submission, diff --git a/ietf/templates/stats/known_countries_list.html b/ietf/templates/stats/known_countries_list.html new file mode 100644 index 000000000..2404af9ef --- /dev/null +++ b/ietf/templates/stats/known_countries_list.html @@ -0,0 +1,33 @@ +{% extends "base.html" %} + +{% load origin %} + +{% load ietf_filters staticfiles bootstrap3 %} + +{% block content %} + {% origin %} + +

{% block title %}Countries known to the Datatracker{% endblock %}

+ + + + + + + + + + {% for c in countries %} + + + + + {% endfor %} + +
NameAliases (lowercase aliases are matched case-insensitive)
{{ c.name }} + {% for a in c.aliases %} + {{ a.alias }}{% if not forloop.last %},{% endif %} + {% endfor %} +
+ +{% endblock %} diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index 39612397d..d05ea0468 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -195,9 +195,7 @@ Authors - {% with submission.authors as authors %} - {{ authors|length }} author{{ authors|pluralize }} - {% endwith %} + {{ submission.authors|length }} author{{ submission.authors|pluralize }} {% if errors.authors %}

{{ errors.authors|safe }}

{% endif %} @@ -207,8 +205,27 @@ Author {{ forloop.counter }} {{ author.name }} {% if author.email %}<{{ author.email }}>{% endif %} - - {% if author.affiliation %}{{ author.affiliation }}{% else %}unknown affiliation{% endif %} - - {% if author.country %}{{ author.country }}{% else %}unknown country{% endif %} + - + {% if author.affiliation %} + {{ author.affiliation }} + {% else %} + unknown affiliation + {% endif %} + - + {% if author.country %} + {{ author.country }} + {% if author.cleaned_country and author.country != author.cleaned_country %} + (understood to be {{ author.cleaned_country }}) + {% endif %} + {% else %} + unknown country + {% endif %} + + {% if author.country and not author.cleaned_country %} +
+ Unrecognized country: "{{ author.country }}": Please use a recognized country name. + {% endif %} + {% endfor %} From 32c32a5c076849b9063afeaf3ee52ec313368944 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Thu, 16 Feb 2017 18:42:54 +0000 Subject: [PATCH 38/49] Fix a couple of bugs in the document statistics - Legacy-Id: 12863 --- ietf/stats/views.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 3a0b0c61f..5014703bb 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -145,13 +145,13 @@ def document_stats(request, stats_type=None): if any(stats_type == t[0] for t in possible_document_stats_types): # filter documents - docalias_qs = DocAlias.objects.filter(document__type="draft") + docalias_filters = Q(document__type="draft") rfc_state = State.objects.get(type="draft", slug="rfc") if document_type == "rfc": - docalias_qs = docalias_qs.filter(document__states=rfc_state) + docalias_filters &= Q(document__states=rfc_state) elif document_type == "draft": - docalias_qs = docalias_qs.exclude(document__states=rfc_state) + docalias_filters &= ~Q(document__states=rfc_state) if from_time: # this is actually faster than joining in the database, @@ -162,7 +162,9 @@ def document_stats(request, stats_type=None): docevent__type__in=["published_rfc", "new_revision"], ).values_list("pk")) - docalias_qs = docalias_qs.filter(document__in=docs_within_time_constraint) + docalias_filters &= Q(document__in=docs_within_time_constraint) + + docalias_qs = DocAlias.objects.filter(docalias_filters) if document_type == "rfc": doc_label = "RFC" @@ -171,7 +173,7 @@ def document_stats(request, stats_type=None): else: doc_label = "document" - total_docs = docalias_qs.count() + total_docs = docalias_qs.values_list("document").distinct().count() def generate_canonical_names(docalias_qs): for doc_id, ts in itertools.groupby(docalias_qs.order_by("document"), lambda t: t[0]): @@ -180,9 +182,9 @@ def document_stats(request, stats_type=None): if chosen is None: chosen = t else: - if t[0].startswith("rfc"): + if t[1].startswith("rfc"): chosen = t - elif t[0].startswith("draft") and not chosen[0].startswith("rfc"): + elif t[1].startswith("draft") and not chosen[1].startswith("rfc"): chosen = t yield chosen @@ -192,8 +194,8 @@ def document_stats(request, stats_type=None): bins = defaultdict(list) - for name, author_count in generate_canonical_names(docalias_qs.values_list("name").annotate(Count("document__documentauthor"))): - bins[author_count].append(name) + for name, canonical_name, author_count in generate_canonical_names(docalias_qs.values_list("document", "name").annotate(Count("document__documentauthor"))): + bins[author_count].append(canonical_name) series_data = [] for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): @@ -211,8 +213,8 @@ def document_stats(request, stats_type=None): bins = defaultdict(list) - for name, pages in generate_canonical_names(docalias_qs.values_list("name", "document__pages")): - bins[pages].append(name) + for name, canonical_name, pages in generate_canonical_names(docalias_qs.values_list("document", "name", "document__pages")): + bins[pages].append(canonical_name) series_data = [] for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]): @@ -233,8 +235,8 @@ def document_stats(request, stats_type=None): bins = defaultdict(list) - for name, words in generate_canonical_names(docalias_qs.values_list("name", "document__words")): - bins[put_into_bin(words, bin_size)].append(name) + for name, canonical_name, words in generate_canonical_names(docalias_qs.values_list("document", "name", "document__words")): + bins[put_into_bin(words, bin_size)].append(canonical_name) series_data = [] for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]): @@ -261,7 +263,7 @@ def document_stats(request, stats_type=None): submission_types[doc_name] = file_types doc_names_with_missing_types = {} - for canonical_name, rev, doc_name in generate_canonical_names(docalias_qs.values_list("name", "document__rev", "document__name")): + for doc_name, canonical_name, rev in generate_canonical_names(docalias_qs.values_list("document", "name", "document__rev")): types = submission_types.get(doc_name) if types: for dot_ext in types.split(","): @@ -308,8 +310,8 @@ def document_stats(request, stats_type=None): bins = defaultdict(list) - for name, formal_language_name in generate_canonical_names(docalias_qs.values_list("name", "document__formal_languages__name")): - bins[formal_language_name].append(name) + for name, canonical_name, formal_language_name in generate_canonical_names(docalias_qs.values_list("document", "name", "document__formal_languages__name")): + bins[formal_language_name].append(canonical_name) series_data = [] for formal_language, names in sorted(bins.iteritems(), key=lambda t: t[0]): From 6964337853bfda8236872274b92232dacf5c6852 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Thu, 16 Feb 2017 18:45:46 +0000 Subject: [PATCH 39/49] Move bug fixes - Legacy-Id: 12864 --- ietf/stats/tests.py | 2 +- ietf/stats/views.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 042d976a7..fbb9d4e48 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -43,7 +43,7 @@ class StatisticsTests(TestCase): make_test_data() # check redirect - url = urlreverse(ietf.stats.views.known_country_list) + url = urlreverse(ietf.stats.views.known_countries_list) r = self.client.get(url) self.assertEqual(r.status_code, 200) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 5014703bb..f8e2bb305 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -199,7 +199,7 @@ def document_stats(request, stats_type=None): series_data = [] for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / total_docs + percentage = len(names) * 100.0 / (total_docs or 1) series_data.append((author_count, percentage)) table_data.append((author_count, percentage, names)) @@ -218,7 +218,7 @@ def document_stats(request, stats_type=None): series_data = [] for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / total_docs + percentage = len(names) * 100.0 / (total_docs or 1) if pages is not None: series_data.append((pages, len(names))) table_data.append((pages, percentage, names)) @@ -240,7 +240,7 @@ def document_stats(request, stats_type=None): series_data = [] for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]): - percentage = len(names) * 100.0 / total_docs + percentage = len(names) * 100.0 / (total_docs or 1) if words is not None: series_data.append((value, len(names))) @@ -295,7 +295,7 @@ def document_stats(request, stats_type=None): series_data = [] for fmt, names in sorted(bins.iteritems(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / total_docs + percentage = len(names) * 100.0 / (total_docs or 1) series_data.append((fmt, len(names))) table_data.append((fmt, percentage, names)) @@ -315,7 +315,7 @@ def document_stats(request, stats_type=None): series_data = [] for formal_language, names in sorted(bins.iteritems(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / total_docs + percentage = len(names) * 100.0 / (total_docs or 1) if formal_language is not None: series_data.append((formal_language, len(names))) table_data.append((formal_language, percentage, names)) @@ -376,7 +376,7 @@ def document_stats(request, stats_type=None): series_data = [] for document_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): - percentage = len(names) * 100.0 / total_persons + percentage = len(names) * 100.0 / (total_persons or 1) series_data.append((document_count, percentage)) table_data.append((document_count, percentage, names)) @@ -407,7 +407,7 @@ def document_stats(request, stats_type=None): series_data = [] for affiliation, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): - percentage = len(names) * 100.0 / total_persons + percentage = len(names) * 100.0 / (total_persons or 1) if affiliation: series_data.append((affiliation, len(names))) table_data.append((affiliation, percentage, names)) @@ -454,7 +454,7 @@ def document_stats(request, stats_type=None): series_data = [] for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): - percentage = len(names) * 100.0 / total_persons + percentage = len(names) * 100.0 / (total_persons or 1) if country: series_data.append((country, len(names))) table_data.append((country, percentage, names)) @@ -494,7 +494,7 @@ def document_stats(request, stats_type=None): series_data = [] for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): - percentage = len(names) * 100.0 / total_persons + percentage = len(names) * 100.0 / (total_persons or 1) if continent: series_data.append((continent, len(names))) table_data.append((continent, percentage, names)) From d5e98c9644d58ab9504c2740891d22b60549c611 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 17 Feb 2017 10:21:34 +0000 Subject: [PATCH 40/49] Add mail to link to Secretariat on country page in case people are missing a country or alias, adjust wording a bit on status page. - Legacy-Id: 12866 --- ietf/settings.py | 2 ++ ietf/stats/models.py | 2 +- ietf/stats/utils.py | 15 +++++++++++++++ ietf/stats/views.py | 3 ++- ietf/templates/stats/known_countries_list.html | 16 +++++++++++++++- ietf/templates/submit/submission_status.html | 2 +- 6 files changed, 36 insertions(+), 4 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index 6f815617f..ea3681dda 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -518,6 +518,8 @@ IPR_EMAIL_FROM = 'ietf-ipr@ietf.org' IANA_EVAL_EMAIL = "drafts-eval@icann.org" +SECRETARIAT_TICKET_EMAIL = "ietf-action@ietf.org" + # Put real password in settings_local.py IANA_SYNC_PASSWORD = "secret" IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" diff --git a/ietf/stats/models.py b/ietf/stats/models.py index 875a81afa..9aa919b46 100644 --- a/ietf/stats/models.py +++ b/ietf/stats/models.py @@ -30,7 +30,7 @@ class CountryAlias(models.Model): """Records that alias should be treated as country for statistical purposes.""" - alias = models.CharField(max_length=255, help_text="Note that lower-case aliases are matched case-insensitive while aliases with at least one uppercase letter is matched case-sensitive.") + alias = models.CharField(max_length=255, help_text="Note that lower-case aliases are matched case-insensitive while aliases with at least one uppercase letter is matched case-sensitive. So 'United States' is best entered as 'united states' so it both matches 'United States' and 'United states' and 'UNITED STATES', whereas 'US' is best entered as 'US' so it doesn't accidentally match an ordinary word like 'us'.") country = models.ForeignKey(CountryName, max_length=255) def __unicode__(self): diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py index 78468dff5..1108417f7 100644 --- a/ietf/stats/utils.py +++ b/ietf/stats/utils.py @@ -190,3 +190,18 @@ def clean_country_name(country_name): return country_name return "" + + +def compute_hirsch_index(citation_counts): + """Computes the h-index given a sequence containing the number of + citations for each document.""" + + i = 0 + + for count in sorted(citation_counts, reverse=True): + if i + 1 > count: + break + + i += 1 + + return i diff --git a/ietf/stats/views.py b/ietf/stats/views.py index f8e2bb305..268cc9505 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -536,7 +536,8 @@ def known_countries_list(request, stats_type=None, acronym=None): c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) return render(request, "stats/known_countries_list.html", { - "countries": countries + "countries": countries, + "ticket_email_address": settings.SECRETARIAT_TICKET_EMAIL, }) diff --git a/ietf/templates/stats/known_countries_list.html b/ietf/templates/stats/known_countries_list.html index 2404af9ef..37f4fa8cb 100644 --- a/ietf/templates/stats/known_countries_list.html +++ b/ietf/templates/stats/known_countries_list.html @@ -9,6 +9,12 @@

{% block title %}Countries known to the Datatracker{% endblock %}

+

In case you think a country or an alias is missing from the list, you can file a ticket.

+ + {% if request.user.is_staff %} +

Note: since you're an admin, the country names are linked to their corresponding admin page.

+ {% endif %} + @@ -19,7 +25,15 @@ {% for c in countries %} - + From 4d8dd3ee4f69e1ed8b2aed329a67bb1fe755daf5 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 17 Feb 2017 17:17:53 +0000 Subject: [PATCH 41/49] Format rfc123 -> RFC 123 on references pages - Legacy-Id: 12867 --- ietf/doc/templatetags/ietf_filters.py | 5 +++++ ietf/templates/doc/document_referenced_by.html | 6 +++--- ietf/templates/doc/document_references.html | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 2ebc8e6fe..e88a0e5e2 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -191,6 +191,11 @@ def rfcnospace(string): else: return string +@register.filter +def prettystdname(string): + from ietf.doc.utils import prettify_std_name + return prettify_std_name(unicode(string or "")) + @register.filter(name='rfcurl') def rfclink(string): """ diff --git a/ietf/templates/doc/document_referenced_by.html b/ietf/templates/doc/document_referenced_by.html index ca2171749..619db053b 100644 --- a/ietf/templates/doc/document_referenced_by.html +++ b/ietf/templates/doc/document_referenced_by.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin staticfiles %} +{% load origin staticfiles ietf_filters %} {% block pagehead %} @@ -44,7 +44,7 @@ {% with ref.source.canonical_name as name %} - + - + {% endfor %} diff --git a/ietf/templates/stats/document_stats_author_hindex.html b/ietf/templates/stats/document_stats_author_hindex.html new file mode 100644 index 000000000..d5d67329c --- /dev/null +++ b/ietf/templates/stats/document_stats_author_hindex.html @@ -0,0 +1,74 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ +
{{ c.name }} + {% if request.user.is_staff %} + + {% endif %} + {{ c.name }} + {% if request.user.is_staff %} + + {% endif %} + {% for a in c.aliases %} {{ a.alias }}{% if not forloop.last %},{% endif %} diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index d05ea0468..473829130 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -223,7 +223,7 @@ {% if author.country and not author.cleaned_country %}
- Unrecognized country: "{{ author.country }}": Please use a recognized country name. + Unrecognized country: "{{ author.country }}": See recognized country names. {% endif %}
- {{name}} + {{name|prettystdname}} {% if ref.target.name != alias_name %}
As {{ref.target.name}} {% endif %} @@ -72,4 +72,4 @@ {% block js %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/doc/document_references.html b/ietf/templates/doc/document_references.html index 0d1feec97..e98f3755b 100644 --- a/ietf/templates/doc/document_references.html +++ b/ietf/templates/doc/document_references.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin staticfiles %} +{% load origin staticfiles ietf_filters %} {% block pagehead %} @@ -35,7 +35,7 @@ {% for ref in refs %} {% with ref.target.name as name %}
{{name}}{{name|prettystdname}} {{ref.target.document.title}}
Refs @@ -59,4 +59,4 @@ {% block js %} -{% endblock %} \ No newline at end of file +{% endblock %} From f180147cbdbd0d82152b75633b139dbf0a38ec08 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 17 Feb 2017 17:38:11 +0000 Subject: [PATCH 42/49] Put chart height in the CSS to prevent flickering when loading a stats page - Legacy-Id: 12868 --- ietf/static/ietf/css/ietf.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ietf/static/ietf/css/ietf.css b/ietf/static/ietf/css/ietf.css index f095e9eb7..67a18b1f6 100644 --- a/ietf/static/ietf/css/ietf.css +++ b/ietf/static/ietf/css/ietf.css @@ -573,6 +573,10 @@ table.simple-table td:last-child { text-indent: -1em; } +.document-stats #chart { + height: 25em; +} + .stats-time-graph { height: 15em; } From c61babb418e57c3427239ea61354fd3c1f000078 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Fri, 17 Feb 2017 17:43:14 +0000 Subject: [PATCH 43/49] Add citation and h-index statistics - Legacy-Id: 12869 --- ietf/person/models.py | 5 +- ietf/person/name.py | 4 + ietf/stats/urls.py | 2 +- ietf/stats/views.py | 79 +++++++++++++++++-- .../document_stats_author_citations.html | 66 ++++++++++++++++ .../document_stats_author_documents.html | 2 +- .../stats/document_stats_author_hindex.html | 74 +++++++++++++++++ .../includes/number_with_details_cell.html | 8 +- 8 files changed, 226 insertions(+), 14 deletions(-) create mode 100644 ietf/templates/stats/document_stats_author_citations.html create mode 100644 ietf/templates/stats/document_stats_author_hindex.html diff --git a/ietf/person/models.py b/ietf/person/models.py index 61fa6b2c0..9ee992930 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -14,7 +14,7 @@ from django.utils.text import slugify import debug # pyflakes:ignore -from ietf.person.name import name_parts, initials +from ietf.person.name import name_parts, initials, plain_name from ietf.utils.mail import send_mail_preformatted from ietf.utils.storage import NoLocationMigrationFileSystemStorage @@ -47,8 +47,7 @@ class PersonInfo(models.Model): return (first and first[0]+"." or "")+(middle or "")+" "+last+(suffix and " "+suffix or "") def plain_name(self): if not hasattr(self, '_cached_plain_name'): - prefix, first, middle, last, suffix = name_parts(self.name) - self._cached_plain_name = u" ".join([first, last]) + self._cached_plain_name = plain_name(self.name) return self._cached_plain_name def ascii_name(self): if not hasattr(self, '_cached_ascii_name'): diff --git a/ietf/person/name.py b/ietf/person/name.py index 997f3def8..5337d1a1e 100644 --- a/ietf/person/name.py +++ b/ietf/person/name.py @@ -50,6 +50,10 @@ def initials(name): initials = u" ".join([ n[0]+'.' for n in given.split() ]) return initials +def plain_name(name): + prefix, first, middle, last, suffix = name_parts(name) + return u" ".join([first, last]) + if __name__ == "__main__": import sys name = u" ".join(sys.argv[1:]) diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index cb626c401..9ae43f60e 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -5,7 +5,7 @@ import ietf.stats.views urlpatterns = patterns('', url("^$", ietf.stats.views.stats_index), - url("^document/(?:(?Pauthors|pages|words|format|formlang|author/documents|author/affiliation|author/country|author/continent|author/citation)/)?$", ietf.stats.views.document_stats), + url("^document/(?:(?Pauthors|pages|words|format|formlang|author/documents|author/affiliation|author/country|author/continent|author/citations||author/hindex)/)?$", ietf.stats.views.document_stats), url("^knowncountries/$", ietf.stats.views.known_countries_list), url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, ietf.stats.views.review_stats), ) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 268cc9505..26439a275 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -23,9 +23,10 @@ from ietf.review.utils import (extract_review_request_data, from ietf.submit.models import Submission from ietf.group.models import Role, Group from ietf.person.models import Person -from ietf.name.models import ReviewRequestStateName, ReviewResultName, CountryName +from ietf.name.models import ReviewRequestStateName, ReviewResultName, CountryName, DocRelationshipName +from ietf.person.name import plain_name from ietf.doc.models import DocAlias, Document, State -from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries +from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries, compute_hirsch_index from ietf.ietfauth.utils import has_role def stats_index(request): @@ -103,7 +104,8 @@ def document_stats(request, stats_type=None): ("author/affiliation", "Affiliation"), ("author/country", "Country"), ("author/continent", "Continent"), - ("author/citation", "Citations"), + ("author/citations", "Citations"), + ("author/hindex", "Impact"), ], lambda slug: build_document_stats_url(stats_type_override=slug)) @@ -346,7 +348,7 @@ def document_stats(request, stats_type=None): person_filters &= Q(documentauthor__document__in=docs_within_time_constraint) - person_qs = Person.objects.filter(person_filters) + person_qs = Person.objects.filter(person_filters, documentauthor__document="draft-arkko-dual-stack-extra-lite") if document_type == "rfc": doc_label = "RFC" @@ -369,6 +371,8 @@ def document_stats(request, stats_type=None): bins = defaultdict(list) + person_qs = Person.objects.filter(person_filters) + for name, document_count in person_qs.values_list("name").annotate(Count("documentauthor")): bins[document_count].append(name) @@ -378,7 +382,7 @@ def document_stats(request, stats_type=None): for document_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): percentage = len(names) * 100.0 / (total_persons or 1) series_data.append((document_count, percentage)) - table_data.append((document_count, percentage, names)) + table_data.append((document_count, percentage, [plain_name(n) for n in names])) chart_data.append({ "data": series_data, @@ -390,6 +394,8 @@ def document_stats(request, stats_type=None): bins = defaultdict(list) + person_qs = Person.objects.filter(person_filters) + # Since people don't write the affiliation names in the # same way, and we don't want to go back and edit them # either, we transform them here. @@ -410,7 +416,7 @@ def document_stats(request, stats_type=None): percentage = len(names) * 100.0 / (total_persons or 1) if affiliation: series_data.append((affiliation, len(names))) - table_data.append((affiliation, percentage, names)) + table_data.append((affiliation, percentage, [plain_name(n) for n in names])) series_data.sort(key=lambda t: t[1], reverse=True) series_data = series_data[:30] @@ -428,6 +434,8 @@ def document_stats(request, stats_type=None): bins = defaultdict(list) + person_qs = Person.objects.filter(person_filters) + # Since people don't write the country names in the # same way, and we don't want to go back and edit them # either, we transform them here. @@ -457,7 +465,7 @@ def document_stats(request, stats_type=None): percentage = len(names) * 100.0 / (total_persons or 1) if country: series_data.append((country, len(names))) - table_data.append((country, percentage, names)) + table_data.append((country, percentage, [plain_name(n) for n in names])) series_data.sort(key=lambda t: t[1], reverse=True) series_data = series_data[:30] @@ -477,6 +485,8 @@ def document_stats(request, stats_type=None): bins = defaultdict(list) + person_qs = Person.objects.filter(person_filters) + name_country_set = set((name, country) for name, country in person_qs.values_list("name", "documentauthor__country")) @@ -497,7 +507,7 @@ def document_stats(request, stats_type=None): percentage = len(names) * 100.0 / (total_persons or 1) if continent: series_data.append((continent, len(names))) - table_data.append((continent, percentage, names)) + table_data.append((continent, percentage, [plain_name(n) for n in names])) series_data.sort(key=lambda t: t[1], reverse=True) @@ -506,6 +516,59 @@ def document_stats(request, stats_type=None): "animation": False, }) + elif stats_type == "author/citations": + stats_title = "Number of citations of {}s written by author".format(doc_label) + + bins = defaultdict(list) + + cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold'])) + person_filters &= Q(documentauthor__document__docalias__relateddocument__relationship__in=cite_relationships) + + person_qs = Person.objects.filter(person_filters) + + for name, citations in person_qs.values_list("name").annotate(Count("documentauthor__document__docalias__relateddocument")): + bins[citations].append(name) + + total_persons = count_bins(bins) + + series_data = [] + for citations, names in sorted(bins.iteritems(), key=lambda t: t[0], reverse=True): + percentage = len(names) * 100.0 / (total_persons or 1) + series_data.append((citations, percentage)) + table_data.append((citations, percentage, [plain_name(n) for n in names])) + + chart_data.append({ + "data": sorted(series_data, key=lambda t: t[0]), + "animation": False, + }) + + elif stats_type == "author/hindex": + stats_title = "h-index for {}s written by author".format(doc_label) + + bins = defaultdict(list) + + cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold'])) + person_filters &= Q(documentauthor__document__docalias__relateddocument__relationship__in=cite_relationships) + + person_qs = Person.objects.filter(person_filters) + + values = person_qs.values_list("name", "documentauthor__document").annotate(Count("documentauthor__document__docalias__relateddocument")) + for name, ts in itertools.groupby(values.order_by("name"), key=lambda t: t[0]): + h_index = compute_hirsch_index([citations for _, document, citations in ts]) + bins[h_index].append(name) + + total_persons = count_bins(bins) + + series_data = [] + for citations, names in sorted(bins.iteritems(), key=lambda t: t[0], reverse=True): + percentage = len(names) * 100.0 / (total_persons or 1) + series_data.append((citations, percentage)) + table_data.append((citations, percentage, [plain_name(n) for n in names])) + + chart_data.append({ + "data": sorted(series_data, key=lambda t: t[0]), + "animation": False, + }) return render(request, "stats/document_stats.html", { "chart_data": mark_safe(json.dumps(chart_data)), diff --git a/ietf/templates/stats/document_stats_author_citations.html b/ietf/templates/stats/document_stats_author_citations.html new file mode 100644 index 000000000..bcb3cff9e --- /dev/null +++ b/ietf/templates/stats/document_stats_author_citations.html @@ -0,0 +1,66 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for citations, percentage, names in table_data %} + + + + + + {% endfor %} + +
CitationsPercentage of authorsAuthors
{{ citations }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}
+ +

Note that the citation counts do not exclude self-references.

diff --git a/ietf/templates/stats/document_stats_author_documents.html b/ietf/templates/stats/document_stats_author_documents.html index 0d21b41d2..025e8c26f 100644 --- a/ietf/templates/stats/document_stats_author_documents.html +++ b/ietf/templates/stats/document_stats_author_documents.html @@ -58,7 +58,7 @@
{{ document_count }} {{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" %}{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}
+ + + + + + + + + {% for h_index, percentage, names in table_data %} + + + + + + {% endfor %} + +
h-indexPercentage of authorsAuthors
{{ h_index }}{{ percentage|floatformat:2 }}%{% include "stats/includes/number_with_details_cell.html" with content_limit=25 %}
+ +

Hirsch index or h-index is a + measure of the + productivity and impact of the publications of an author. An + author with an h-index of 5 has had 5 publications each cited at + least 5 times - to increase the index to 6, the 5 publications plus + 1 more would have to have been cited at least 6 times, each. Thus a + high h-index requires many highly-cited publications.

+ +

Note that the h-index calculations do not exclude self-references.

diff --git a/ietf/templates/stats/includes/number_with_details_cell.html b/ietf/templates/stats/includes/number_with_details_cell.html index cdadc287a..2dac07977 100644 --- a/ietf/templates/stats/includes/number_with_details_cell.html +++ b/ietf/templates/stats/includes/number_with_details_cell.html @@ -1 +1,7 @@ -{{ names|length }} +{% if content_limit and names|length <= content_limit %} + {% for n in names %} + {{ n }}
+ {% endfor %} +{% else %} + {{ names|length }} +{% endif %} From 33e9577967ab6a6101ca65a6c2ca3ba9255ba7a2 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Tue, 21 Feb 2017 17:07:38 +0000 Subject: [PATCH 44/49] Add yearly charts for affiliation, country and continent, fix some bugs - Legacy-Id: 12894 --- ietf/stats/tests.py | 7 +- ietf/stats/urls.py | 2 +- ietf/stats/views.py | 263 +++++++++++++----- ietf/templates/stats/document_stats.html | 10 + .../document_stats_author_affiliation.html | 5 + .../document_stats_author_citations.html | 5 + .../document_stats_author_continent.html | 5 + .../stats/document_stats_author_country.html | 5 + .../document_stats_author_documents.html | 5 + .../stats/document_stats_author_hindex.html | 5 + .../stats/document_stats_authors.html | 5 + .../stats/document_stats_format.html | 5 + .../stats/document_stats_formlang.html | 5 + .../templates/stats/document_stats_pages.html | 5 + .../templates/stats/document_stats_words.html | 5 + 15 files changed, 265 insertions(+), 72 deletions(-) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index fbb9d4e48..62e4e048d 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -26,7 +26,9 @@ class StatisticsTests(TestCase): # check various stats types for stats_type in ["authors", "pages", "words", "format", "formlang", - "author/documents", "author/affiliation", "author/country", "author/continent"]: + "author/documents", "author/affiliation", "author/country", + "author/continent", "author/citations", "author/hindex", + "yearly/affiliation", "yearly/country", "yearly/continent"]: for document_type in ["", "rfc", "draft"]: for time_choice in ["", "5y"]: url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type }) @@ -37,7 +39,8 @@ class StatisticsTests(TestCase): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('#chart')) - self.assertTrue(q('table.stats-data')) + if not stats_type.startswith("yearly"): + self.assertTrue(q('table.stats-data')) def test_known_country_list(self): make_test_data() diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 9ae43f60e..ea802b952 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -5,7 +5,7 @@ import ietf.stats.views urlpatterns = patterns('', url("^$", ietf.stats.views.stats_index), - url("^document/(?:(?Pauthors|pages|words|format|formlang|author/documents|author/affiliation|author/country|author/continent|author/citations||author/hindex)/)?$", ietf.stats.views.document_stats), + url("^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", ietf.stats.views.document_stats), url("^knowncountries/$", ietf.stats.views.known_countries_list), url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, ietf.stats.views.review_stats), ) diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 26439a275..6a5b4dba6 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -25,7 +25,7 @@ from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewRequestStateName, ReviewResultName, CountryName, DocRelationshipName from ietf.person.name import plain_name -from ietf.doc.models import DocAlias, Document, State +from ietf.doc.models import DocAlias, Document, State, DocEvent from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries, compute_hirsch_index from ietf.ietfauth.utils import has_role @@ -81,6 +81,15 @@ def put_into_bin(value, bin_size): v = (value // bin_size) * bin_size return (v, "{} - {}".format(v, v + bin_size - 1)) +def prune_unknown_bin_with_known(bins): + # remove from the unknown bin all authors within the + # named/known bins + all_known = { n for b, names in bins.iteritems() if b for n in names } + bins[""] = [name for name in bins[""] if name not in all_known] + +def count_bins(bins): + return len({ n for b, names in bins.iteritems() if b for n in names }) + def document_stats(request, stats_type=None): def build_document_stats_url(stats_type_override=Ellipsis, get_overrides={}): kwargs = { @@ -89,7 +98,7 @@ def document_stats(request, stats_type=None): return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) - # statistics type - one of the tables or the chart + # statistics types possible_document_stats_types = add_url_to_choices([ ("authors", "Number of authors"), ("pages", "Pages"), @@ -98,17 +107,22 @@ def document_stats(request, stats_type=None): ("formlang", "Formal languages"), ], lambda slug: build_document_stats_url(stats_type_override=slug)) - # statistics type - one of the tables or the chart possible_author_stats_types = add_url_to_choices([ ("author/documents", "Number of documents"), ("author/affiliation", "Affiliation"), ("author/country", "Country"), ("author/continent", "Continent"), ("author/citations", "Citations"), - ("author/hindex", "Impact"), + ("author/hindex", "h-index"), ], lambda slug: build_document_stats_url(stats_type_override=slug)) - + possible_yearly_stats_types = add_url_to_choices([ + ("yearly/affiliation", "Affiliation"), + ("yearly/country", "Country"), + ("yearly/continent", "Continent"), + ], lambda slug: build_document_stats_url(stats_type_override=slug)) + + if not stats_type: return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_document_stats_types[0][0])) @@ -132,14 +146,15 @@ def document_stats(request, stats_type=None): from_time = None if "y" in time_choice: try: - years = int(time_choice.rstrip("y")) - from_time = datetime.datetime.today() - dateutil.relativedelta.relativedelta(years=years) + y = int(time_choice.rstrip("y")) + from_time = datetime.datetime.today() - dateutil.relativedelta.relativedelta(years=y) except ValueError: pass chart_data = [] table_data = [] stats_title = "" + template_name = stats_type.replace("/", "_") bin_size = 1 alias_data = [] eu_countries = None @@ -205,10 +220,7 @@ def document_stats(request, stats_type=None): series_data.append((author_count, percentage)) table_data.append((author_count, percentage, names)) - chart_data.append({ - "data": series_data, - "animation": False, - }) + chart_data.append({ "data": series_data }) elif stats_type == "pages": stats_title = "Number of pages for each {}".format(doc_label) @@ -225,10 +237,7 @@ def document_stats(request, stats_type=None): series_data.append((pages, len(names))) table_data.append((pages, percentage, names)) - chart_data.append({ - "data": series_data, - "animation": False, - }) + chart_data.append({ "data": series_data }) elif stats_type == "words": stats_title = "Number of words for each {}".format(doc_label) @@ -248,10 +257,7 @@ def document_stats(request, stats_type=None): table_data.append((words, percentage, names)) - chart_data.append({ - "data": series_data, - "animation": False, - }) + chart_data.append({ "data": series_data }) elif stats_type == "format": stats_title = "Submission formats for each {}".format(doc_label) @@ -302,10 +308,7 @@ def document_stats(request, stats_type=None): table_data.append((fmt, percentage, names)) - chart_data.append({ - "data": series_data, - "animation": False, - }) + chart_data.append({ "data": series_data }) elif stats_type == "formlang": stats_title = "Formal languages used for each {}".format(doc_label) @@ -322,10 +325,7 @@ def document_stats(request, stats_type=None): series_data.append((formal_language, len(names))) table_data.append((formal_language, percentage, names)) - chart_data.append({ - "data": series_data, - "animation": False, - }) + chart_data.append({ "data": series_data }) elif any(stats_type == t[0] for t in possible_author_stats_types): person_filters = Q(documentauthor__document__type="draft") @@ -348,7 +348,7 @@ def document_stats(request, stats_type=None): person_filters &= Q(documentauthor__document__in=docs_within_time_constraint) - person_qs = Person.objects.filter(person_filters, documentauthor__document="draft-arkko-dual-stack-extra-lite") + person_qs = Person.objects.filter(person_filters) if document_type == "rfc": doc_label = "RFC" @@ -357,15 +357,6 @@ def document_stats(request, stats_type=None): else: doc_label = "document" - def prune_unknown_bin_with_known(bins): - # remove from the unknown bin all authors within the - # named/known bins - all_known = set(n for b, names in bins.iteritems() if b for n in names) - bins[""] = [name for name in bins[""] if name not in all_known] - - def count_bins(bins): - return len(set(n for b, names in bins.iteritems() if b for n in names)) - if stats_type == "author/documents": stats_title = "Number of {}s per author".format(doc_label) @@ -384,10 +375,7 @@ def document_stats(request, stats_type=None): series_data.append((document_count, percentage)) table_data.append((document_count, percentage, [plain_name(n) for n in names])) - chart_data.append({ - "data": series_data, - "animation": False, - }) + chart_data.append({ "data": series_data }) elif stats_type == "author/affiliation": stats_title = "Number of {} authors per affiliation".format(doc_label) @@ -400,8 +388,10 @@ def document_stats(request, stats_type=None): # same way, and we don't want to go back and edit them # either, we transform them here. - name_affiliation_set = set((name, affiliation) - for name, affiliation in person_qs.values_list("name", "documentauthor__affiliation")) + name_affiliation_set = { + (name, affiliation) + for name, affiliation in person_qs.values_list("name", "documentauthor__affiliation") + } aliases = get_aliased_affiliations(affiliation for _, affiliation in name_affiliation_set) @@ -421,10 +411,7 @@ def document_stats(request, stats_type=None): series_data.sort(key=lambda t: t[1], reverse=True) series_data = series_data[:30] - chart_data.append({ - "data": series_data, - "animation": False, - }) + chart_data.append({ "data": series_data }) for alias, name in sorted(aliases.iteritems(), key=lambda t: t[1]): alias_data.append((name, alias)) @@ -440,14 +427,16 @@ def document_stats(request, stats_type=None): # same way, and we don't want to go back and edit them # either, we transform them here. - name_country_set = set((name, country) - for name, country in person_qs.values_list("name", "documentauthor__country")) + name_country_set = { + (name, country) + for name, country in person_qs.values_list("name", "documentauthor__country") + } aliases = get_aliased_countries(country for _, country in name_country_set) countries = { c.name: c for c in CountryName.objects.all() } eu_name = "EU" - eu_countries = set(c for c in countries.itervalues() if c.in_eu) + eu_countries = { c for c in countries.itervalues() if c.in_eu } for name, country in name_country_set: country_name = aliases.get(country, country) @@ -470,10 +459,7 @@ def document_stats(request, stats_type=None): series_data.sort(key=lambda t: t[1], reverse=True) series_data = series_data[:30] - chart_data.append({ - "data": series_data, - "animation": False, - }) + chart_data.append({ "data": series_data }) for alias, country_name in aliases.iteritems(): alias_data.append((country_name, alias, countries.get(country_name))) @@ -487,8 +473,10 @@ def document_stats(request, stats_type=None): person_qs = Person.objects.filter(person_filters) - name_country_set = set((name, country) - for name, country in person_qs.values_list("name", "documentauthor__country")) + name_country_set = { + (name, country) + for name, country in person_qs.values_list("name", "documentauthor__country") + } aliases = get_aliased_countries(country for _, country in name_country_set) @@ -511,10 +499,7 @@ def document_stats(request, stats_type=None): series_data.sort(key=lambda t: t[1], reverse=True) - chart_data.append({ - "data": series_data, - "animation": False, - }) + chart_data.append({ "data": series_data }) elif stats_type == "author/citations": stats_title = "Number of citations of {}s written by author".format(doc_label) @@ -537,10 +522,7 @@ def document_stats(request, stats_type=None): series_data.append((citations, percentage)) table_data.append((citations, percentage, [plain_name(n) for n in names])) - chart_data.append({ - "data": sorted(series_data, key=lambda t: t[0]), - "animation": False, - }) + chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) }) elif stats_type == "author/hindex": stats_title = "h-index for {}s written by author".format(doc_label) @@ -565,10 +547,152 @@ def document_stats(request, stats_type=None): series_data.append((citations, percentage)) table_data.append((citations, percentage, [plain_name(n) for n in names])) - chart_data.append({ - "data": sorted(series_data, key=lambda t: t[0]), - "animation": False, - }) + chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) }) + + elif any(stats_type == t[0] and stats_type.split("/")[1] in ["affiliation", "country", "continent"] + for t in possible_yearly_stats_types): + + person_filters = Q(documentauthor__document__type="draft") + + # filter persons + rfc_state = State.objects.get(type="draft", slug="rfc") + if document_type == "rfc": + person_filters &= Q(documentauthor__document__states=rfc_state) + elif document_type == "draft": + person_filters &= ~Q(documentauthor__document__states=rfc_state) + + doc_years = defaultdict(set) + + docevent_qs = DocEvent.objects.filter( + doc__type="draft", + type__in=["published_rfc", "new_revision"], + ).values_list("doc", "time").order_by("doc") + + for doc, time in docevent_qs.iterator(): + doc_years[doc].add(time.year) + + person_qs = Person.objects.filter(person_filters) + + if document_type == "rfc": + doc_label = "RFC" + elif document_type == "draft": + doc_label = "draft" + else: + doc_label = "document" + + template_name = "yearly" + + years_from = from_time.year if from_time else 1 + years_to = datetime.date.today().year - 1 + + def add_yearly_chart_data_from_bins(bins, limit): + aggregated_bins = defaultdict(set) + years = set() + for (year, label), names in bins.iteritems(): + years.add(year) + aggregated_bins[label].update(names) + + years = list(sorted(y for y in years)) + + limit = 8 + sorted_bins = sorted(aggregated_bins.iteritems(), key=lambda t: len(t[1]), reverse=True) + top = [ label for label, names in list(sorted_bins)[:limit]] + + for label in top: + series_data = [] + + for y in years: + names = bins.get((y, label), set()) + + series_data.append((y, len(names))) + + chart_data.append({ + "data": series_data, + "name": label + }) + + + if stats_type == "yearly/affiliation": + stats_title = "Number of {} authors per affiliation over the years".format(doc_label) + + person_qs = Person.objects.filter(person_filters) + + name_affiliation_doc_set = { + (name, affiliation, doc) + for name, affiliation, doc in person_qs.values_list("name", "documentauthor__affiliation", "documentauthor__document") + } + + aliases = get_aliased_affiliations(affiliation for _, affiliation, _ in name_affiliation_doc_set) + + bins = defaultdict(set) + for name, affiliation, doc in name_affiliation_doc_set: + a = aliases.get(affiliation, affiliation) + if a: + for year in doc_years.get(doc): + if years_from <= year <= years_to: + bins[(year, a)].add(name) + + add_yearly_chart_data_from_bins(bins, limit=8) + + elif stats_type == "yearly/country": + stats_title = "Number of {} authors per country over the years".format(doc_label) + + person_qs = Person.objects.filter(person_filters) + + name_country_doc_set = { + (name, country, doc) + for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document") + } + + aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set) + + countries = { c.name: c for c in CountryName.objects.all() } + eu_name = "EU" + eu_countries = { c for c in countries.itervalues() if c.in_eu } + + bins = defaultdict(set) + + for name, country, doc in name_country_doc_set: + country_name = aliases.get(country, country) + c = countries.get(country_name) + + if country_name: + for year in doc_years.get(doc): + if years_from <= year <= years_to: + bins[(year, country_name)].add(name) + + if c and c.in_eu: + bins[(year, eu_name)].add(name) + + add_yearly_chart_data_from_bins(bins, limit=8) + + + elif stats_type == "yearly/continent": + stats_title = "Number of {} authors per continent".format(doc_label) + + person_qs = Person.objects.filter(person_filters) + + name_country_doc_set = { + (name, country, doc) + for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document") + } + + aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set) + + country_to_continent = dict(CountryName.objects.values_list("name", "continent__name")) + + bins = defaultdict(set) + + for name, country, doc in name_country_doc_set: + country_name = aliases.get(country, country) + continent_name = country_to_continent.get(country_name, "") + + if continent_name: + for year in doc_years.get(doc): + if years_from <= year <= years_to: + bins[(year, continent_name)].add(name) + + add_yearly_chart_data_from_bins(bins, limit=8) return render(request, "stats/document_stats.html", { "chart_data": mark_safe(json.dumps(chart_data)), @@ -576,6 +700,7 @@ def document_stats(request, stats_type=None): "stats_title": stats_title, "possible_document_stats_types": possible_document_stats_types, "possible_author_stats_types": possible_author_stats_types, + "possible_yearly_stats_types": possible_yearly_stats_types, "stats_type": stats_type, "possible_document_types": possible_document_types, "document_type": document_type, @@ -587,7 +712,7 @@ def document_stats(request, stats_type=None): "hide_aliases_url": build_document_stats_url(get_overrides={ "showaliases": None }), "alias_data": alias_data, "eu_countries": sorted(eu_countries or [], key=lambda c: c.name), - "content_template": "stats/document_stats_{}.html".format(stats_type.replace("/", "_")), + "content_template": "stats/document_stats_{}.html".format(template_name), }) diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html index 8ff53471d..18eb1ee9c 100644 --- a/ietf/templates/stats/document_stats.html +++ b/ietf/templates/stats/document_stats.html @@ -36,6 +36,16 @@
+
+ Yearly: + +
+ {% for slug, label, url in possible_yearly_stats_types %} + {{ label }} + {% endfor %} +
+
+
Options
diff --git a/ietf/templates/stats/document_stats_author_affiliation.html b/ietf/templates/stats/document_stats_author_affiliation.html index 6bec8d3c8..8c6df8cce 100644 --- a/ietf/templates/stats/document_stats_author_affiliation.html +++ b/ietf/templates/stats/document_stats_author_affiliation.html @@ -7,6 +7,11 @@ chart: { type: 'column' }, + plotOptions: { + column: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, diff --git a/ietf/templates/stats/document_stats_author_citations.html b/ietf/templates/stats/document_stats_author_citations.html index bcb3cff9e..25c41a978 100644 --- a/ietf/templates/stats/document_stats_author_citations.html +++ b/ietf/templates/stats/document_stats_author_citations.html @@ -7,6 +7,11 @@ chart: { type: 'area' }, + plotOptions: { + area: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, diff --git a/ietf/templates/stats/document_stats_author_continent.html b/ietf/templates/stats/document_stats_author_continent.html index 0e0a1f849..b35334f95 100644 --- a/ietf/templates/stats/document_stats_author_continent.html +++ b/ietf/templates/stats/document_stats_author_continent.html @@ -7,6 +7,11 @@ chart: { type: 'column' }, + plotOptions: { + column: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, diff --git a/ietf/templates/stats/document_stats_author_country.html b/ietf/templates/stats/document_stats_author_country.html index 7819e6c2c..ca01636ad 100644 --- a/ietf/templates/stats/document_stats_author_country.html +++ b/ietf/templates/stats/document_stats_author_country.html @@ -7,6 +7,11 @@ chart: { type: 'column' }, + plotOptions: { + column: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, diff --git a/ietf/templates/stats/document_stats_author_documents.html b/ietf/templates/stats/document_stats_author_documents.html index 025e8c26f..30d8882a4 100644 --- a/ietf/templates/stats/document_stats_author_documents.html +++ b/ietf/templates/stats/document_stats_author_documents.html @@ -7,6 +7,11 @@ chart: { type: 'column' }, + plotOptions: { + column: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, diff --git a/ietf/templates/stats/document_stats_author_hindex.html b/ietf/templates/stats/document_stats_author_hindex.html index d5d67329c..e8ba232e6 100644 --- a/ietf/templates/stats/document_stats_author_hindex.html +++ b/ietf/templates/stats/document_stats_author_hindex.html @@ -7,6 +7,11 @@ chart: { type: 'column' }, + plotOptions: { + column: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, diff --git a/ietf/templates/stats/document_stats_authors.html b/ietf/templates/stats/document_stats_authors.html index 70fe249fb..abfc5ff96 100644 --- a/ietf/templates/stats/document_stats_authors.html +++ b/ietf/templates/stats/document_stats_authors.html @@ -7,6 +7,11 @@ chart: { type: 'column' }, + plotOptions: { + column: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, diff --git a/ietf/templates/stats/document_stats_format.html b/ietf/templates/stats/document_stats_format.html index ce1512a09..c7f42f7ae 100644 --- a/ietf/templates/stats/document_stats_format.html +++ b/ietf/templates/stats/document_stats_format.html @@ -7,6 +7,11 @@ chart: { type: 'column' }, + plotOptions: { + column: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, diff --git a/ietf/templates/stats/document_stats_formlang.html b/ietf/templates/stats/document_stats_formlang.html index e4b586d95..7c9470f8c 100644 --- a/ietf/templates/stats/document_stats_formlang.html +++ b/ietf/templates/stats/document_stats_formlang.html @@ -7,6 +7,11 @@ chart: { type: 'column' }, + plotOptions: { + column: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, diff --git a/ietf/templates/stats/document_stats_pages.html b/ietf/templates/stats/document_stats_pages.html index dca167b1c..40cc55fa9 100644 --- a/ietf/templates/stats/document_stats_pages.html +++ b/ietf/templates/stats/document_stats_pages.html @@ -7,6 +7,11 @@ chart: { type: 'line' }, + plotOptions: { + line: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, diff --git a/ietf/templates/stats/document_stats_words.html b/ietf/templates/stats/document_stats_words.html index 956e49ea7..96bcb0e75 100644 --- a/ietf/templates/stats/document_stats_words.html +++ b/ietf/templates/stats/document_stats_words.html @@ -7,6 +7,11 @@ chart: { type: 'line' }, + plotOptions: { + line: { + animation: false + } + }, title: { text: '{{ stats_title|escapejs }}' }, From 36f42738d9381462231a196df4ec7e1a7d7d0dfd Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Tue, 21 Feb 2017 17:08:04 +0000 Subject: [PATCH 45/49] Add template missing from previous commit - Legacy-Id: 12895 --- .../stats/document_stats_yearly.html | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 ietf/templates/stats/document_stats_yearly.html diff --git a/ietf/templates/stats/document_stats_yearly.html b/ietf/templates/stats/document_stats_yearly.html new file mode 100644 index 000000000..b77988448 --- /dev/null +++ b/ietf/templates/stats/document_stats_yearly.html @@ -0,0 +1,53 @@ +

{{ stats_title }}

+ +
+ + From ef4d55f0c9b0d523b31f69e8687eb7736dabf9bd Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 27 Mar 2017 08:33:49 +0000 Subject: [PATCH 46/49] Apply patch from Henrik Levkowetz to fix some problems of author parse errors where the affiliation is mistakenly thought to be an extra author (some of these still remain) - Legacy-Id: 13142 --- ietf/utils/draft.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index 603a63afa..dc420d320 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -703,10 +703,15 @@ class Draft(): break found_pos = [] + company_or_author = None for i in range(len(authors)): _debug("1: authors[%s]: %s" % (i, authors[i])) _debug(" company[%s]: %s" % (i, companies[i])) author = authors[i] + if i+1 < len(authors): + company_or_author = authors[i+1] + else: + company_or_author = None if author in [ None, '', ]: continue suffix_match = re.search(" %(suffix)s$" % aux, author) @@ -875,7 +880,8 @@ class Draft(): if authmatch: _debug(" ? Other author or company ? : %s" % authmatch) _debug(" Line: "+line.strip()) - if nonblank_count == 1 or (nonblank_count == 2 and not blanklines): + _debug(" C or A: %s"%company_or_author) + if nonblank_count == 1 or (nonblank_count == 2 and not blanklines) or (company_or_author==line.strip() and not blanklines): # First line after an author -- this is a company companies_seen += [ c.lower() for c in authmatch ] companies_seen += [ line.strip().lower() ] # XXX fix this for columnized author list From 5b8ac15c0e80f8e739344a686cdb8665256f48ac Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 27 Mar 2017 08:35:34 +0000 Subject: [PATCH 47/49] Improve debug output - Legacy-Id: 13143 --- ietf/stats/backfill_data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py index cf4d7ed28..44ce6d9a3 100644 --- a/ietf/stats/backfill_data.py +++ b/ietf/stats/backfill_data.py @@ -84,9 +84,9 @@ for doc in docs_qs.prefetch_related("docalias_set", "formal_languages", "documen if author.email_id: old_authors_by_email[author.email_id] = author - # the draft parser sometimes has a problem if affiliation - # isn't in the second line, then it will report an extra - # author - skip those + # the draft parser sometimes has a problem when + # affiliation isn't in the second line and it then thinks + # it's an extra author - skip those extra authors seen = set() for full, _, _, _, _, email, country, company in d.get_author_list(): if email in seen: @@ -118,7 +118,7 @@ for doc in docs_qs.prefetch_related("docalias_set", "formal_languages", "documen country = country.decode("latin-1") if old_author.country != country: - print "new country", canonical_name ,"[", full, "]", old_author.country.encode("utf-8"), "->", country.encode("utf-8") + print "new country", canonical_name ,"[", full, email, "]", old_author.country.encode("utf-8"), "->", country.encode("utf-8") old_author.country = country old_author.save(update_fields=["country"]) updated = True From 5608ad006180982d0b1fac53593df7c841ac185d Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 27 Mar 2017 08:45:20 +0000 Subject: [PATCH 48/49] Move the assert on new X in the test coverage up before the percentage to try get the list of URLs out before the percentage - Legacy-Id: 13144 --- ietf/utils/test_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 259bd10bc..ed3b2f7c6 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -257,11 +257,11 @@ class CoverageTest(TestCase): if self.runner.run_full_test_suite: # Permit 0.02% variation in results -- otherwise small code changes become a pain fudge_factor = 0.00005 # 0.005% -- a small change, less than the last digit we show + self.assertLessEqual(len(test_missing), len(master_missing), + msg = "New %s without test coverage since %s: %s" % (test, latest_coverage_version, list(set(test_missing) - set(master_missing)))) self.assertGreaterEqual(test_coverage, master_coverage - fudge_factor, msg = "The %s coverage percentage is now lower (%.2f%%) than for version %s (%.2f%%)" % ( test, test_coverage*100, latest_coverage_version, master_coverage*100, )) - self.assertLessEqual(len(test_missing), len(master_missing), - msg = "New %s without test coverage since %s: %s" % (test, latest_coverage_version, list(set(test_missing) - set(master_missing)))) def template_coverage_test(self): global loaded_templates From 22c979d506b49f0d2025ae8ca2240eec342ee867 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 27 Mar 2017 08:49:15 +0000 Subject: [PATCH 49/49] Drop django-countries dependency, it is not needed when we have a list country names ourselves - Legacy-Id: 13145 --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d5eb29170..16aba8935 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ decorator>=3.4.0 defusedxml>=0.4.1 # for TastyPie when ussing xml; not a declared dependency Django>=1.8.16,<1.9 django-bootstrap3>=5.1.1,<7.0.0 # django-bootstrap 7.0 requires django 1.8 -django-countries>=4.0 django-formtools>=1.0 # instead of django.contrib.formtools in 1.8 django-markup>=1.1 django-tastypie>=0.13.1