From 641d92cf49e597e0c4475916294cd0c0f89e6b14 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Mon, 16 Jan 2017 17:06:54 +0000 Subject: [PATCH] Add many-to-many field with formal languages to Document and add formal language statistics - Legacy-Id: 12658 --- .../doc/migrations/0020_auto_20170112_0753.py | 11 ++++ ietf/doc/models.py | 3 +- ietf/name/admin.py | 3 +- .../migrations/0017_formallanguagename.py | 28 +++++++++ .../migrations/0018_add_formlang_names.py | 26 ++++++++ ietf/name/models.py | 2 + ietf/stats/backfill_data.py | 25 +++++++- ietf/stats/views.py | 33 +++++++--- .../stats/document_stats_formlang.html | 60 +++++++++++++++++++ 9 files changed, 179 insertions(+), 12 deletions(-) create mode 100644 ietf/name/migrations/0017_formallanguagename.py create mode 100644 ietf/name/migrations/0018_add_formlang_names.py create mode 100644 ietf/templates/stats/document_stats_formlang.html diff --git a/ietf/doc/migrations/0020_auto_20170112_0753.py b/ietf/doc/migrations/0020_auto_20170112_0753.py index da22265f7..9ca0cafa6 100644 --- a/ietf/doc/migrations/0020_auto_20170112_0753.py +++ b/ietf/doc/migrations/0020_auto_20170112_0753.py @@ -7,6 +7,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ + ('name', '0017_formallanguagename'), ('doc', '0019_auto_20161207_1036'), ] @@ -21,4 +22,14 @@ class Migration(migrations.Migration): name='words', field=models.IntegerField(null=True, blank=True), ), + migrations.AddField( + model_name='dochistory', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), + migrations.AddField( + model_name='document', + name='formal_languages', + field=models.ManyToManyField(help_text=b'Formal languages used in document', to='name.FormalLanguageName', blank=True), + ), ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 6ef127dc3..e4c776004 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -15,7 +15,7 @@ import debug # pyflakes:ignore from ietf.group.models import Group from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName, - DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName ) + DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, FormalLanguageName ) from ietf.person.models import Email, Person from ietf.utils.admin import admin_link @@ -76,6 +76,7 @@ class DocumentInfo(models.Model): rev = models.CharField(verbose_name="revision", max_length=16, blank=True) pages = models.IntegerField(blank=True, null=True) words = models.IntegerField(blank=True, null=True) + formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document") order = models.IntegerField(default=1, blank=True) # This is probably obviated by SessionPresentaion.order intended_std_level = models.ForeignKey(IntendedStdLevelName, verbose_name="Intended standardization level", blank=True, null=True) std_level = models.ForeignKey(StdLevelName, verbose_name="Standardization level", blank=True, null=True) diff --git a/ietf/name/admin.py b/ietf/name/admin.py index 684e930ce..642d36aff 100644 --- a/ietf/name/admin.py +++ b/ietf/name/admin.py @@ -3,7 +3,7 @@ from django.contrib import admin from ietf.name.models import ( BallotPositionName, ConstraintName, DBTemplateTypeName, DocRelationshipName, DocReminderTypeName, DocTagName, DocTypeName, DraftSubmissionStateName, - FeedbackTypeName, GroupMilestoneStateName, GroupStateName, GroupTypeName, + FeedbackTypeName, FormalLanguageName, GroupMilestoneStateName, GroupStateName, GroupTypeName, IntendedStdLevelName, IprDisclosureStateName, IprEventTypeName, IprLicenseTypeName, LiaisonStatementEventTypeName, LiaisonStatementPurposeName, LiaisonStatementState, LiaisonStatementTagName, MeetingTypeName, NomineePositionStateName, @@ -32,6 +32,7 @@ admin.site.register(DBTemplateTypeName, NameAdmin) admin.site.register(DocReminderTypeName, NameAdmin) admin.site.register(DocTagName, NameAdmin) admin.site.register(DraftSubmissionStateName, NameAdmin) +admin.site.register(FormalLanguageName, NameAdmin) admin.site.register(FeedbackTypeName, NameAdmin) admin.site.register(GroupMilestoneStateName, NameAdmin) admin.site.register(GroupStateName, NameAdmin) diff --git a/ietf/name/migrations/0017_formallanguagename.py b/ietf/name/migrations/0017_formallanguagename.py new file mode 100644 index 000000000..94a7dfc4e --- /dev/null +++ b/ietf/name/migrations/0017_formallanguagename.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0016_auto_20161013_1010'), + ] + + operations = [ + migrations.CreateModel( + name='FormalLanguageName', + fields=[ + ('slug', models.CharField(max_length=32, serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255)), + ('desc', models.TextField(blank=True)), + ('used', models.BooleanField(default=True)), + ('order', models.IntegerField(default=0)), + ], + options={ + 'ordering': ['order', 'name'], + 'abstract': False, + }, + ), + ] diff --git a/ietf/name/migrations/0018_add_formlang_names.py b/ietf/name/migrations/0018_add_formlang_names.py new file mode 100644 index 000000000..542cb6cf5 --- /dev/null +++ b/ietf/name/migrations/0018_add_formlang_names.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations + +def insert_initial_formal_language_names(apps, schema_editor): + FormalLanguageName = apps.get_model("name", "FormalLanguageName") + FormalLanguageName.objects.get_or_create(slug="abnf", name="ABNF", desc="Augmented Backus-Naur Form", order=1) + FormalLanguageName.objects.get_or_create(slug="asn1", name="ASN.1", desc="Abstract Syntax Notation One", order=2) + FormalLanguageName.objects.get_or_create(slug="cbor", name="CBOR", desc="Concise Binary Object Representation", order=3) + FormalLanguageName.objects.get_or_create(slug="ccode", name="C Code", desc="Code in the C Programming Language", order=4) + FormalLanguageName.objects.get_or_create(slug="json", name="JSON", desc="Javascript Object Notation", order=5) + FormalLanguageName.objects.get_or_create(slug="xml", name="XML", desc="Extensible Markup Language", order=6) + +def noop(apps, schema_editor): + pass + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0017_formallanguagename'), + ] + + operations = [ + migrations.RunPython(insert_initial_formal_language_names, noop) + ] diff --git a/ietf/name/models.py b/ietf/name/models.py index c1c69b4c6..d208a867b 100644 --- a/ietf/name/models.py +++ b/ietf/name/models.py @@ -46,6 +46,8 @@ class StdLevelName(NameModel): class IntendedStdLevelName(NameModel): """Proposed Standard, (Draft Standard), Internet Standard, Experimental, Informational, Best Current Practice, Historic, ...""" +class FormalLanguageName(NameModel): + """ABNF, ASN.1, C code, CBOR, JSON, XML, ...""" class DocReminderTypeName(NameModel): "Stream state" class BallotPositionName(NameModel): diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py index f088de081..bdcf00193 100644 --- a/ietf/stats/backfill_data.py +++ b/ietf/stats/backfill_data.py @@ -14,20 +14,24 @@ django.setup() from django.conf import settings from ietf.doc.models import Document +from ietf.name.models import FormalLanguageName from ietf.utils.draft import Draft parser = argparse.ArgumentParser() parser.add_argument("--document", help="specific document name") parser.add_argument("--words", action="store_true", help="fill in word count") +parser.add_argument("--formlang", action="store_true", help="fill in formal languages") args = parser.parse_args() +formal_language_dict = { l.pk: l for l in FormalLanguageName.objects.all() } + docs_qs = Document.objects.filter(type="draft") if args.document: docs_qs = docs_qs.filter(docalias__name=args.document) -for doc in docs_qs.prefetch_related("docalias_set"): +for doc in docs_qs.prefetch_related("docalias_set", "formal_languages"): canonical_name = doc.name for n in doc.docalias_set.all(): if n.name.startswith("rfc"): @@ -45,6 +49,8 @@ for doc in docs_qs.prefetch_related("docalias_set"): with open(path, 'r') as f: d = Draft(f.read(), path) + updated = False + updates = {} if args.words: @@ -52,7 +58,24 @@ for doc in docs_qs.prefetch_related("docalias_set"): if words != doc.words: updates["words"] = words + if args.formlang: + langs = d.get_formal_languages() + + new_formal_languages = set(formal_language_dict[l] for l in langs) + old_formal_languages = set(doc.formal_languages.all()) + + if new_formal_languages != old_formal_languages: + for l in new_formal_languages - old_formal_languages: + doc.formal_languages.add(l) + updated = True + for l in old_formal_languages - new_formal_languages: + doc.formal_languages.remove(l) + updated = True + if updates: Document.objects.filter(pk=doc.pk).update(**updates) + updated = True + + if updated: print "updated", canonical_name diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 6601d0266..a580bb277 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -134,6 +134,8 @@ def document_stats(request, stats_type=None, document_type=None): stats_title = "" bin_size = 1 + total_docs = docalias_qs.count() + if stats_type == "authors": stats_title = "Number of authors for each {}".format(doc_label) @@ -142,8 +144,6 @@ def document_stats(request, stats_type=None, document_type=None): for name, author_count in generate_canonical_names(docalias_qs.values_list("name").annotate(Count("document__authors"))): bins[author_count].append(name) - total_docs = sum(len(names) for author_count, names in bins.iteritems()) - series_data = [] for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): percentage = len(names) * 100.0 / total_docs @@ -163,8 +163,6 @@ def document_stats(request, stats_type=None, document_type=None): for name, pages in generate_canonical_names(docalias_qs.values_list("name", "document__pages")): bins[pages].append(name) - total_docs = sum(len(names) for pages, names in bins.iteritems()) - series_data = [] for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]): percentage = len(names) * 100.0 / total_docs @@ -187,8 +185,6 @@ def document_stats(request, stats_type=None, document_type=None): for name, words in generate_canonical_names(docalias_qs.values_list("name", "document__words")): bins[put_into_bin(words, bin_size)].append(name) - total_docs = sum(len(names) for words, names in bins.iteritems()) - series_data = [] for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]): percentage = len(names) * 100.0 / total_docs @@ -203,7 +199,7 @@ def document_stats(request, stats_type=None, document_type=None): }) elif stats_type == "format": - stats_title = "Formats for each {}".format(doc_label) + stats_title = "Submission formats for each {}".format(doc_label) bins = defaultdict(list) @@ -244,8 +240,6 @@ def document_stats(request, stats_type=None, document_type=None): if canonical_name: bins[ext.upper()].append(canonical_name) - total_docs = sum(len(names) for fmt, names in bins.iteritems()) - series_data = [] for fmt, names in sorted(bins.iteritems(), key=lambda t: t[0]): percentage = len(names) * 100.0 / total_docs @@ -258,6 +252,27 @@ def document_stats(request, stats_type=None, document_type=None): "animation": False, }) + elif stats_type == "formlang": + stats_title = "Formal languages used for each {}".format(doc_label) + + bins = defaultdict(list) + + for name, formal_language_name in generate_canonical_names(docalias_qs.values_list("name", "document__formal_languages__name")): + bins[formal_language_name].append(name) + + series_data = [] + for formal_language, names in sorted(bins.iteritems(), key=lambda t: t[0]): + percentage = len(names) * 100.0 / total_docs + if formal_language is not None: + series_data.append((formal_language, len(names))) + table_data.append((formal_language, percentage, names)) + + chart_data.append({ + "data": series_data, + "animation": False, + }) + + return render(request, "stats/document_stats.html", { "chart_data": mark_safe(json.dumps(chart_data)), "table_data": table_data, diff --git a/ietf/templates/stats/document_stats_formlang.html b/ietf/templates/stats/document_stats_formlang.html new file mode 100644 index 000000000..248a45b82 --- /dev/null +++ b/ietf/templates/stats/document_stats_formlang.html @@ -0,0 +1,60 @@ +

{{ stats_title }}

+ +
+ + + +

Data

+ + + + + + + + + + + {% for formal_language, percentage, names in table_data %} + + + + + + {% endfor %} + +
Formal languagePercentage of {{ doc_label }}s{{ doc_label|capfirst }}s
{{ formal_language }}{{ percentage|floatformat:2 }}%{% include "stats/includes/docnames_cell.html" %}