altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

fix: remove unreliable statistics (#8307)

Browse source
This commit is contained in:
Robert Sparks 2024-12-09 10:33:03 -06:00 committed by GitHub
parent 167752ba76
commit 3055d17eb1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 18 additions and 2382 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
ietf/settings.py
View file

@ -809,8 +809,8 @@ AUDIO_IMPORT_EMAIL = ['ietf@meetecho.com']
SESSION_REQUEST_FROM_EMAIL = 'IETF Meeting Session Request Tool <session-request@ietf.org>'
SECRETARIAT_SUPPORT_EMAIL = "support@ietf.org"
SECRETARIAT_ACTION_EMAIL = "ietf-action@ietf.org"
SECRETARIAT_INFO_EMAIL = "ietf-info@ietf.org"
SECRETARIAT_ACTION_EMAIL = SECRETARIAT_SUPPORT_EMAIL
SECRETARIAT_INFO_EMAIL = SECRETARIAT_SUPPORT_EMAIL
# Put real password in settings_local.py
IANA_SYNC_PASSWORD = "secret"

127
ietf/stats/tests.py
View file

@ -13,22 +13,16 @@ from requests import Response
import debug # pyflakes:ignore
from django.urls import reverse as urlreverse
from django.utils import timezone
from ietf.utils.test_utils import login_testing_unauthorized, TestCase
import ietf.stats.views
from ietf.submit.models import Submission
from ietf.doc.factories import WgDraftFactory, WgRfcFactory
from ietf.doc.models import Document, State, RelatedDocument, NewRevisionDocEvent, DocumentAuthor
from ietf.group.factories import RoleFactory
from ietf.meeting.factories import MeetingFactory, AttendedFactory
from ietf.meeting.factories import MeetingFactory
from ietf.person.factories import PersonFactory
from ietf.person.models import Person, Email
from ietf.name.models import FormalLanguageName, DocRelationshipName, CountryName
from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory
from ietf.stats.models import MeetingRegistration, CountryAlias
from ietf.stats.factories import MeetingRegistrationFactory
from ietf.stats.models import MeetingRegistration
from ietf.stats.tasks import fetch_meeting_attendance_task
from ietf.stats.utils import get_meeting_registration_data, FetchStats, fetch_attendance_from_meetings
from ietf.utils.timezone import date_today
@ -41,121 +35,14 @@ class StatisticsTests(TestCase):
self.assertEqual(r.status_code, 200)
def test_document_stats(self):
WgRfcFactory()
draft = WgDraftFactory()
DocumentAuthor.objects.create(
document=draft,
person=Person.objects.get(email__address="aread@example.org"),
email=Email.objects.get(address="aread@example.org"),
country="Germany",
affiliation="IETF",
order=1
)
r = self.client.get(urlreverse("ietf.stats.views.document_stats"))
self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index"))
# create some data for the statistics
Submission.objects.create(
authors=[ { "name": "Some Body", "email": "somebody@example.com", "affiliation": "Some Inc.", "country": "US" }],
pages=30,
rev=draft.rev,
words=4000,
draft=draft,
file_types=".txt",
state_id="posted",
)
draft.formal_languages.add(FormalLanguageName.objects.get(slug="xml"))
Document.objects.filter(pk=draft.pk).update(words=4000)
# move it back so it shows up in the yearly summaries
NewRevisionDocEvent.objects.filter(doc=draft, rev=draft.rev).update(
time=timezone.now() - datetime.timedelta(days=500))
referencing_draft = Document.objects.create(
name="draft-ietf-mars-referencing",
type_id="draft",
title="Referencing",
stream_id="ietf",
abstract="Test",
rev="00",
pages=2,
words=100
)
referencing_draft.set_state(State.objects.get(used=True, type="draft", slug="active"))
RelatedDocument.objects.create(
source=referencing_draft,
target=draft,
relationship=DocRelationshipName.objects.get(slug="refinfo")
)
NewRevisionDocEvent.objects.create(
type="new_revision",
by=Person.objects.get(name="(System)"),
doc=referencing_draft,
desc="New revision available",
rev=referencing_draft.rev,
time=timezone.now() - datetime.timedelta(days=1000)
)
# check redirect
url = urlreverse(ietf.stats.views.document_stats)
authors_url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": "authors" })
r = self.client.get(url)
self.assertEqual(r.status_code, 302)
self.assertTrue(authors_url in r["Location"])
# check various stats types
for stats_type in ["authors", "pages", "words", "format", "formlang",
"author/documents", "author/affiliation", "author/country",
"author/continent", "author/citations", "author/hindex",
"yearly/affiliation", "yearly/country", "yearly/continent"]:
for document_type in ["", "rfc", "draft"]:
for time_choice in ["", "5y"]:
url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type })
r = self.client.get(url, {
"type": document_type,
"time": time_choice,
})
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertTrue(q('#chart'))
if not stats_type.startswith("yearly"):
self.assertTrue(q('table.stats-data'))
def test_meeting_stats(self):
# create some data for the statistics
meeting = MeetingFactory(type_id='ietf', date=date_today(), number="96")
MeetingRegistrationFactory(first_name='John', last_name='Smith', country_code='US', email="john.smith@example.us", meeting=meeting, attended=True)
CountryAlias.objects.get_or_create(alias="US", country=CountryName.objects.get(slug="US"))
p = MeetingRegistrationFactory(first_name='Jaume', last_name='Guillaume', country_code='FR', email="jaume.guillaume@example.fr", meeting=meeting, attended=False).person
CountryAlias.objects.get_or_create(alias="FR", country=CountryName.objects.get(slug="FR"))
AttendedFactory(session__meeting=meeting,person=p)
# check redirect
url = urlreverse(ietf.stats.views.meeting_stats)
r = self.client.get(urlreverse("ietf.stats.views.meeting_stats"))
self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index"))
authors_url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": "overview" })
r = self.client.get(url)
self.assertEqual(r.status_code, 302)
self.assertTrue(authors_url in r["Location"])
# check various stats types
for stats_type in ["overview", "country", "continent"]:
url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": stats_type })
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertTrue(q('#chart'))
if stats_type == "overview":
self.assertTrue(q('table.stats-data'))
for stats_type in ["country", "continent"]:
url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": stats_type, "num": meeting.number })
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertTrue(q('#chart'))
self.assertTrue(q('table.stats-data'))
def test_known_country_list(self):
# check redirect

894
ietf/stats/views.py
View file

@ -2,25 +2,18 @@
# -*- coding: utf-8 -*-
import os
import calendar
import datetime
import email.utils
import itertools
import json
import dateutil.relativedelta
from collections import defaultdict
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.cache import cache
from django.db.models import Count, Q, Subquery, OuterRef
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
from django.shortcuts import render
from django.urls import reverse as urlreverse
from django.utils import timezone
from django.utils.safestring import mark_safe
from django.utils.text import slugify
import debug # pyflakes:ignore
@ -29,18 +22,12 @@ from ietf.review.utils import (extract_review_assignment_data,
ReviewAssignmentData,
sum_period_review_assignment_stats,
sum_raw_review_assignment_aggregations)
from ietf.submit.models import Submission
from ietf.group.models import Role, Group
from ietf.person.models import Person
from ietf.name.models import ReviewResultName, CountryName, DocRelationshipName, ReviewAssignmentStateName
from ietf.person.name import plain_name
from ietf.doc.models import Document, RelatedDocument, State, DocEvent
from ietf.meeting.models import Meeting
from ietf.stats.models import MeetingRegistration, CountryAlias
from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries, compute_hirsch_index
from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName
from ietf.ietfauth.utils import has_role
from ietf.utils.response import permission_denied
from ietf.utils.timezone import date_today, DEADLINE_TZINFO, RPC_TZINFO
from ietf.utils.timezone import date_today, DEADLINE_TZINFO
def stats_index(request):
@ -135,632 +122,8 @@ def add_labeled_top_series_from_bins(chart_data, bins, limit):
})
def document_stats(request, stats_type=None):
def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None):
if get_overrides is None:
get_overrides={}
kwargs = {
"stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override,
}
return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index"))
return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.items() if v is not None }) + generate_query_string(request.GET, get_overrides)
# the length limitation is to keep the key shorter than memcached's limit
# of 250 after django has added the key_prefix and key_version parameters
cache_key = ("stats:document_stats:%s:%s" % (stats_type, slugify(request.META.get('QUERY_STRING',''))))[:228]
data = cache.get(cache_key)
if not data:
names_limit = settings.STATS_NAMES_LIMIT
# statistics types
possible_document_stats_types = add_url_to_choices([
("authors", "Number of authors"),
("pages", "Pages"),
("words", "Words"),
("format", "Format"),
("formlang", "Formal languages"),
], lambda slug: build_document_stats_url(stats_type_override=slug))
possible_author_stats_types = add_url_to_choices([
("author/documents", "Number of documents"),
("author/affiliation", "Affiliation"),
("author/country", "Country"),
("author/continent", "Continent"),
("author/citations", "Citations"),
("author/hindex", "h-index"),
], lambda slug: build_document_stats_url(stats_type_override=slug))
possible_yearly_stats_types = add_url_to_choices([
("yearly/affiliation", "Affiliation"),
("yearly/country", "Country"),
("yearly/continent", "Continent"),
], lambda slug: build_document_stats_url(stats_type_override=slug))
if not stats_type:
return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_document_stats_types[0][0]))
possible_document_types = add_url_to_choices([
("", "All"),
("rfc", "RFCs"),
("draft", "Internet-Drafts"),
], lambda slug: build_document_stats_url(get_overrides={ "type": slug }))
document_type = get_choice(request, "type", possible_document_types) or ""
possible_time_choices = add_url_to_choices([
("", "All time"),
("5y", "Past 5 years"),
], lambda slug: build_document_stats_url(get_overrides={ "time": slug }))
time_choice = request.GET.get("time") or ""
from_time = None
if "y" in time_choice:
try:
y = int(time_choice.rstrip("y"))
from_time = timezone.now() - dateutil.relativedelta.relativedelta(years=y)
except ValueError:
pass
chart_data = []
table_data = []
stats_title = ""
template_name = stats_type.replace("/", "_")
bin_size = 1
alias_data = []
eu_countries = None
if any(stats_type == t[0] for t in possible_document_stats_types):
# filter documents
document_filters = Q(type__in=["draft","rfc"]) # TODO - review lots of "rfc is a draft" assumptions below
rfc_state = State.objects.get(type="rfc", slug="published")
if document_type == "rfc":
document_filters &= Q(states=rfc_state)
elif document_type == "draft":
document_filters &= ~Q(states=rfc_state)
if from_time:
# this is actually faster than joining in the database,
# despite the round-trip back and forth
docs_within_time_constraint = list(Document.objects.filter(
type="draft",
docevent__time__gte=from_time,
docevent__type__in=["published_rfc", "new_revision"],
).values_list("pk",flat=True))
document_filters &= Q(pk__in=docs_within_time_constraint)
document_qs = Document.objects.filter(document_filters)
if document_type == "rfc":
doc_label = "RFC"
elif document_type == "draft":
doc_label = "draft"
else:
doc_label = "document"
total_docs = document_qs.values_list("name").distinct().count()
if stats_type == "authors":
stats_title = "Number of authors for each {}".format(doc_label)
bins = defaultdict(set)
for name, author_count in document_qs.values_list("name").annotate(Count("documentauthor")).values_list("name","documentauthor__count"):
bins[author_count or 0].add(name)
series_data = []
for author_count, names in sorted(bins.items(), key=lambda t: t[0]):
percentage = len(names) * 100.0 / (total_docs or 1)
series_data.append((author_count, percentage))
table_data.append((author_count, percentage, len(names), list(names)[:names_limit]))
chart_data.append({ "data": series_data })
elif stats_type == "pages":
stats_title = "Number of pages for each {}".format(doc_label)
bins = defaultdict(set)
for name, pages in document_qs.values_list("name", "pages"):
bins[pages or 0].add(name)
series_data = []
for pages, names in sorted(bins.items(), key=lambda t: t[0]):
percentage = len(names) * 100.0 / (total_docs or 1)
if pages is not None:
series_data.append((pages, len(names)))
table_data.append((pages, percentage, len(names), list(names)[:names_limit]))
chart_data.append({ "data": series_data })
elif stats_type == "words":
stats_title = "Number of words for each {}".format(doc_label)
bin_size = 500
bins = defaultdict(set)
for name, words in document_qs.values_list("name", "words"):
bins[put_into_bin(words, bin_size)].add(name)
series_data = []
for (value, words), names in sorted(bins.items(), key=lambda t: t[0][0]):
percentage = len(names) * 100.0 / (total_docs or 1)
if words is not None:
series_data.append((value, len(names)))
table_data.append((words, percentage, len(names), list(names)[:names_limit]))
chart_data.append({ "data": series_data })
elif stats_type == "format":
stats_title = "Submission formats for each {}".format(doc_label)
bins = defaultdict(set)
# on new documents, we should have a Submission row with the file types
submission_types = {}
for doc_name, file_types in Submission.objects.values_list("draft", "file_types").order_by("submission_date", "id"):
submission_types[doc_name] = file_types
doc_names_with_missing_types = {}
for doc_name, doc_type, rev in document_qs.values_list("name", "type_id", "rev"):
types = submission_types.get(doc_name)
if types:
for dot_ext in types.split(","):
bins[dot_ext.lstrip(".").upper()].add(doc_name)
else:
if doc_type == "rfc":
filename = doc_name
else:
filename = doc_name + "-" + rev
doc_names_with_missing_types[filename] = doc_name
# look up the remaining documents on disk
for filename in itertools.chain(os.listdir(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR), os.listdir(settings.RFC_PATH)):
t = filename.split(".", 1)
if len(t) != 2:
continue
basename, ext = t
ext = ext.lower()
if not any(ext==allowlisted_ext for allowlisted_ext in settings.DOCUMENT_FORMAT_ALLOWLIST):
continue
name = doc_names_with_missing_types.get(basename)
if name:
bins[ext.upper()].add(name)
series_data = []
for fmt, names in sorted(bins.items(), key=lambda t: t[0]):
percentage = len(names) * 100.0 / (total_docs or 1)
series_data.append((fmt, len(names)))
table_data.append((fmt, percentage, len(names), list(names)[:names_limit]))
chart_data.append({ "data": series_data })
elif stats_type == "formlang":
stats_title = "Formal languages used for each {}".format(doc_label)
bins = defaultdict(set)
for name, formal_language_name in document_qs.values_list("name", "formal_languages__name"):
bins[formal_language_name or ""].add(name)
series_data = []
for formal_language, names in sorted(bins.items(), key=lambda t: t[0]):
percentage = len(names) * 100.0 / (total_docs or 1)
if formal_language is not None:
series_data.append((formal_language, len(names)))
table_data.append((formal_language, percentage, len(names), list(names)[:names_limit]))
chart_data.append({ "data": series_data })
elif any(stats_type == t[0] for t in possible_author_stats_types):
person_filters = Q(documentauthor__document__type="draft")
# filter persons
rfc_state = State.objects.get(type="rfc", slug="published")
if document_type == "rfc":
person_filters &= Q(documentauthor__document__states=rfc_state)
elif document_type == "draft":
person_filters &= ~Q(documentauthor__document__states=rfc_state)
if from_time:
# this is actually faster than joining in the database,
# despite the round-trip back and forth
docs_within_time_constraint = set(Document.objects.filter(
type="draft",
docevent__time__gte=from_time,
docevent__type__in=["published_rfc", "new_revision"],
).values_list("pk"))
person_filters &= Q(documentauthor__document__in=docs_within_time_constraint)
person_qs = Person.objects.filter(person_filters)
if document_type == "rfc":
doc_label = "RFC"
elif document_type == "draft":
doc_label = "draft"
else:
doc_label = "document"
if stats_type == "author/documents":
stats_title = "Number of {}s per author".format(doc_label)
bins = defaultdict(set)
person_qs = Person.objects.filter(person_filters)
for name, document_count in person_qs.values_list("name").annotate(Count("documentauthor")):
bins[document_count or 0].add(name)
total_persons = count_bins(bins)
series_data = []
for document_count, names in sorted(bins.items(), key=lambda t: t[0]):
percentage = len(names) * 100.0 / (total_persons or 1)
series_data.append((document_count, percentage))
plain_names = sorted([ plain_name(n) for n in names ])
table_data.append((document_count, percentage, len(plain_names), list(plain_names)[:names_limit]))
chart_data.append({ "data": series_data })
elif stats_type == "author/affiliation":
stats_title = "Number of {} authors per affiliation".format(doc_label)
bins = defaultdict(set)
person_qs = Person.objects.filter(person_filters)
# Since people don't write the affiliation names in the
# same way, and we don't want to go back and edit them
# either, we transform them here.
name_affiliation_set = {
(name, affiliation)
for name, affiliation in person_qs.values_list("name", "documentauthor__affiliation")
}
aliases = get_aliased_affiliations(affiliation for _, affiliation in name_affiliation_set)
for name, affiliation in name_affiliation_set:
bins[aliases.get(affiliation, affiliation)].add(name)
prune_unknown_bin_with_known(bins)
total_persons = count_bins(bins)
series_data = []
for affiliation, names in sorted(bins.items(), key=lambda t: t[0].lower()):
percentage = len(names) * 100.0 / (total_persons or 1)
if affiliation:
series_data.append((affiliation, len(names)))
plain_names = sorted([ plain_name(n) for n in names ])
table_data.append((affiliation, percentage, len(plain_names), list(plain_names)[:names_limit]))
series_data.sort(key=lambda t: t[1], reverse=True)
series_data = series_data[:30]
chart_data.append({ "data": series_data })
for alias, name in sorted(aliases.items(), key=lambda t: t[1]):
alias_data.append((name, alias))
elif stats_type == "author/country":
stats_title = "Number of {} authors per country".format(doc_label)
bins = defaultdict(set)
person_qs = Person.objects.filter(person_filters)
# Since people don't write the country names in the
# same way, and we don't want to go back and edit them
# either, we transform them here.
name_country_set = {
(name, country)
for name, country in person_qs.values_list("name", "documentauthor__country")
}
aliases = get_aliased_countries(country for _, country in name_country_set)
countries = { c.name: c for c in CountryName.objects.all() }
eu_name = "EU"
eu_countries = { c for c in countries.values() if c.in_eu }
for name, country in name_country_set:
country_name = aliases.get(country, country)
bins[country_name].add(name)
c = countries.get(country_name)
if c and c.in_eu:
bins[eu_name].add(name)
prune_unknown_bin_with_known(bins)
total_persons = count_bins(bins)
series_data = []
for country, names in sorted(bins.items(), key=lambda t: t[0].lower()):
percentage = len(names) * 100.0 / (total_persons or 1)
if country:
series_data.append((country, len(names)))
plain_names = sorted([ plain_name(n) for n in names ])
table_data.append((country, percentage, len(plain_names), list(plain_names)[:names_limit]))
series_data.sort(key=lambda t: t[1], reverse=True)
series_data = series_data[:30]
chart_data.append({ "data": series_data })
for alias, country_name in aliases.items():
alias_data.append((country_name, alias, countries.get(country_name)))
alias_data.sort()
elif stats_type == "author/continent":
stats_title = "Number of {} authors per continent".format(doc_label)
bins = defaultdict(set)
person_qs = Person.objects.filter(person_filters)
name_country_set = {
(name, country)
for name, country in person_qs.values_list("name", "documentauthor__country")
}
aliases = get_aliased_countries(country for _, country in name_country_set)
country_to_continent = dict(CountryName.objects.values_list("name", "continent__name"))
for name, country in name_country_set:
country_name = aliases.get(country, country)
continent_name = country_to_continent.get(country_name, "")
bins[continent_name].add(name)
prune_unknown_bin_with_known(bins)
total_persons = count_bins(bins)
series_data = []
for continent, names in sorted(bins.items(), key=lambda t: t[0].lower()):
percentage = len(names) * 100.0 / (total_persons or 1)
if continent:
series_data.append((continent, len(names)))
plain_names = sorted([ plain_name(n) for n in names ])
table_data.append((continent, percentage, len(plain_names), list(plain_names)[:names_limit]))
series_data.sort(key=lambda t: t[1], reverse=True)
chart_data.append({ "data": series_data })
elif stats_type == "author/citations":
stats_title = "Number of citations of {}s written by author".format(doc_label)
bins = defaultdict(set)
cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold']))
person_filters &= Q(documentauthor__document__relateddocument__relationship__in=cite_relationships)
person_qs = Person.objects.filter(person_filters)
for name, citations in person_qs.values_list("name").annotate(Count("documentauthor__document__relateddocument")):
bins[citations or 0].add(name)
total_persons = count_bins(bins)
series_data = []
for citations, names in sorted(bins.items(), key=lambda t: t[0], reverse=True):
percentage = len(names) * 100.0 / (total_persons or 1)
series_data.append((citations, percentage))
plain_names = sorted([ plain_name(n) for n in names ])
table_data.append((citations, percentage, len(plain_names), list(plain_names)[:names_limit]))
chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) })
elif stats_type == "author/hindex":
stats_title = "h-index for {}s written by author".format(doc_label)
bins = defaultdict(set)
cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold']))
person_filters &= Q(documentauthor__document__relateddocument__relationship__in=cite_relationships)
person_qs = Person.objects.filter(person_filters)
values = person_qs.values_list("name", "documentauthor__document").annotate(Count("documentauthor__document__relateddocument"))
for name, ts in itertools.groupby(values.order_by("name"), key=lambda t: t[0]):
h_index = compute_hirsch_index([citations for _, document, citations in ts])
bins[h_index or 0].add(name)
total_persons = count_bins(bins)
series_data = []
for citations, names in sorted(bins.items(), key=lambda t: t[0], reverse=True):
percentage = len(names) * 100.0 / (total_persons or 1)
series_data.append((citations, percentage))
plain_names = sorted([ plain_name(n) for n in names ])
table_data.append((citations, percentage, len(plain_names), list(plain_names)[:names_limit]))
chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) })
elif any(stats_type == t[0] for t in possible_yearly_stats_types):
# filter persons
rfc_state = State.objects.get(type="rfc", slug="published")
if document_type == "rfc":
person_filters = Q(documentauthor__document__type="rfc")
person_filters &= Q(documentauthor__document__states=rfc_state)
elif document_type == "draft":
person_filters = Q(documentauthor__document__type="draft")
person_filters &= ~Q(documentauthor__document__states=rfc_state)
else:
person_filters = Q(documentauthor__document__type="rfc")
person_filters |= Q(documentauthor__document__type="draft")
doc_years = defaultdict(set)
draftevent_qs = DocEvent.objects.filter(
doc__type="draft",
type = "new_revision",
).values_list("doc","time").order_by("doc")
for doc_id, time in draftevent_qs.iterator():
# RPC_TZINFO is used to match the timezone handling in Document.pub_date()
doc_years[doc_id].add(time.astimezone(RPC_TZINFO).year)
rfcevent_qs = (
DocEvent.objects.filter(doc__type="rfc", type="published_rfc")
.annotate(
draft=Subquery(
RelatedDocument.objects.filter(
target=OuterRef("doc__pk"), relationship_id="became_rfc"
).values_list("source", flat=True)[:1]
)
)
.values_list("doc", "time")
.order_by("doc")
)
for doc_id, time in rfcevent_qs.iterator():
doc_years[doc_id].add(time.astimezone(RPC_TZINFO).year)
person_qs = Person.objects.filter(person_filters)
if document_type == "rfc":
doc_label = "RFC"
elif document_type == "draft":
doc_label = "draft"
else:
doc_label = "document"
template_name = "yearly"
years_from = from_time.year if from_time else 1
years_to = timezone.now().year - 1
if stats_type == "yearly/affiliation":
stats_title = "Number of {} authors per affiliation over the years".format(doc_label)
person_qs = Person.objects.filter(person_filters)
name_affiliation_doc_set = {
(name, affiliation, doc)
for name, affiliation, doc in person_qs.values_list("name", "documentauthor__affiliation", "documentauthor__document")
}
aliases = get_aliased_affiliations(affiliation for _, affiliation, _ in name_affiliation_doc_set)
bins = defaultdict(set)
for name, affiliation, doc in name_affiliation_doc_set:
a = aliases.get(affiliation, affiliation)
if a:
years = doc_years.get(doc)
if years:
for year in years:
if years_from <= year <= years_to:
bins[(year, a)].add(name)
add_labeled_top_series_from_bins(chart_data, bins, limit=8)
elif stats_type == "yearly/country":
stats_title = "Number of {} authors per country over the years".format(doc_label)
person_qs = Person.objects.filter(person_filters)
name_country_doc_set = {
(name, country, doc)
for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document")
}
aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set)
countries = { c.name: c for c in CountryName.objects.all() }
eu_name = "EU"
eu_countries = { c for c in countries.values() if c.in_eu }
bins = defaultdict(set)
for name, country, doc in name_country_doc_set:
country_name = aliases.get(country, country)
c = countries.get(country_name)
years = doc_years.get(doc)
if country_name and years:
for year in years:
if years_from <= year <= years_to:
bins[(year, country_name)].add(name)
if c and c.in_eu:
bins[(year, eu_name)].add(name)
add_labeled_top_series_from_bins(chart_data, bins, limit=8)
elif stats_type == "yearly/continent":
stats_title = "Number of {} authors per continent".format(doc_label)
person_qs = Person.objects.filter(person_filters)
name_country_doc_set = {
(name, country, doc)
for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document")
}
aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set)
country_to_continent = dict(CountryName.objects.values_list("name", "continent__name"))
bins = defaultdict(set)
for name, country, doc in name_country_doc_set:
country_name = aliases.get(country, country)
continent_name = country_to_continent.get(country_name, "")
if continent_name:
years = doc_years.get(doc)
if years:
for year in years:
if years_from <= year <= years_to:
bins[(year, continent_name)].add(name)
add_labeled_top_series_from_bins(chart_data, bins, limit=8)
data = {
"chart_data": mark_safe(json.dumps(chart_data)),
"table_data": table_data,
"stats_title": stats_title,
"possible_document_stats_types": possible_document_stats_types,
"possible_author_stats_types": possible_author_stats_types,
"possible_yearly_stats_types": possible_yearly_stats_types,
"stats_type": stats_type,
"possible_document_types": possible_document_types,
"document_type": document_type,
"possible_time_choices": possible_time_choices,
"time_choice": time_choice,
"doc_label": doc_label,
"bin_size": bin_size,
"show_aliases_url": build_document_stats_url(get_overrides={ "showaliases": "1" }),
"hide_aliases_url": build_document_stats_url(get_overrides={ "showaliases": None }),
"alias_data": alias_data,
"eu_countries": sorted(eu_countries or [], key=lambda c: c.name),
"content_template": "stats/document_stats_{}.html".format(template_name),
}
# Logs are full of these, but nobody is using them
# log("Cache miss for '%s'. Data size: %sk" % (cache_key, len(str(data))/1000))
cache.set(cache_key, data, 24*60*60)
return render(request, "stats/document_stats.html", data)
def known_countries_list(request, stats_type=None, acronym=None):
countries = CountryName.objects.prefetch_related("countryalias_set")
@ -774,252 +137,7 @@ def known_countries_list(request, stats_type=None, acronym=None):
})
def meeting_stats(request, num=None, stats_type=None):
meeting = None
if num is not None:
meeting = get_object_or_404(Meeting, number=num, type="ietf")
def build_meeting_stats_url(number=None, stats_type_override=Ellipsis, get_overrides=None):
if get_overrides is None:
get_overrides = {}
kwargs = {
"stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override,
}
if number is not None:
kwargs["num"] = number
return urlreverse(meeting_stats, kwargs={ k: v for k, v in kwargs.items() if v is not None }) + generate_query_string(request.GET, get_overrides)
cache_key = ("stats:meeting_stats:%s:%s:%s" % (num, stats_type, slugify(request.META.get('QUERY_STRING',''))))[:228]
data = cache.get(cache_key)
if not data:
names_limit = settings.STATS_NAMES_LIMIT
# statistics types
if meeting:
possible_stats_types = add_url_to_choices([
("country", "Country"),
("continent", "Continent"),
], lambda slug: build_meeting_stats_url(number=meeting.number, stats_type_override=slug))
else:
possible_stats_types = add_url_to_choices([
("overview", "Overview"),
("country", "Country"),
("continent", "Continent"),
], lambda slug: build_meeting_stats_url(number=None, stats_type_override=slug))
if not stats_type:
return HttpResponseRedirect(build_meeting_stats_url(number=num, stats_type_override=possible_stats_types[0][0]))
chart_data = []
piechart_data = []
table_data = []
stats_title = ""
template_name = stats_type
bin_size = 1
eu_countries = None
def get_country_mapping(attendees):
return {
alias.alias: alias.country
for alias in CountryAlias.objects.filter(alias__in=set(r.country_code for r in attendees)).select_related("country", "country__continent")
if alias.alias.isupper()
}
def reg_name(r):
return email.utils.formataddr(((r.first_name + " " + r.last_name).strip(), r.email))
if meeting and any(stats_type == t[0] for t in possible_stats_types):
attendees = MeetingRegistration.objects.filter(
meeting=meeting,
reg_type__in=['onsite', 'remote']
).filter(
Q( attended=True) | Q( checkedin=True )
)
if stats_type == "country":
stats_title = "Number of attendees for {} {} per country".format(meeting.type.name, meeting.number)
bins = defaultdict(set)
country_mapping = get_country_mapping(attendees)
eu_name = "EU"
eu_countries = set(CountryName.objects.filter(in_eu=True))
for r in attendees:
name = reg_name(r)
c = country_mapping.get(r.country_code)
bins[c.name if c else ""].add(name)
if c and c.in_eu:
bins[eu_name].add(name)
prune_unknown_bin_with_known(bins)
total_attendees = count_bins(bins)
series_data = []
for country, names in sorted(bins.items(), key=lambda t: t[0].lower()):
percentage = len(names) * 100.0 / (total_attendees or 1)
if country:
series_data.append((country, len(names)))
table_data.append((country, percentage, len(names), list(names)[:names_limit]))
if country and country != eu_name:
piechart_data.append({ "name": country, "y": percentage })
series_data.sort(key=lambda t: t[1], reverse=True)
series_data = series_data[:20]
piechart_data.sort(key=lambda d: d["y"], reverse=True)
pie_cut_off = 8
piechart_data = piechart_data[:pie_cut_off] + [{ "name": "Other", "y": sum(d["y"] for d in piechart_data[pie_cut_off:])}]
chart_data.append({ "data": series_data })
elif stats_type == "continent":
stats_title = "Number of attendees for {} {} per continent".format(meeting.type.name, meeting.number)
bins = defaultdict(set)
country_mapping = get_country_mapping(attendees)
for r in attendees:
name = reg_name(r)
c = country_mapping.get(r.country_code)
bins[c.continent.name if c else ""].add(name)
prune_unknown_bin_with_known(bins)
total_attendees = count_bins(bins)
series_data = []
for continent, names in sorted(bins.items(), key=lambda t: t[0].lower()):
percentage = len(names) * 100.0 / (total_attendees or 1)
if continent:
series_data.append((continent, len(names)))
table_data.append((continent, percentage, len(names), list(names)[:names_limit]))
series_data.sort(key=lambda t: t[1], reverse=True)
chart_data.append({ "data": series_data })
elif not meeting and any(stats_type == t[0] for t in possible_stats_types):
template_name = "overview"
attendees = MeetingRegistration.objects.filter(
meeting__type="ietf",
attended=True,
reg_type__in=['onsite', 'remote']
).filter(
Q( attended=True) | Q( checkedin=True )
).select_related('meeting')
if stats_type == "overview":
stats_title = "Number of attendees per meeting"
continents = {}
meetings = Meeting.objects.filter(type='ietf', date__lte=date_today()).order_by('number')
for m in meetings:
country = CountryName.objects.get(slug=m.country)
continents[country.continent.name] = country.continent.name
bins = defaultdict(set)
for r in attendees:
meeting_number = int(r.meeting.number)
name = reg_name(r)
bins[meeting_number].add(name)
series_data = {}
for continent in list(continents.keys()):
series_data[continent] = []
for m in meetings:
country = CountryName.objects.get(slug=m.country)
url = build_meeting_stats_url(number=m.number,
stats_type_override="country")
for continent in list(continents.keys()):
if continent == country.continent.name:
d = {
"name": "IETF {} - {}, {}".format(int(m.number), m.city, country),
"x": int(m.number),
"y": m.attendees,
"date": m.date.strftime("%d %b %Y"),
"url": url,
}
else:
d = {
"x": int(m.number),
"y": 0,
}
series_data[continent].append(d)
table_data.append((m, url,
m.attendees, country))
for continent in list(continents.keys()):
# series_data[continent].sort(key=lambda t: t[0]["x"])
chart_data.append( { "name": continent,
"data": series_data[continent] })
table_data.sort(key=lambda t: int(t[0].number), reverse=True)
elif stats_type == "country":
stats_title = "Number of attendees per country across meetings"
country_mapping = get_country_mapping(attendees)
eu_name = "EU"
eu_countries = set(CountryName.objects.filter(in_eu=True))
bins = defaultdict(set)
for r in attendees:
meeting_number = int(r.meeting.number)
name = reg_name(r)
c = country_mapping.get(r.country_code)
if c:
bins[(meeting_number, c.name)].add(name)
if c.in_eu:
bins[(meeting_number, eu_name)].add(name)
add_labeled_top_series_from_bins(chart_data, bins, limit=8)
elif stats_type == "continent":
stats_title = "Number of attendees per continent across meetings"
country_mapping = get_country_mapping(attendees)
bins = defaultdict(set)
for r in attendees:
meeting_number = int(r.meeting.number)
name = reg_name(r)
c = country_mapping.get(r.country_code)
if c:
bins[(meeting_number, c.continent.name)].add(name)
add_labeled_top_series_from_bins(chart_data, bins, limit=8)
data = {
"chart_data": mark_safe(json.dumps(chart_data)),
"piechart_data": mark_safe(json.dumps(piechart_data)),
"table_data": table_data,
"stats_title": stats_title,
"possible_stats_types": possible_stats_types,
"stats_type": stats_type,
"bin_size": bin_size,
"meeting": meeting,
"eu_countries": sorted(eu_countries or [], key=lambda c: c.name),
"content_template": "stats/meeting_stats_{}.html".format(template_name),
}
# Logs are full of these, but nobody is using them...
# log("Cache miss for '%s'. Data size: %sk" % (cache_key, len(str(data))/1000))
cache.set(cache_key, data, 24*60*60)
#
return render(request, "stats/meeting_stats.html", data)
return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index"))
@login_required

86
ietf/templates/stats/document_stats.html
View file

@ -1,86 +0,0 @@
{% extends "base.html" %}
{% load origin %}
{% load ietf_filters static %}
{% block title %}{{ stats_title }}{% endblock %}
{% block pagehead %}
<link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
<link rel="stylesheet" href="{% static "ietf/css/highcharts.css" %}">
{% endblock %}
{% block content %}
{% origin %}
<h1>Internet-Draft and RFC statistics</h1>
<div class="row my-3">
<label class="fw-bold col-sm-2 col-form-label">Documents:</label>
<div class="btn-group col-sm-10">
{% for slug, label, url in possible_document_stats_types %}
<a class="btn btn-outline-primary
{% if slug == stats_type %}
active
{% endif %}"
href="{{ url }}">{{ label }}</a>
{% endfor %}
</div>
</div>
<div class="row my-3">
<label class="fw-bold col-sm-2 col-form-label">Authors:</label>
<div class="btn-group col-sm-10">
{% for slug, label, url in possible_author_stats_types %}
<a class="btn btn-outline-primary
{% if slug == stats_type %}
active
{% endif %}"
href="{{ url }}">{{ label }}</a>
{% endfor %}
</div>
</div>
<div class="row my-3">
<label class="fw-bold col-sm-2 col-form-label">Yearly:</label>
<div class="btn-group col-sm-10">
{% for slug, label, url in possible_yearly_stats_types %}
<a class="btn btn-outline-primary
{% if slug == stats_type %}
active
{% endif %}"
href="{{ url }}">{{ label }}</a>
{% endfor %}
</div>
</div>
<h2>Options</h2>
<div class="row my-3">
<label class="fw-bold col-sm-2 col-form-label">Document type:</label>
<div class="btn-group col-sm-10">
{% for slug, label, url in possible_document_types %}
<a class="btn btn-outline-primary
{% if slug == document_type %}
active
{% endif %}"
href="{{ url }}">{{ label }}</a>
{% endfor %}
</div>
</div>
<div class="row my-3">
<label class="fw-bold col-sm-2 col-form-label">Time:</label>
<div class="btn-group col-sm-10">
{% for slug, label, url in possible_time_choices %}
<a class="btn btn-outline-primary
{% if slug == time_choice %}
active
{% endif %}"
href="{{ url }}">{{ label }}</a>
{% endfor %}
</div>
</div>
<div class="alert alert-info my-3">
<b>Please Note:</b> The author information in the datatracker about RFCs
with numbers lower than about 1300 and Internet-Drafts from before 2001 is
unreliable and in many cases absent. For this reason, statistics on these
pages does not show correct author stats for corpus selections that involve such
documents.
</div>
{% include content_template %}
{% endblock %}
{% block js %}
<script src="{% static 'ietf/js/highcharts.js' %}"></script>
<script src="{% static 'ietf/js/stats.js' %}"></script>
<script src="{% static "ietf/js/list.js" %}"></script>
{% endblock %}

113
ietf/templates/stats/document_stats_author_affiliation.html
View file

@ -1,113 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'column'
},
plotOptions: {
column: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
type: "category",
title: {
text: 'Affiliation'
}
},
yAxis: {
title: {
text: 'Number of authors'
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.points[0].key + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="affiliation">Affiliation</th>
<th scope="col" data-sort="percent">Percentage of authors</th>
<th scope="col" data-sort="count">Authors</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for affiliation, percentage, count, names in table_data %}
<tr>
<td>{{ affiliation|default:"(unknown)" }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>
<p>
The statistics are based entirely on the author affiliation
provided with each Internet-Draft. Since this may vary across documents, an
author may be counted with more than one affiliation, making the
total sum more than 100%.
</p>
<h2>Affiliation Aliases</h2>
<p>
In generating the above statistics, some heuristics have been
applied to determine the affiliations of each author.
</p>
{% if request.GET.showaliases %}
<p>
<a href="{{ hide_aliases_url }}" class="btn btn-primary">Hide generated aliases</a>
</p>
{% if request.user.is_staff %}
<p>
Note: since you're an admin, you can
<a href="{% url "admin:stats_affiliationalias_add" %}">add an extra known alias</a>
or see the
<a href="{% url "admin:stats_affiliationalias_changelist" %}">existing known aliases</a>
and
<a href="{% url "admin:stats_affiliationignoredending_changelist" %}">generally ignored endings</a>.
</p>
{% endif %}
{% if alias_data %}
<table class="table table-sm table-striped tablesorter">
<thead>
<tr>
<th scope="col" data-sort="affiliation">Affiliation</th>
<th scope="col" data-sort="alias">Alias</th>
</tr>
</thead>
{% if alias_data %}
<tbody>
{% for name, alias in alias_data %}
<tr>
<td>{{ name|default:"(unknown)" }}</td>
<td>{{ alias }}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>
{% endif %}
{% else %}
<p>
<a href="{{ show_aliases_url }}" class="btn btn-primary">Show generated aliases</a>
</p>
{% endif %}

72
ietf/templates/stats/document_stats_author_citations.html
View file

@ -1,72 +0,0 @@
{% load origin %}{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'area'
},
plotOptions: {
area: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
title: {
text: 'Number of citations of {{ doc_label }}s by author'
},
max: 500
},
yAxis: {
title: {
text: 'Percentage of authors'
},
labels: {
formatter: function () {
return this.value + '%';
}
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.x + ' ' + (this.x == 1 ? "citation" : 'citations') + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y.toFixed(1) + '%';
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="num">Citations</th>
<th scope="col" data-sort="percent">Percentage of authors</th>
<th scope="col" data-sort="authors">Authors</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for citations, percentage, count, names in table_data %}
<tr>
<td>{{ citations }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>
<p>Note that the citation counts do not exclude self-references.</p>

69
ietf/templates/stats/document_stats_author_continent.html
View file

@ -1,69 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'column'
},
plotOptions: {
column: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
type: "category",
title: {
text: 'Continent'
}
},
yAxis: {
title: {
text: 'Number of authors'
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.points[0].key + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="continent">Continent</th>
<th scope="col" data-sort="percent">Percentage of authors</th>
<th scope="col" data-sort="count">Authors</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for continent, percentage, count, names in table_data %}
<tr>
<td>{{ continent|default:"(unknown)" }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>
<p>
The statistics are based entirely on the author addresses provided
with each Internet-Draft. Since this varies across documents, a traveling
author may be counted in more than country, making the total sum
more than 100%.
</p>

136
ietf/templates/stats/document_stats_author_country.html
View file

@ -1,136 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'column'
},
plotOptions: {
column: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
type: "category",
title: {
text: 'Country'
}
},
yAxis: {
title: {
text: 'Number of authors'
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.points[0].key + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="country">Country</th>
<th scope="col" class="text-end" data-sort="percent">Percentage of authors</th>
<th scope="col" class="text-end" data-sort="authors">Authors</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for country, percentage, count, names in table_data %}
<tr>
<td>{{ country|default:"(unknown)" }}</td>
<td class="text-end">{{ percentage|floatformat:2 }}%</td>
<td class="text-end">{% include "stats/includes/number_with_details_cell.html" %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>
<p>
The statistics are based entirely on the author addresses provided
with each Internet-Draft. Since this varies across documents, a traveling
author may be counted in more than country, making the total sum
more than 100%.
</p>
<p>
In case no country information is found for an author in the time
period, the author is counted as (unknown).
</p>
<p>
EU (European Union) is not a country, but has been added for reference, as the sum of
all current EU member countries:
{% for c in eu_countries %}
{{ c.name }}{% if not forloop.last %},{% endif %}
{% endfor %}
.
</p>
<h2>Country Aliases</h2>
<p>
In generating the above statistics, some heuristics have been
applied to figure out which country each author is from.
</p>
{% if request.GET.showaliases %}
<p>
<a href="{{ hide_aliases_url }}" class="btn btn-primary">Hide generated aliases</a>
</p>
{% if request.user.is_staff %}
<p class="alert alert-info my-3">
Note: since you're an admin, some extra links are visible. You
can either correct a document author entry directly in case the
information is obviously missing or add an alias if an unknown
<a href="{% url "admin:name_countryname_changelist" %}">country name</a>
is being used.
</p>
{% endif %}
{% if alias_data %}
<table class="table table-sm table-striped tablesorter">
<thead>
<th scope="col" data-sort="country">Country</th>
<th scope="col" data-sort="alias">Alias</th>
</thead>
{% if alias_data %}
<tbody>
{% for name, alias, country in alias_data %}
<tr>
<td>
{% if country and request.user.is_staff %}
<a href="{% url "admin:name_countryname_change" country.pk %}">{{ name|default:"(unknown)" }}</a>
{% else %}
{{ name|default:"(unknown)" }}
{% endif %}
</td>
<td>
{{ alias }}
{% if request.user.is_staff and name != "EU" %}
<a class="float-end btn btn-primary btn-sm"
href="{% url "admin:doc_documentauthor_changelist" %}?country={{ alias|urlencode }}">
Matching authors
</a>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>
{% endif %}
{% else %}
<p>
<a href="{{ show_aliases_url }}" class="btn btn-primary">Show generated aliases</a>
</p>
{% endif %}

69
ietf/templates/stats/document_stats_author_documents.html
View file

@ -1,69 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'column'
},
plotOptions: {
column: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
tickInterval: 1,
title: {
text: 'Author of number of {{ doc_label }}s'
},
max: 20
},
yAxis: {
title: {
text: 'Percentage of authors'
},
labels: {
formatter: function () {
return this.value + '%';
}
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.x + ' ' + (this.x == 1 ? "{{ doc_label }}" : '{{ doc_label }}s') + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y.toFixed(1) + '%';
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="num">Documents</th>
<th scope="col" data-sort="percent">Percentage of authors</th>
<th scope="col" data-sort="count">Authors</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for document_count, percentage, count, names in table_data %}
<tr>
<td>{{ document_count }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>

83
ietf/templates/stats/document_stats_author_hindex.html
View file

@ -1,83 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'column'
},
plotOptions: {
column: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
tickInterval: 1,
title: {
text: 'h-index of {{ doc_label }}s by author'
}
},
yAxis: {
title: {
text: 'Percentage of authors'
},
labels: {
formatter: function () {
return this.value + '%';
}
}
},
tooltip: {
formatter: function () {
var s = '<b>' + ' h-index ' + this.x + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y.toFixed(1) + '%';
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="num">h-index</th>
<th scope="col" data-sort="percent">Percentage of authors</th>
<th scope="col" data-sort="Authors">Authors</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for h_index, percentage, count, names in table_data %}
<tr>
<td>{{ h_index }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" with content_limit=25 %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>
<p>
Hirsch index or h-index is a
<a href="https://www.wikipedia.org/wiki/H-index">
measure of the
productivity and impact of the publications of an author
</a>.
An
author with an h-index of 5 has had 5 publications each cited at
least 5 times - to increase the index to 6, the 5 publications plus
1 more would have to have been cited at least 6 times, each. Thus a
high h-index requires many highly-cited publications.
</p>
<p>
Note that the h-index calculations do not exclude self-references.
</p>

68
ietf/templates/stats/document_stats_authors.html
View file

@ -1,68 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'column'
},
plotOptions: {
column: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
tickInterval: 1,
title: {
text: 'Number of authors'
}
},
yAxis: {
title: {
text: 'Percentage of {{ doc_label }}s'
},
labels: {
formatter: function () {
return this.value + '%';
}
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.x + ' ' + (this.x == 1 ? "author" : 'authors') + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y.toFixed(1) + '%';
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h3>Data</h3>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="num">Authors</th>
<th scope="col" data-sort="percent">Percentage of {{ doc_label }}s</th>
<th scope="col" data-sort="count">{{ doc_label|capfirst }}s</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for author_count, percentage, count, names in table_data %}
<tr>
<td>{{ author_count }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>

63
ietf/templates/stats/document_stats_format.html
View file

@ -1,63 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'column'
},
plotOptions: {
column: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
type: "category",
title: {
text: 'Format'
}
},
yAxis: {
title: {
text: 'Number of {{ doc_label }}s'
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.points[0].key + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="format">Format</th>
<th scope="col" data-sort="percent">Percentage of {{ doc_label }}s</th>
<th scope="col" data-sort="label">{{ doc_label|capfirst }}s</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for pages, percentage, count, names in table_data %}
<tr>
<td>{{ pages }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>

63
ietf/templates/stats/document_stats_formlang.html
View file

@ -1,63 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'column'
},
plotOptions: {
column: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
type: "category",
title: {
text: 'Formal language'
}
},
yAxis: {
title: {
text: 'Number of {{ doc_label }}s'
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.points[0].key + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="formal">Formal language</th>
<th scope="col" data-sort="percent">Percentage of {{ doc_label }}s</th>
<th scope="col" data-sort="count">{{ doc_label|capfirst }}s</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for formal_language, percentage, count, names in table_data %}
<tr>
<td>{{ formal_language }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>

62
ietf/templates/stats/document_stats_pages.html
View file

@ -1,62 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'line'
},
plotOptions: {
line: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
title: {
text: 'Number of pages'
}
},
yAxis: {
title: {
text: 'Number of {{ doc_label }}s'
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.x + ' ' + (this.x == 1 ? "page" : 'pages') + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="num">Pages</th>
<th scope="col" data-sort="percent">Percentage of {{ doc_label }}s</th>
<th scope="col" data-sort="count">{{ doc_label|capfirst }}s</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for pages, percentage, count, names in table_data %}
<tr>
<td>{{ pages }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>

62
ietf/templates/stats/document_stats_words.html
View file

@ -1,62 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'line'
},
plotOptions: {
line: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
title: {
text: 'Number of words'
}
},
yAxis: {
title: {
text: 'Number of {{ doc_label }}s'
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.x + ' - ' + (this.x + {{ bin_size }} - 1) + ' ' + (this.x == 1 ? "word" : 'words') + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped tablesorter stats-data">
<thead>
<tr>
<th scope="col" data-sort="num">Words</th>
<th scope="col" data-sort="percent">Percentage of {{ doc_label }}s</th>
<th scope="col" data-sort="count">{{ doc_label|capfirst }}s</th>
</tr>
</thead>
{% if table_data %}
<tbody>
{% for pages, percentage, count, names in table_data %}
<tr>
<td>{{ pages }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
</tr>
{% endfor %}
</tbody>
{% endif %}
</table>

52
ietf/templates/stats/document_stats_yearly.html
View file

@ -1,52 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'line',
},
plotOptions: {
line: {
marker: {
enabled: false
},
animation: false
}
},
legend: {
align: "right",
verticalAlign: "middle",
layout: "vertical",
enabled: true
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
tickInterval: 1,
title: {
text: 'Year'
}
},
yAxis: {
min: 0,
title: {
text: 'Authors active in year'
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.x + '</b>';
$.each(this.points, function () {
s += '<br>' + this.series.name + ': ' + this.y;
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>

15
ietf/templates/stats/includes/number_with_details_cell.html
View file

@ -1,15 +0,0 @@
{% load person_filters %}
{% if content_limit and count <= content_limit %}
{% for n in names %}
{% with n|person_by_name as person %}
{% if person %}
{% person_link person %}
{% else %}
{{ n }}
{% endif %}
<br>
{% endwith %}
{% endfor %}
{% else %}
{{ count }}
{% endif %}

9
ietf/templates/stats/index.html
View file

@ -10,15 +10,12 @@
Statistics on...
</p>
<ul>
<li>
<a href="{% url "ietf.stats.views.document_stats" %}">Internet-Drafts and RFCs (authors, countries, formats, ...)</a>
</li>
<li>
<a href="{% url "ietf.stats.views.meeting_stats" %}">Meeting attendance</a>
</li>
<li>
<a rel="nofollow" href="{% url "ietf.stats.views.review_stats" %}">Reviews of Internet-Drafts in review teams</a>
(requires login)
</li>
</ul>
<p>
Statistics on meetings and authorship are not currently available.
</p>
{% endblock %}

35
ietf/templates/stats/meeting_stats.html
View file

@ -1,35 +0,0 @@
{% extends "base.html" %}
{% load origin %}
{% load ietf_filters static django_bootstrap5 %}
{% block title %}{{ stats_title }}{% endblock %}
{% block pagehead %}
<link rel="stylesheet" href="{% static 'ietf/css/list.css' %}">
<link rel="stylesheet" href="{% static "ietf/css/highcharts.css" %}">
{% endblock %}
{% block content %}
{% origin %}
<h1>Meeting Statistics</h1>
{% if meeting %}
<p>
<a href="{% url "ietf.stats.views.meeting_stats" %}">&laquo; Back to overview</a>
</p>
{% endif %}
<div class="row my-3">
<label class="fw-bold col-sm-2 col-form-label">Attendees:</label>
<div class="btn-group col-sm-10">
{% for slug, label, url in possible_stats_types %}
<a class="btn btn-outline-primary
{% if slug == stats_type %}
active
{% endif %}"
href="{{ url }}">{{ label }}</a>
{% endfor %}
</div>
</div>
<div class="document-stats">{% include content_template %}</div>
{% endblock %}
{% block js %}
<script src="{% static 'ietf/js/highcharts.js' %}"></script>
<script src="{% static 'ietf/js/stats.js' %}"></script>
<script src="{% static "ietf/js/list.js" %}"></script>
{% endblock %}

61
ietf/templates/stats/meeting_stats_continent.html
View file

@ -1,61 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'column'
},
plotOptions: {
column: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
type: "category",
title: {
text: 'Continent'
}
},
yAxis: {
title: {
text: 'Number of attendees'
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.points[0].key + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<h2>Data</h2>
<table class="table table-sm table-striped stats-data tablesorter">
<thead>
<tr>
<th scope="col" data-sort="continent">Continent</th>
<th scope="col" data-sort="percent">Percentage of attendees</th>
<th scope="col" data-sort="count">Attendees</th>
</tr>
</thead>
<tbody>
{% for continent, percentage, count, names in table_data %}
<tr>
<td>{{ continent|default:"(unknown)" }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
</tr>
{% endfor %}
</tbody>
</table>

97
ietf/templates/stats/meeting_stats_country.html
View file

@ -1,97 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart"></div>
<script>
var chartConf = {
chart: {
type: 'column'
},
plotOptions: {
column: {
animation: false
}
},
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
type: "category",
title: {
text: 'Country'
}
},
yAxis: {
title: {
text: 'Number of attendees'
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.points[0].key + '</b>';
$.each(this.points, function () {
s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
});
return s;
},
shared: true
},
series: {{ chart_data }}
};
</script>
<div id="pie-chart"></div>
<script>
var pieChartConf = {
chart: {
type: 'pie'
},
plotOptions: {
pie: {
animation: false,
dataLabels: {
enabled: true,
format: "{point.name}: {point.percentage:.1f}%"
},
enableMouseTracking: false
}
},
title: {
text: "Countries at IETF {{ meeting.number }}"
},
tooltip: {
},
series: [ {
name: "Countries",
colorByPoint: true,
data: {{ piechart_data }}
}]
};
</script>
<h2>Data</h2>
<table class="table table-sm stats-data tablesorter">
<thead>
<tr>
<th scope="col">Country</th>
<th scope="col">Percentage of attendees</th>
<th scope="col">Attendees</th>
</tr>
</thead>
<tbody>
{% for country, percentage, count, names in table_data %}
<tr>
<td>{{ country|default:"(unknown)" }}</td>
<td>{{ percentage|floatformat:2 }}%</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
</tr>
{% endfor %}
</tbody>
</table>
<p>
EU (European Union) is not a country, but has been added for reference, as the sum of
all current EU member countries:
{% for c in eu_countries %}
{{ c.name }}{% if not forloop.last %},{% endif %}
{% endfor %}
.
</p>

160
ietf/templates/stats/meeting_stats_overview.html
View file

@ -1,160 +0,0 @@
{% load origin %}
{% origin %}
<div id="chart" class="chart-{{ stats_type }}"></div>
<script>
var chartConf = {
{% if stats_type == "overview" %}
chart: {
type: 'column',
},
colors: [ 'blue', 'red', 'cyan', 'orange', 'black',
'green', 'yellow' ],
plotOptions: {
series: {
stacking: 'normal',
point: {
events: {
click: function () {
location.href = this.options.url;
}
}
},
},
column: {
marker: {
enabled: false
},
animation: false
}
},
tooltip: {
formatter: function () {
var s = '<b>' + this.point.name + '</b>';
s += '<br>' + this.point.date;
s += '<br>' + this.series.name + '<br> Attendees: ' + this.y;
return s;
},
shared: false
},
legend: {
title: {
text: "Continent of the venue",
},
borderWidth: 1,
align: "center",
verticalAlign: "bottom",
layout: "horizontal",
enabled: true
},
{% else %}
chart: {
type: 'line',
},
plotOptions: {
line: {
marker: {
enabled: false
},
animation: false
}
},
tooltip: {
formatter: function () {
var s = '<b>' + "IETF " + this.x + '</b>';
$.each(this.points, function () {
s += '<br>' + this.series.name + ': ' + this.y;
});
return s;
},
shared: true
},
legend: {
align: "right",
verticalAlign: "middle",
layout: "vertical",
enabled: true
},
{% endif %}
title: {
text: '{{ stats_title|escapejs }}'
},
xAxis: {
tickInterval: 1,
title: {
text: 'Meeting'
}
},
yAxis: {
min: 0,
title: {
text: 'Attendees at meeting'
}
},
exporting: {
buttons: {
contextButton: {
menuItems: [
'printChart',
'separator',
'downloadPNG',
'downloadJPEG',
'downloadSVG',
'separator',
'downloadCSV'
]
}
}
},
series: {{ chart_data }}
};
</script>
{% if table_data %}
<h2>Data</h2>
<table class="table table-sm table-striped stats-data tablesorter">
<thead>
<tr>
<th scope="col" data-sort="num">Meeting</th>
<th scope="col" data-sort="date">Date</th>
<th scope="col" data-sort="city">City</th>
<th scope="col" data-sort="country">Country</th>
<th scope="col" data-sort="continent">Continent</th>
<th scope="col" data-sort="count">Attendees</th>
</tr>
</thead>
<tbody>
{% for meeting, url, count, country in table_data %}
<tr>
{% if meeting.get_number > 71 %}
<td>
<a href="{{ url }}">{{ meeting.number }}</a>
</td>
<td>{{ meeting.date }}</td>
<td>
<a href="{{ url }}">{{ meeting.city }}</a>
</td>
<td>{{ country.name }}</td>
<td>{{ country.continent }}</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
{% else %}
<td>{{ meeting.number }}</td>
<td>{{ meeting.date }}</td>
<td>{{ meeting.city }}</td>
<td>{{ country.name }}</td>
<td>{{ country.continent }}</td>
<td>{% include "stats/includes/number_with_details_cell.html" %}</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}