# Copyright The IETF Trust 2016-2020, All Rights Reserved # -*- coding: utf-8 -*- import calendar import datetime import itertools import json import dateutil.relativedelta from collections import defaultdict from django.contrib.auth.decorators import login_required from django.http import HttpResponseRedirect from django.shortcuts import render from django.urls import reverse as urlreverse import debug # pyflakes:ignore from ietf.review.utils import (extract_review_assignment_data, aggregate_raw_period_review_assignment_stats, ReviewAssignmentData, sum_period_review_assignment_stats, sum_raw_review_assignment_aggregations) from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName from ietf.ietfauth.utils import has_role from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO def stats_index(request): return render(request, "stats/index.html") def generate_query_string(query_dict, overrides): query_part = "" if query_dict or overrides: d = query_dict.copy() for k, v in overrides.items(): if type(v) in (list, tuple): if not v: if k in d: del d[k] else: d.setlist(k, v) else: if v is None or v == "": if k in d: del d[k] else: d[k] = v if d: query_part = "?" + d.urlencode() return query_part def get_choice(request, get_parameter, possible_choices, multiple=False): # the statistics are built with links to make navigation faster, # so we don't really have a form in most cases, so just use this # helper instead to select between the choices values = request.GET.getlist(get_parameter) found = [t[0] for t in possible_choices if t[0] in values] if multiple: return found else: if found: return found[0] else: return None def add_url_to_choices(choices, url_builder): return [ (slug, label, url_builder(slug)) for slug, label in choices] def put_into_bin(value, bin_size): if value is None: return (0, '') v = (value // bin_size) * bin_size return (v, "{} - {}".format(v, v + bin_size - 1)) def prune_unknown_bin_with_known(bins): # remove from the unknown bin all authors within the # named/known bins all_known = { n for b, names in bins.items() if b for n in names } bins[""] = [name for name in bins[""] if name not in all_known] if not bins[""]: del bins[""] def count_bins(bins): return len({ n for b, names in bins.items() if b for n in names }) def add_labeled_top_series_from_bins(chart_data, bins, limit): """Take bins on the form (x, label): [name1, name2, ...], figure out how many there are per label, take the overall top ones and put them into sorted series like [(x1, len(names1)), (x2, len(names2)), ...].""" aggregated_bins = defaultdict(set) xs = set() for (x, label), names in bins.items(): xs.add(x) aggregated_bins[label].update(names) xs = list(sorted(xs)) sorted_bins = sorted(aggregated_bins.items(), key=lambda t: len(t[1]), reverse=True) top = [ label for label, names in list(sorted_bins)[:limit]] for label in top: series_data = [] for x in xs: names = bins.get((x, label), set()) series_data.append((x, len(names))) chart_data.append({ "data": series_data, "name": label }) def document_stats(request, stats_type=None): return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) def known_countries_list(request, stats_type=None, acronym=None): countries = CountryName.objects.prefetch_related("countryalias_set") for c in countries: # the sorting is a bit of a hack - it puts the ISO code first # since it was added in a migration c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) return render(request, "stats/known_countries_list.html", { "countries": countries, }) def meeting_stats(request, num=None, stats_type=None): return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) @login_required def review_stats(request, stats_type=None, acronym=None): # This view is a bit complex because we want to show a bunch of # tables with various filtering options, and both a team overview # and a reviewers-within-team overview - and a time series chart. # And in order to make the UI quick to navigate, we're not using # one big form but instead presenting a bunch of immediate # actions, with a URL scheme where the most common options (level # and statistics type) are incorporated directly into the URL to # be a bit nicer. def build_review_stats_url(stats_type_override=Ellipsis, acronym_override=Ellipsis, get_overrides=None): if get_overrides is None: get_overrides = {} kwargs = { "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, } acr = acronym if acronym_override is Ellipsis else acronym_override if acr: kwargs["acronym"] = acr return urlreverse(review_stats, kwargs=kwargs) + generate_query_string(request.GET, get_overrides) # which overview - team or reviewer if acronym: level = "reviewer" else: level = "team" # statistics type - one of the tables or the chart possible_stats_types = [ ("completion", "Completion status"), ("results", "Review results"), ("states", "Assignment states"), ] if level == "team": possible_stats_types.append(("time", "Changes over time")) possible_stats_types = add_url_to_choices(possible_stats_types, lambda slug: build_review_stats_url(stats_type_override=slug)) if not stats_type: return HttpResponseRedirect(build_review_stats_url(stats_type_override=possible_stats_types[0][0])) # what to count possible_count_choices = add_url_to_choices([ ("", "Review requests"), ("pages", "Reviewed pages"), ], lambda slug: build_review_stats_url(get_overrides={ "count": slug })) count = get_choice(request, "count", possible_count_choices) or "" # time range def parse_date(s): if not s: return None try: return datetime.datetime.strptime(s.strip(), "%Y-%m-%d").date() except ValueError: return None today = date_today(DEADLINE_TZINFO) from_date = parse_date(request.GET.get("from")) or today - dateutil.relativedelta.relativedelta(years=1) to_date = parse_date(request.GET.get("to")) or today from_time = datetime.datetime.combine(from_date, datetime.time.min, tzinfo=DEADLINE_TZINFO) to_time = datetime.datetime.combine(to_date, datetime.time.max, tzinfo=DEADLINE_TZINFO) # teams/reviewers teams = list(Group.objects.exclude(reviewrequest=None).distinct().order_by("name")) reviewer_filter_args = {} # - interlude: access control if has_role(request.user, ["Secretariat", "Area Director"]): pass else: secr_access = set() reviewer_only_access = set() for r in Role.objects.filter(person__user=request.user, name__in=["secr", "reviewer"], group__in=teams).distinct(): if r.name_id == "secr": secr_access.add(r.group_id) reviewer_only_access.discard(r.group_id) elif r.name_id == "reviewer": if not r.group_id in secr_access: reviewer_only_access.add(r.group_id) if not secr_access and not reviewer_only_access: permission_denied(request, "You do not have the necessary permissions to view this page") teams = [t for t in teams if t.pk in secr_access or t.pk in reviewer_only_access] for t in reviewer_only_access: reviewer_filter_args[t] = { "user": request.user } reviewers_for_team = None if level == "team": for t in teams: t.reviewer_stats_url = build_review_stats_url(acronym_override=t.acronym) query_teams = teams query_reviewers = None group_by_objs = { t.pk: t for t in query_teams } group_by_index = ReviewAssignmentData._fields.index("team") elif level == "reviewer": for t in teams: if t.acronym == acronym: reviewers_for_team = t break else: return HttpResponseRedirect(urlreverse(review_stats)) query_reviewers = list(Person.objects.filter( email__reviewassignment__review_request__time__gte=from_time, email__reviewassignment__review_request__time__lte=to_time, email__reviewassignment__review_request__team=reviewers_for_team, **reviewer_filter_args.get(t.pk, {}) ).distinct()) query_reviewers.sort(key=lambda p: p.last_name()) query_teams = [t] group_by_objs = { r.pk: r for r in query_reviewers } group_by_index = ReviewAssignmentData._fields.index("reviewer") # now filter and aggregate the data possible_teams = possible_completion_types = possible_results = possible_states = None selected_teams = selected_completion_type = selected_result = selected_state = None if stats_type == "time": possible_teams = [(t.acronym, t.acronym) for t in teams] selected_teams = get_choice(request, "team", possible_teams, multiple=True) def add_if_exists_else_subtract(element, l): if element in l: return [x for x in l if x != element] else: return l + [element] possible_teams = add_url_to_choices( possible_teams, lambda slug: build_review_stats_url(get_overrides={ "team": add_if_exists_else_subtract(slug, selected_teams) }) ) query_teams = [t for t in query_teams if t.acronym in selected_teams] extracted_data = extract_review_assignment_data(query_teams, query_reviewers, from_time, to_time) req_time_index = ReviewAssignmentData._fields.index("req_time") def time_key_fn(t): d = t[req_time_index].date() #d -= datetime.timedelta(days=d.weekday()) # weekly # NOTE: Earlier releases had an off-by-one error here - some stat counts may move a month. d -= datetime.timedelta(days=d.day-1) # monthly return d found_results = set() found_states = set() aggrs = [] for d, request_data_items in itertools.groupby(extracted_data, key=time_key_fn): raw_aggr = aggregate_raw_period_review_assignment_stats(request_data_items, count=count) aggr = sum_period_review_assignment_stats(raw_aggr) aggrs.append((d, aggr)) for slug in aggr["result"]: found_results.add(slug) for slug in aggr["state"]: found_states.add(slug) results = ReviewResultName.objects.filter(slug__in=found_results) states = ReviewAssignmentStateName.objects.filter(slug__in=found_states) # choice possible_completion_types = add_url_to_choices([ ("completed_in_time_or_late", "Completed (in time or late)"), ("not_completed", "Not completed"), ("average_assignment_to_closure_days", "Avg. compl. days"), ], lambda slug: build_review_stats_url(get_overrides={ "completion": slug, "result": None, "state": None })) selected_completion_type = get_choice(request, "completion", possible_completion_types) possible_results = add_url_to_choices( [(r.slug, r.name) for r in results], lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": slug, "state": None }) ) selected_result = get_choice(request, "result", possible_results) possible_states = add_url_to_choices( [(s.slug, s.name) for s in states], lambda slug: build_review_stats_url(get_overrides={ "completion": None, "result": None, "state": slug }) ) selected_state = get_choice(request, "state", possible_states) if not selected_completion_type and not selected_result and not selected_state: selected_completion_type = "completed_in_time_or_late" standard_color = '#3d22b3' if selected_completion_type == 'completed_in_time_or_late': graph_data = [ {'label': 'in time', 'color': standard_color, 'data': []}, {'label': 'late', 'color': '#b42222', 'data': []} ] else: graph_data = [{'color': standard_color, 'data': []}] if selected_completion_type == "completed_combined": pass else: for d, aggr in aggrs: v1 = 0 v2 = None js_timestamp = calendar.timegm(d.timetuple()) * 1000 if selected_completion_type == 'completed_in_time_or_late': v1 = aggr['completed_in_time'] v2 = aggr['completed_late'] elif selected_completion_type is not None: v1 = aggr[selected_completion_type] elif selected_result is not None: v1 = aggr["result"][selected_result] elif selected_state is not None: v1 = aggr["state"][selected_state] graph_data[0]['data'].append((js_timestamp, v1)) if v2 is not None: graph_data[1]['data'].append((js_timestamp, v2)) data = json.dumps(graph_data) else: # tabular data extracted_data = extract_review_assignment_data(query_teams, query_reviewers, from_time, to_time, ordering=[level]) data = [] found_results = set() found_states = set() raw_aggrs = [] for group_pk, request_data_items in itertools.groupby(extracted_data, key=lambda t: t[group_by_index]): raw_aggr = aggregate_raw_period_review_assignment_stats(request_data_items, count=count) raw_aggrs.append(raw_aggr) aggr = sum_period_review_assignment_stats(raw_aggr) # skip zero-valued rows if aggr["open"] == 0 and aggr["completed"] == 0 and aggr["not_completed"] == 0: continue aggr["obj"] = group_by_objs.get(group_pk) for slug in aggr["result"]: found_results.add(slug) for slug in aggr["state"]: found_states.add(slug) data.append(aggr) # add totals row if len(raw_aggrs) > 1: totals = sum_period_review_assignment_stats(sum_raw_review_assignment_aggregations(raw_aggrs)) totals["obj"] = "Totals" data.append(totals) results = ReviewResultName.objects.filter(slug__in=found_results) states = ReviewAssignmentStateName.objects.filter(slug__in=found_states) # massage states/results breakdowns for template rendering for aggr in data: aggr["state_list"] = [aggr["state"].get(x.slug, 0) for x in states] aggr["result_list"] = [aggr["result"].get(x.slug, 0) for x in results] return render(request, 'stats/review_stats.html', { "team_level_url": build_review_stats_url(acronym_override=None), "level": level, "reviewers_for_team": reviewers_for_team, "teams": teams, "data": data, "states": states, "results": results, # options "possible_stats_types": possible_stats_types, "stats_type": stats_type, "possible_count_choices": possible_count_choices, "count": count, "from_date": from_date, "to_date": to_date, "today": today, # time options "possible_teams": possible_teams, "selected_teams": selected_teams, "possible_completion_types": possible_completion_types, "selected_completion_type": selected_completion_type, "possible_results": possible_results, "selected_result": selected_result, "possible_states": possible_states, "selected_state": selected_state, })