feat: Area director workload summary view (#4315)

* feat: Add new page as requested in #4242 to list all area directors
and their current workload. Include links to the specific dashboards
for each area director. This new page is in doc/ad/.

* feat: Add new page as requested in #4242 to list all area directors
and their current workload. Include links to the specific dashboards
for each area director. This new page is in doc/ad/.

* Fixed issues from the previous commit by renaming hash to get_hash.

* Making outer () to be non matching

* Fixed RFC Ed Queue Internet-Draft to RFC Ed Queue

* refactor: split the /ad view apart from the /ad/name view.

* fix: make the new template html valid.

* test: start building a test for the new view

* refactor: make the view testable and test it.

* chore: remove unneeded commented lines

* fix: avoid parenthsized-string-looks-like-tuple bug.

* fix: repair bad closing tag in template

Co-authored-by: Tero Kivinen <kivinen@iki.fi>

ietf/doc/factories.py

@@ -290,6 +290,12 @@ class StatusChangeFactory(BaseDocumentFactory):
 
 class ConflictReviewFactory(BaseDocumentFactory):
     type_id='conflrev'
+
+    group = factory.SubFactory('ietf.group.factories.GroupFactory',acronym='none')
+
+    @factory.lazy_attribute_sequence
+    def name(self, n):
+        return draft_name_generator(self.type_id,self.group,n).replace('conflrev-','conflict-review-')
     
     @factory.post_generation
     def review_of(obj, create, extracted, **kwargs):
@@ -298,7 +304,8 @@ class ConflictReviewFactory(BaseDocumentFactory):
         if extracted:
             obj.relateddocument_set.create(relationship_id='conflrev',target=extracted.docalias.first())
         else:
-            obj.relateddocument_set.create(relationship_id='conflrev',target=DocumentFactory(type_id='draft',group=Group.objects.get(type_id='individ')).docalias.first())
+            obj.relateddocument_set.create(relationship_id='conflrev',target=DocumentFactory(name=obj.name.replace('conflict-review-','draft-'),type_id='draft',group=Group.objects.get(type_id='individ')).docalias.first())
+
 
     @factory.post_generation
     def states(obj, create, extracted, **kwargs):

ietf/doc/tests.py

@@ -10,12 +10,14 @@ import bibtexparser
 import mock
 import json
 import copy
+import random
 
 from http.cookies import SimpleCookie
 from pathlib import Path
 from pyquery import PyQuery
 from urllib.parse import urlparse, parse_qs
 from tempfile import NamedTemporaryFile
+from collections import defaultdict
 
 from django.core.management import call_command
 from django.urls import reverse as urlreverse
@@ -36,10 +38,11 @@ from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactor
     ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, 
     IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, 
     BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory,
-    StatusChangeFactory)
+    StatusChangeFactory, BofreqFactory)
 from ietf.doc.fields import SearchableDocumentsField
 from ietf.doc.utils import create_ballot_if_not_open, uppercase_std_abbreviated_name
-from ietf.group.models import Group
+from ietf.doc.views_search import ad_dashboard_group, ad_dashboard_group_type, shorten_group_name # TODO: red flag that we're importing from views in tests. Move these to utils.
+from ietf.group.models import Group, Role
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.ipr.factories import HolderIprDisclosureFactory
 from ietf.meeting.models import Meeting, SessionPresentation, SchedulingEvent
@@ -228,6 +231,45 @@ class SearchTests(TestCase):
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "Document Search")
 
+    def test_ad_workload(self):
+        Role.objects.filter(name_id='ad').delete()
+        ad = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active',person__name='Example Areadirector').person
+        doc_type_names = ['bofreq', 'charter', 'conflrev', 'draft', 'statchg']
+        expected = defaultdict(lambda :0)
+        for doc_type_name in doc_type_names:
+            if doc_type_name=='draft':
+                states = State.objects.filter(type='draft-iesg', used=True).values_list('slug', flat=True)
+            else:
+                states = State.objects.filter(type=doc_type_name, used=True).values_list('slug', flat=True)
+
+            for state in states:
+                target_num = random.randint(0,2)
+                for _ in range(target_num):
+                    if doc_type_name == 'draft':
+                        doc = IndividualDraftFactory(ad=ad,states=[('draft-iesg', state),('draft','rfc' if state=='pub' else 'active')])
+                    elif doc_type_name == 'charter':
+                        doc = CharterFactory(ad=ad, states=[(doc_type_name, state)])
+                    elif doc_type_name == 'bofreq':
+                        # Note that the view currently doesn't handle bofreqs
+                        doc = BofreqFactory(states=[(doc_type_name, state)], bofreqresponsibledocevent__responsible=[ad])
+                    elif doc_type_name == 'conflrev':
+                        doc = ConflictReviewFactory(ad=ad, states=State.objects.filter(type_id=doc_type_name, slug=state))
+                    elif doc_type_name == 'statchg':
+                        doc = StatusChangeFactory(ad=ad, states=State.objects.filter(type_id=doc_type_name, slug=state))
+                    else:
+                        # Currently unreachable
+                        doc = DocumentFactory(type_id=doc_type_name, ad=ad, states=[(doc_type_name, state)])
+
+                    if not slugify(ad_dashboard_group_type(doc)) in ('document', 'none'):
+                        expected[(slugify(ad_dashboard_group_type(doc)), slugify(ad.full_name_as_key()), slugify(shorten_group_name(ad_dashboard_group(doc))))] += 1
+        
+        url = urlreverse('ietf.doc.views_search.ad_workload')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        for group_type, ad, group in expected:
+            self.assertEqual(int(q(f'#{group_type}-{ad}-{group}').text()),expected[(group_type, ad, group)])
+
     def test_docs_for_ad(self):
         ad = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active').person
         draft = IndividualDraftFactory(ad=ad)

ietf/doc/urls.py

@@ -50,6 +50,7 @@ urlpatterns = [
     url(r'^$', views_search.search),
     url(r'^search/?$', views_search.search),
     url(r'^in-last-call/?$', views_search.drafts_in_last_call),
+    url(r'^ad/?$', views_search.ad_workload),
     url(r'^ad/(?P<name>[^/]+)/?$', views_search.docs_for_ad),
     url(r'^ad2/(?P<name>[\w.-]+)/$', RedirectView.as_view(url='/doc/ad/%(name)s/', permanent=True)),
     url(r'^rfc-status-changes/?$', views_status_change.rfc_status_changes),

ietf/doc/views_search.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2009-2020, All Rights Reserved
+# Copyright The IETF Trust 2009-2022, All Rights Reserved
 # -*- coding: utf-8 -*-
 #
 # Some parts Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
@@ -307,6 +307,32 @@ def search_for_name(request, name):
 
     return cached_redirect(cache_key, urlreverse('ietf.doc.views_search.search') + search_args)
 
+def ad_dashboard_group_type(doc):
+    # Return group type for document for dashboard.
+    # If doc is not defined return list of all possible
+    # group types
+    if not doc:
+        return ('I-D', 'RFC', 'Conflict Review', 'Status Change', 'Charter')
+    if doc.type.slug=='draft':
+        if doc.get_state_slug('draft') == 'rfc':
+            return 'RFC'
+        elif doc.get_state_slug('draft') == 'active' and doc.get_state_slug('draft-iesg') and doc.get_state('draft-iesg').name =='RFC Ed Queue':
+            return 'RFC'
+        elif doc.get_state_slug('draft') == 'active' and doc.get_state_slug('draft-iesg') and doc.get_state('draft-iesg').name in ('Dead', 'I-D Exists', 'AD is watching'):
+             return None
+        elif doc.get_state('draft').name in ('Expired', 'Replaced'):
+            return None
+        else:
+            return 'I-D'
+    elif doc.type.slug=='conflrev':
+          return 'Conflict Review'
+    elif doc.type.slug=='statchg':
+        return 'Status Change'
+    elif doc.type.slug=='charter':
+        return "Charter"
+    else:
+        return "Document"
+
 def ad_dashboard_group(doc):
 
     if doc.type.slug=='draft':
@@ -338,6 +364,12 @@ def ad_dashboard_group(doc):
     else:
         return "Document"
 
+def shorten_group_name(name):
+    for s in [' Internet-Draft', ' Conflict Review', ' Status Change', ' (Internal Steering Group/IAB Review) Charter', 'Charter']:
+        if name.endswith(s):
+            name = name[:-len(s)]
+    return name
+
 def ad_dashboard_sort_key(doc):
 
     if doc.type.slug=='draft' and doc.get_state_slug('draft') == 'rfc':
@@ -393,6 +425,107 @@ def ad_dashboard_sort_key(doc):
 
     return "3%s" % seed
 
+def ad_workload(request):
+    ads = []
+    responsible = Document.objects.values_list('ad', flat=True).distinct()
+    for p in Person.objects.filter(
+        Q(
+            role__name__in=("pre-ad", "ad"),
+            role__group__type="area",
+            role__group__state="active"
+        )
+        | Q(pk__in=responsible)
+    ).distinct():
+        if p in get_active_ads():
+                ads.append(p)
+
+    doctypes = list(DocTypeName.objects.filter(used=True).exclude(slug='draft').values_list("pk", flat=True))
+
+    group_types = ad_dashboard_group_type(None)
+
+    groups = {}
+    group_names = {}
+    for g in group_types:
+        groups[g] = {}
+        group_names[g] = []
+        
+    # Prefill groups in preferred sort order
+    id = 0
+    for g in [
+            'Publication Requested Internet-Draft',
+            'Waiting for Writeup Internet-Draft',
+            'AD Evaluation Internet-Draft',
+            'In Last Call Internet-Draft',
+            'IESG Evaluation - Defer Internet-Draft',
+            'IESG Evaluation Internet-Draft',
+            'Waiting for AD Go-Ahead Internet-Draft',
+            'Approved-announcement to be sent Internet-Draft',
+            'Approved-announcement sent Internet-Draft']:
+        groups['I-D'][g] = id
+        group_names['I-D'].append(g)
+        id += 1;
+    id = 0
+    for g in ['RFC Ed Queue Internet-Draft', 'RFC']:
+        groups['RFC'][g] = id
+        group_names['RFC'].append(g)
+        id += 1;
+    id = 0
+    for g in ['AD Review Conflict Review',
+                'Needs Shepherd Conflict Review',
+                'IESG Evaluation Conflict Review',
+                'Approved Conflict Review',
+                'Withdrawn Conflict Review']:
+        groups['Conflict Review'][g] = id
+        group_names['Conflict Review'].append(g)
+        id += 1;
+    id = 0
+    for g in [ 'Start Chartering/Rechartering (Internal Steering Group/IAB Review) Charter',
+                'Replaced Charter',
+                'Approved Charter',
+                'Not currently under review Charter']:
+        groups['Charter'][g] = id
+        group_names['Charter'].append(g)
+        id += 1;
+
+    for ad in ads:
+        form = SearchForm({'by':'ad','ad': ad.id,
+                            'rfcs':'on', 'activedrafts':'on',
+                            'olddrafts':'on',
+                            'doctypes': doctypes})
+        data = retrieve_search_results(form)
+        ad.dashboard = urlreverse("ietf.doc.views_search.docs_for_ad", kwargs=dict(name=ad.full_name_as_key()))
+        counts = {}
+        for g in group_types:
+            counts[g] = []
+        for doc in data:
+            group_type = ad_dashboard_group_type(doc)
+            if group_type and group_type in groups: # Right now, anything with group_type "Document", such as a bofreq is not handled.
+                group = ad_dashboard_group(doc)
+                if group not in groups[group_type]:
+                    groups[group_type][group] = len(groups[group_type])
+                    group_names[group_type].append(group)
+                if len(counts[group_type]) < len(groups[group_type]):
+                    counts[group_type].extend([0] * (len(groups[group_type]) - len(counts[group_type])))
+                counts[group_type][groups[group_type][group]] += 1
+        ad.counts = counts
+    for ad in ads:
+        for group_type in group_types:
+            if len(ad.counts[group_type]) < len(groups[group_type]):
+                ad.counts[group_type].extend([0] * (len(groups[group_type]) - len(ad.counts[group_type])))
+    # Shorten the names of groups
+    for gt in group_types:
+        for idx,g in enumerate(group_names[gt]):
+            group_names[gt][idx] = shorten_group_name(g)
+
+    workload = []
+    for gt in group_types:
+        workload.append(dict(group_type=gt,group_names=group_names[gt],counts=[(ad, [(group_names[gt][index],ad.counts[gt][index]) for index in range(len(group_names[gt]))]) for ad in ads]))
+
+    return render(request, 'doc/ad_list.html', {
+        'workload': workload
+    })
+        
+
 def docs_for_ad(request, name):
     ad = None
     responsible = Document.objects.values_list('ad', flat=True).distinct()
@@ -454,7 +587,6 @@ def docs_for_ad(request, name):
     return render(request, 'doc/drafts_for_ad.html', {
         'form':form, 'docs':results, 'meta':meta, 'ad_name': ad.plain_name(), 'blocked_docs': blocked_docs
     })
-
 def drafts_in_last_call(request):
     lc_state = State.objects.get(type="draft-iesg", slug="lc").pk
     form = SearchForm({'by':'state','state': lc_state, 'rfcs':'on', 'activedrafts':'on'})

ietf/templates/doc/ad_list.html

@@ -0,0 +1,38 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin static %}
+{% load ietf_filters %}
+{% block pagehead %}
+<link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
+{% endblock %}
+{% block title %}Area directors{% endblock %}
+{% block content %}
+{% origin %}
+<h1>Area Directors Workload</h1>
+{% for group in workload %}
+<h2>{{ group.group_type }}</h2>
+<table class="table table-sm table-striped tablesorter">
+  <thead>
+    <tr>
+      <th scope="col" data-sort="name">Name</th>
+      {% for g in group.group_names %}
+      <th scope="col" class="text-end" data-sort="{{ g|slugify }}-num">{{ g }}</th>
+      {% endfor %}
+    </tr>
+  </thead>
+  <tbody>
+    {% for ad, ad_counts in group.counts %}
+    <tr>
+      <td><a href="{{ ad.dashboard }}">{{ ad.name }}</a></td>
+      {% for label, count in ad_counts %}
+        <td id="{{group.group_type|slugify}}-{{ad.full_name_as_key|slugify}}-{{label|slugify}}">{{count}}</td>
+      {% endfor %}
+    </tr>
+    {% endfor %}
+  </tbody>
+</table>
+{% endfor %}
+{% endblock %}
+{% block js %}
+<script src="{% static "ietf/js/list.js" %}"></script>
+{% endblock %}