feat: total ids, pre-pubreq counts and pages left to ballot on on the AD dashboard (#7813)

* feat: Total ids on IESG dashboard

* IESG I-D code comments

* Using Robert's query forIESG dashboard total_ids

* Hiding columns in later IESG Dashboard tables

* Changing IESG dashboard var name to match column table

* Updating IESG pre_pubreqquery

* IESG dashboard prepub req safeParser and graphs

* IESG dashboard fixing Playwright API usage

* IESG dashboard fixing Playwright API usage (2)

* Updating .gitignore for /geckodriver.log

* IESG ad test title

* feat: pages left to ballot on [WIP]

* Adding geckodriver.log to gitignore

* [WIP] pages left to ballot on

* integrating pages left to ballot on WIP

* Tests for ad pages remaining

* Setting states to test ballot items

* refactor ad_pages_left_to_ballot_on count logic

* WIP tests for pages left to ballot on

* chore: remove whitespace change

* fix: look into the BallotPositionDocEventObject

* chore: remove prints

* fix: restructure test

* style: fix js code styling

* fix: only show graph for ADs/Secretariat

---------

Co-authored-by: Matthew Holloway <Matthew Holloway>
Co-authored-by: holloway <matthew@holloway.co.nz>
Co-authored-by: Nicolas Giard <github@ngpixel.com>
Co-authored-by: Matthew Holloway <matthew@staff.ietf.org>

.gitignore

@@ -17,6 +17,7 @@ datatracker.sublime-workspace
 /docker/docker-compose.extend-custom.yml
 /env
 /ghostdriver.log
+/geckodriver.log
 /htmlcov
 /ietf/static/dist-neue
 /latest-coverage.json

ietf/doc/views_search.py

@@ -485,6 +485,29 @@ def ad_workload(request):
         )
         ad.buckets = copy.deepcopy(bucket_template)
 
+        # https://github.com/ietf-tools/datatracker/issues/4577
+        docs_via_group_ad = Document.objects.exclude(
+            group__acronym="none"
+        ).filter(
+            group__role__name="ad",
+            group__role__person=ad
+        ).filter(
+            states__type="draft-stream-ietf",
+            states__slug__in=["wg-doc","wg-lc","waiting-for-implementation","chair-w","writeupw"]
+        )
+
+        doc_for_ad = Document.objects.filter(ad=ad)
+
+        ad.pre_pubreq = (docs_via_group_ad | doc_for_ad).filter(
+            type="draft"
+        ).filter(
+            states__type="draft",
+            states__slug="active"
+        ).filter(
+            states__type="draft-iesg",
+            states__slug="idexists"
+        ).distinct().count()
+
         for doc in Document.objects.exclude(type_id="rfc").filter(ad=ad):
             dt = doc_type(doc)
             state = doc_state(doc)

ietf/iesg/tests.py

@@ -18,7 +18,7 @@ import debug                            # pyflakes:ignore
 
 from ietf.doc.models import DocEvent, BallotPositionDocEvent, TelechatDocEvent
 from ietf.doc.models import Document, State, RelatedDocument
-from ietf.doc.factories import WgDraftFactory, IndividualDraftFactory, ConflictReviewFactory, BaseDocumentFactory, CharterFactory, WgRfcFactory, IndividualRfcFactory
+from ietf.doc.factories import BallotDocEventFactory, BallotPositionDocEventFactory, TelechatDocEventFactory, WgDraftFactory, IndividualDraftFactory, ConflictReviewFactory, BaseDocumentFactory, CharterFactory, WgRfcFactory, IndividualRfcFactory
 from ietf.doc.utils import create_ballot_if_not_open
 from ietf.group.factories import RoleFactory, GroupFactory, DatedGroupMilestoneFactory, DatelessGroupMilestoneFactory
 from ietf.group.models import Group, GroupMilestone, Role
@@ -30,7 +30,6 @@ from ietf.utils.test_utils import TestCase, login_testing_unauthorized, uniconte
 from ietf.iesg.factories import IESGMgmtItemFactory, TelechatAgendaContentFactory
 from ietf.utils.timezone import date_today, DEADLINE_TZINFO
 
-
 class IESGTests(TestCase):
     def test_feed(self):
         draft = WgDraftFactory(states=[('draft','active'),('draft-iesg','iesg-eva')],ad=Person.objects.get(user__username='ad'))
@@ -509,12 +508,13 @@ class IESGAgendaTests(TestCase):
     def test_agenda_documents(self):
         url = urlreverse("ietf.iesg.views.agenda_documents")
         r = self.client.get(url)
+
         self.assertEqual(r.status_code, 200)
 
         for k, d in self.telechat_docs.items():
             self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name, ))
-            self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title, ))
+            self.assertContains(r, d.title, msg_prefix="%s '%s' not in response" % (k, d.title, ))
-
+    
     def test_past_documents(self):
         url = urlreverse("ietf.iesg.views.past_documents")
         # We haven't put any documents on past telechats, so this should be empty
@@ -589,6 +589,66 @@ class IESGAgendaTests(TestCase):
         draft = Document.objects.get(name="draft-ietf-mars-test")
         self.assertEqual(draft.telechat_date(),today)
 
+class IESGAgendaTelechatPagesTests(TestCase):
+    def setUp(self):
+        super().setUp()
+        # make_immutable_test_data made a set of future telechats - only need one
+        # We'll take the "next" one
+        self.telechat_date = get_agenda_date()
+        # make_immutable_test_data made and area with only one ad - give it another
+        ad = Person.objects.get(user__username="ad")
+        adrole = Role.objects.get(person=ad, name="ad")
+        ad2 = RoleFactory(group=adrole.group, name_id="ad").person
+        self.ads=[ad,ad2]
+        
+        # Make some drafts
+        docs = [
+            WgDraftFactory(pages=2, states=[('draft-iesg','iesg-eva'),]),
+            IndividualDraftFactory(pages=20, states=[('draft-iesg','iesg-eva'),]),
+            WgDraftFactory(pages=200, states=[('draft-iesg','iesg-eva'),]),
+        ]
+        # Put them on the telechat
+        for doc in docs:
+            TelechatDocEventFactory(doc=doc, telechat_date=self.telechat_date)
+        # Give them ballots
+        ballots = [BallotDocEventFactory(doc=doc) for doc in docs]
+
+        # Give the "ad" Area-Director a discuss on one 
+        BallotPositionDocEventFactory(balloter=ad, doc=docs[0], pos_id="discuss", ballot=ballots[0])
+        # and a "norecord" position on another
+        BallotPositionDocEventFactory(balloter=ad, doc=docs[1], pos_id="norecord", ballot=ballots[1])
+        # Now "ad" should have 220 pages left to ballot on.
+        # Every other ad should have 222 pages left to ballot on.
+
+    def test_ad_pages_left_to_ballot_on(self):
+        url = urlreverse("ietf.iesg.views.agenda_documents")
+
+        # A non-AD user won't get "pages left"
+        response = self.client.get(url)
+        telechat = response.context["telechats"][0]
+        self.assertEqual(telechat["date"], self.telechat_date)
+        self.assertEqual(telechat["ad_pages_left_to_ballot_on"],0)
+        self.assertNotContains(response,"pages left to ballot on")
+
+        username=self.ads[0].user.username
+        self.assertTrue(self.client.login(username=username, password=f"{username}+password"))
+
+        response = self.client.get(url)
+        telechat = response.context["telechats"][0]
+        self.assertEqual(telechat["ad_pages_left_to_ballot_on"],220)
+        self.assertContains(response,"220 pages left to ballot on")
+
+        self.client.logout()
+        username=self.ads[1].user.username
+        self.assertTrue(self.client.login(username=username, password=f"{username}+password"))
+
+        response = self.client.get(url)
+        telechat = response.context["telechats"][0]
+        self.assertEqual(telechat["ad_pages_left_to_ballot_on"],222)
+
+
+
+
 class RescheduleOnAgendaTests(TestCase):
     def test_reschedule(self):
         draft = WgDraftFactory()

ietf/iesg/utils.py

@@ -7,11 +7,11 @@ from ietf.doc.utils_search import fill_in_telechat_date
 from ietf.iesg.agenda import get_doc_section
 
 
-TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related'])
+TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related','ad_pages_left_to_ballot_on'])
 
-def telechat_page_count(date=None, docs=None):
+def telechat_page_count(date=None, docs=None, ad=None):
     if not date and not docs:
-        return TelechatPageCount(0, 0, 0)
+        return TelechatPageCount(0, 0, 0, 0)
 
     if not docs:
         candidates = Document.objects.filter(docevent__telechatdocevent__telechat_date=date).distinct() 
@@ -24,7 +24,18 @@ def telechat_page_count(date=None, docs=None):
 
     drafts = [d for d in for_approval if d.type_id == 'draft']
 
-    pages_for_approval = sum([d.pages or 0 for d in drafts])
+    ad_pages_left_to_ballot_on = 0
+    pages_for_approval = 0
+    
+    for draft in drafts:
+        pages_for_approval += draft.pages or 0
+        if ad:
+            ballot = draft.active_ballot()
+            if ballot:
+                positions = ballot.active_balloter_positions()
+                ad_position = positions[ad]
+                if ad_position is None or ad_position.pos_id == "norecord":
+                    ad_pages_left_to_ballot_on += draft.pages or 0
 
     pages_for_action = 0
     for d in for_action:
@@ -53,4 +64,5 @@ def telechat_page_count(date=None, docs=None):
     
     return TelechatPageCount(for_approval=pages_for_approval,
                              for_action=pages_for_action,
-                             related=related_pages)
+                             related=related_pages,
+                             ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on)

ietf/iesg/views.py

@@ -360,6 +360,8 @@ def handle_reschedule_form(request, doc, dates, status):
     return form
 
 def agenda_documents(request):
+    ad = request.user.person if has_role(request.user, "Area Director") else None
+
     dates = list(TelechatDate.objects.active().order_by('date').values_list("date", flat=True)[:4])
 
     docs_by_date = dict((d, []) for d in dates)
@@ -389,11 +391,13 @@ def agenda_documents(request):
         # the search_result_row view to display them (which expects them)
         fill_in_document_table_attributes(docs_by_date[date], have_telechat_date=True)
         fill_in_agenda_docs(date, sections, docs_by_date[date])
-        pages = telechat_page_count(docs=docs_by_date[date]).for_approval
+        page_count = telechat_page_count(docs=docs_by_date[date], ad=ad)
-
+        pages = page_count.for_approval
+        
         telechats.append({
                 "date":     date,
                 "pages":    pages,
+                "ad_pages_left_to_ballot_on": page_count.ad_pages_left_to_ballot_on,
                 "sections": sorted((num, section) for num, section in sections.items()
                                    if "2" <= num < "5")
                 })

ietf/templates/doc/ad_list.html

@@ -35,9 +35,12 @@
             <thead class="wrap-anywhere">
                 <tr>
                     <th scope="col" data-sort="name">Area Director</th>
+                    {% if dt.type.1 == "Internet-Draft" %}
+                        <th scope="col" data-sort="pre-pubreq">Pre pubreq</th>
+                    {% endif %}                    
                     {% for state, state_name in dt.states %}
-                        <th scope="col" class="col-1" title=""
+                        <th scope="col" class="col-1" data-sort="{{ state }}-num"
-                            data-sort="{{ state }}-num">
+                        >
                             <a href="{% url 'ietf.doc.views_help.state_help' type='draft-iesg' %}#{{ state }}">
                                 {{ state_name|split:'/'|join:'/<wbr>' }}
                             </a>
@@ -51,6 +54,17 @@
                         <td>
                             <a href="{{ ad.dashboard }}">{{ ad.name }}</a>
                         </td>
+                        {% if dt.type.1 == "Internet-Draft" %}
+                            <td
+                                class="col-1 align-bottom"
+                                data-sum="pre-pubreq"
+                                {% if user|has_role:"Area Director,Secretariat" %}
+                                    data-series-graph
+                                {% endif %}
+                            >
+                                {{ ad.pre_pubreq }}
+                            </td>
+                        {% endif %}            
                         {% for state, state_name in dt.states %}
                         <td class="col-1 align-bottom"
                             id="{{ dt.type.0 }}-{{ ad|slugify }}-{{ state }}">
@@ -63,6 +77,16 @@
             <tfoot class="table-group-divider">
                 <tr>
                     <th scope="row">Sum</th>
+                    {% if dt.type.1 == "Internet-Draft" %}
+                        <td class="align-bottom">
+                            <div
+                                data-sum-result="pre-pubreq"
+                                {% if user|has_role:"Area Director,Secretariat" %}
+                                    data-series-graph
+                                {% endif %}
+                            ></div>
+                        </td>
+                    {% endif %}
                     {% for state, state_name in dt.states %}
                         <td class="align-bottom">
                             <div id="chart-{{ dt.type.0 }}-sum-{{ state }}"></div>
@@ -87,37 +111,151 @@
     </script>
     <script src="{% static "ietf/js/highcharts.js" %}"></script>
     {{ data|json_script:"data" }}
+
     <script>
-        const data = JSON.parse(document.getElementById("data").textContent);
+        function highchartsConfigFactory({ element, ymax, series }){
+            return {
+                title: { text: undefined },
+                chart: {
+                    type: "line",
+                    animation: false,
+                    renderTo: element,
+                    panning: { enabled: false },
+                    spacing: [4, 0, 5, 0],
+                    height: "45%"
+                },
+                scrollbar: { enabled: false },
+                tooltip: { enabled: false },
+                navigator: { enabled: false },
+                exporting: { enabled: false },
+                legend: { enabled: false },
+                credits: { enabled: false },
+                xAxis: {
+                    title: { text: undefined},
+                    labels: { enabled: false },
+                    zoomEnabled: false,
+                    tickLength: 0
+                },
+                yAxis: {
+                    title: { text: undefined},
+                    zoomEnabled: false,
+                    tickLength: 0,
+                    labels: { x: -3 },
+                    min: 0,
+                    max: ymax,
+                    tickInterval: ymax
+                },
+                plotOptions: {
+                    series: {
+                        animation: false,
+                        dataLabels: {
+                            enabled: true,
+                            inside: true,
+                            padding: 0,
+                            formatter: function() {
+                                // only label the last
+                                // (= current-day) point in
+                                // the curve with today's
+                                // value
+                                if (this.point.index + 1 ==
+                                    this.series.points.length) {
+                                    return this.y
+                                }
+                                return undefined
+                            }
+                        }
+                    }
+                },
+                series: [{
+                    name: undefined,
+                    data: series,
+                    enableMouseTracking: false
+                }]
+            }
+        }
+    </script>
+
+    <script>
+        const GRAPH_BUFFER = 2;
+
+        function safeParseFloat(text) {
+            const trimNumber = text.trim()
+            if(!trimNumber.match(/^[0-9.]+$/)) {
+                console.warn(`Unable to parse "${trimNumber}" as a number.`)
+                return Number.NaN
+            }
+            return parseFloat(text)
+        }
+
+        Array.from(document.querySelectorAll("table"))
+            .filter(table => Boolean(table.querySelector("[data-sum]")))
+            .forEach(table => {
+                const sums = Array.from(table.querySelectorAll("[data-sum]")).reduce((
+                    sumsAccumulator,
+                    cell
+                ) => {                    
+                    const key = cell.dataset.sum
+                    const value = safeParseFloat(cell.textContent)
+                    if(key && !isNaN(value)) {
+                        sumsAccumulator[key] = (sumsAccumulator[key] || 0) + (value || 0)
+                    }
+                    return sumsAccumulator
+                }, {})
+
+                Array.from(table.querySelectorAll('[data-sum-result]')).forEach(result => {
+                    const key = result.dataset.sumResult
+                    const value = sums[key]
+
+                    if(value) {
+                        result.innerText = value
+                    }
+                })
+
+                Array.from(table.querySelectorAll('[data-series-graph]')).forEach(element => {
+                    const endValue = safeParseFloat(element.innerText)
+                    if(isNaN(endValue)) throw Error("Can't render Highcharts chart with non-numerical " + element.innerText)
+
+                    const ymax = Math.max(1, endValue + GRAPH_BUFFER)
+
+                    Highcharts.chart(
+                        highchartsConfigFactory({
+                            element,
+                            ymax,
+                            series: [endValue]
+                        })
+                    )
+                })
+            })
+    </script>
+    <script>
+        const data = JSON.parse(document.getElementById("data").textContent)
 
         Object.entries(data).forEach(([dt, ads]) => {
-            max = {};
+            max = {}
             Object.entries(ads).forEach(([ad, states]) => {
                 Object.entries(states).forEach(([state, buckets]) => {
-                    buckets.series = buckets.map((x) => x.length);
+                    buckets.series = buckets.map((x) => x.length)
                     if (ad != "sum") {
-                        max[state] = Math.max(...buckets.series,
+                        max[state] = Math.max(...buckets.series, max[state] ? max[state] : 0)
-                                              max[state] ? max[state] : 0);
                     }
-                });
+                })
-            });
+            })
 
             Object.entries(ads).forEach(([ad, states]) => {
                 Object.entries(states).forEach(([state, buckets]) => {
-                    const cell = `chart-${dt}-${ad}-${state}`;
+                    const cell = `chart-${dt}-${ad}-${state}`
 
                     // if there is only a single datapoint in the
                     // bucket, display it without a graph
                     if (buckets.series.length == 1) {
-                        document.getElementById(cell).innerHTML =
+                        document.getElementById(cell).innerHTML = buckets.series[0]
-                            buckets.series[0];
+                        return
-                        return;
                     }
 
                     // if a bucket has all zeroes, fake a Highcharts
                     // plot with HTML, to reduce the number of plot
                     // objects on the page
-                    if (buckets.series.every((x) => x == 0)) {
+                    if (buckets.series.every((x) => x === 0)) {
                         // document.getElementById(cell).innerHTML = `
                         //     <div class="position-relative">
                         //         <div class="position-absolute bottom-0 start-0">
@@ -128,71 +266,20 @@
                         //         </div>
                         //     </div>
                         // `;
-                        return;
+                        return
                     }
 
                     // else actually create a graph
-                    const ymax = Math.max(1, ad != "sum" ? max[state] : Math.max(...buckets.series));
+                    const ymax = Math.max(1, ad !== "sum" ? max[state] : Math.max(...buckets.series))
-                    Highcharts.chart({
+                    Highcharts.chart(
-                        title: { text: undefined },
+                        highchartsConfigFactory({
-                        chart: {
+                            element: cell,
-                            type: "line",
+                            ymax,
-                            animation: false,
+                            series: buckets.series
-                            renderTo: cell,
+                        })
-                            panning: { enabled: false },
+                    )
-                            spacing: [4, 0, 5, 0],
+                })
-                            height: "45%",
+            })
-                        },
+        })
-                        scrollbar: { enabled: false },
-                        tooltip: { enabled: false },
-                        navigator: { enabled: false },
-                        exporting: { enabled: false },
-                        legend: { enabled: false },
-                        credits: { enabled: false },
-                        xAxis: {
-                            title: { text: undefined},
-                            labels: { enabled: false },
-                            zoomEnabled: false,
-                            tickLength: 0,
-                        },
-                        yAxis: {
-                            title: { text: undefined},
-                            zoomEnabled: false,
-                            tickLength: 0,
-                            labels: { x: -3 },
-                            min: 0,
-                            max: ymax,
-                            tickInterval: ymax,
-                        },
-                        plotOptions: {
-                            series: {
-                                animation: false,
-                                dataLabels: {
-                                    enabled: true,
-                                    inside: true,
-                                    padding: 0,
-                                    formatter: function() {
-                                        // only label the last
-                                        // (= current-day) point in
-                                        // the curve with today's
-                                        // value
-                                        if (this.point.index + 1 ==
-                                            this.series.points.length) {
-                                            return this.y;
-                                        }
-                                        return undefined;
-                                    }
-                                }
-                            }
-                        },
-                        series: [{
-                            name: undefined,
-                            data: buckets.series,
-                            enableMouseTracking: false,
-                        }],
-                    });
-                });
-            });
-        });
     </script>
 {% endblock %}

ietf/templates/iesg/agenda_documents.html

@@ -21,7 +21,12 @@
             <h2>
                 IESG telechat {{ t.date }}
                 <br>
-                <small class="text-body-secondary">{{ t.pages }} page{{ t.pages|pluralize }}</small>
+                <small class="text-body-secondary">
+                    {{ t.pages }} page{{ t.pages|pluralize }}
+                    {% if t.ad_pages_left_to_ballot_on %}
+                        ({{ t.ad_pages_left_to_ballot_on }} pages left to ballot on)
+                    {% endif %}
+                </small>
             </h2>
             <div class="buttonlist">
                 <a class="btn btn-primary" role="button" href="{% url 'ietf.iesg.views.agenda' %}">

playwright/tests-legacy/docs/ad.spec.js

@@ -0,0 +1,26 @@
+const { test, expect } = require('@playwright/test')
+const viewports = require('../../helpers/viewports')
+
+// ====================================================================
+// IESG Dashboard
+// ====================================================================
+
+test.describe('/doc/ad/', () => {
+  test.beforeEach(async ({ page }) => {
+    await page.setViewportSize({
+      width: viewports.desktop[0],
+      height: viewports.desktop[1]
+    })
+
+    await page.goto('/doc/ad/')
+  })
+
+  test('Pre pubreq', async ({ page }) => {
+    const tablesLocator = page.locator('table')
+    const tablesCount = await tablesLocator.count()
+    expect(tablesCount).toBeGreaterThan(0)
+    const firstTable = tablesLocator.nth(0)
+    const theadTexts = await firstTable.locator('thead').allInnerTexts()
+    expect(theadTexts.join('')).toContain('Pre pubreq')
+  })
+})