chore: merge main and fix new views

.github/workflows/build.yml

@@ -129,7 +129,7 @@ jobs:
         fetch-depth: 0
         
     - name: Setup Node.js
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: '16'
     

.github/workflows/tests.yml

@@ -79,7 +79,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - uses: actions/setup-node@v3
+    - uses: actions/setup-node@v4
       with:
         node-version: '18'
 

dev/coverage-action/package.json

@@ -14,9 +14,9 @@
     "luxon": "3.4.3"
   },
   "devDependencies": {
-    "eslint": "8.51.0",
+    "eslint": "8.52.0",
     "eslint-config-standard": "17.1.0",
-    "eslint-plugin-import": "2.28.1",
+    "eslint-plugin-import": "2.29.0",
     "eslint-plugin-node": "11.1.0",
     "eslint-plugin-promise": "6.1.1",
     "npm-check-updates": "16.14.6"

docker/app.Dockerfile

@@ -11,7 +11,7 @@ COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian
 RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh
 
 # Add Postgresql Apt Repository to get 14    
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \

docker/base.Dockerfile

@@ -2,6 +2,7 @@ FROM python:3.9-bullseye
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
+ENV NODE_MAJOR=16
 
 # Update system packages
 RUN apt-get update \
@@ -9,19 +10,22 @@ RUN apt-get update \
     && apt-get -y install --no-install-recommends apt-utils dialog 2>&1
 
 # Add Node.js Source
-RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -
+RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \
+    && mkdir -p /etc/apt/keyrings\
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
 
 # Add Docker Source
 RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
 RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
-    $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+    $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
     
 # Add PostgreSQL Source 
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 
 # Install the packages we need
-RUN apt-get update --fix-missing && apt-get install -qy \
+RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends \
 	apache2-utils \
 	apt-file \
 	bash \
@@ -30,6 +34,7 @@ RUN apt-get update --fix-missing && apt-get install -qy \
 	default-jdk \
 	docker-ce-cli \
 	enscript \
+	firefox-esr \
 	gawk \
 	g++ \
 	gcc \
@@ -87,11 +92,23 @@ RUN /tmp/app-install-chromedriver.sh
 # Fix /dev/shm permissions for chromedriver
 RUN chmod 1777 /dev/shm
 
+# GeckoDriver
+ARG GECKODRIVER_VERSION=latest
+RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0.33.0"; else echo $GECKODRIVER_VERSION; fi) \
+  && echo "Using GeckoDriver version: "$GK_VERSION \
+  && wget --no-verbose -O /tmp/geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v$GK_VERSION/geckodriver-v$GK_VERSION-linux64.tar.gz \
+  && rm -rf /opt/geckodriver \
+  && tar -C /opt -zxf /tmp/geckodriver.tar.gz \
+  && rm /tmp/geckodriver.tar.gz \
+  && mv /opt/geckodriver /opt/geckodriver-$GK_VERSION \
+  && chmod 755 /opt/geckodriver-$GK_VERSION \
+  && ln -fs /opt/geckodriver-$GK_VERSION /usr/bin/geckodriver
+
 # Activate Yarn
 RUN corepack enable
 
 # Get rid of installation files we don't need in the image, to reduce size
-RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
 
 # "fake" dbus address to prevent errors
 # https://github.com/SeleniumHQ/docker-selenium/issues/87

ietf/doc/migrations/0008_alter_docevent_type.py

@@ -0,0 +1,91 @@
+# Generated by Django 4.2.7 on 2023-11-04 13:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("doc", "0007_alter_docevent_type"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="docevent",
+            name="type",
+            field=models.CharField(
+                choices=[
+                    ("new_revision", "Added new revision"),
+                    ("new_submission", "Uploaded new revision"),
+                    ("changed_document", "Changed document metadata"),
+                    ("added_comment", "Added comment"),
+                    ("added_message", "Added message"),
+                    ("edited_authors", "Edited the documents author list"),
+                    ("deleted", "Deleted document"),
+                    ("changed_state", "Changed state"),
+                    ("changed_stream", "Changed document stream"),
+                    ("expired_document", "Expired document"),
+                    ("extended_expiry", "Extended expiry of document"),
+                    ("requested_resurrect", "Requested resurrect"),
+                    ("completed_resurrect", "Completed resurrect"),
+                    ("changed_consensus", "Changed consensus"),
+                    ("published_rfc", "Published RFC"),
+                    (
+                        "added_suggested_replaces",
+                        "Added suggested replacement relationships",
+                    ),
+                    (
+                        "reviewed_suggested_replaces",
+                        "Reviewed suggested replacement relationships",
+                    ),
+                    ("changed_action_holders", "Changed action holders for document"),
+                    ("changed_group", "Changed group"),
+                    ("changed_protocol_writeup", "Changed protocol writeup"),
+                    ("changed_charter_milestone", "Changed charter milestone"),
+                    ("initial_review", "Set initial review time"),
+                    ("changed_review_announcement", "Changed WG Review text"),
+                    ("changed_action_announcement", "Changed WG Action text"),
+                    ("started_iesg_process", "Started IESG process on document"),
+                    ("created_ballot", "Created ballot"),
+                    ("closed_ballot", "Closed ballot"),
+                    ("sent_ballot_announcement", "Sent ballot announcement"),
+                    ("changed_ballot_position", "Changed ballot position"),
+                    ("changed_ballot_approval_text", "Changed ballot approval text"),
+                    ("changed_ballot_writeup_text", "Changed ballot writeup text"),
+                    ("changed_rfc_editor_note_text", "Changed RFC Editor Note text"),
+                    ("changed_last_call_text", "Changed last call text"),
+                    ("requested_last_call", "Requested last call"),
+                    ("sent_last_call", "Sent last call"),
+                    ("scheduled_for_telechat", "Scheduled for telechat"),
+                    ("iesg_approved", "IESG approved document (no problem)"),
+                    ("iesg_disapproved", "IESG disapproved document (do not publish)"),
+                    ("approved_in_minute", "Approved in minute"),
+                    ("iana_review", "IANA review comment"),
+                    ("rfc_in_iana_registry", "RFC is in IANA registry"),
+                    (
+                        "rfc_editor_received_announcement",
+                        "Announcement was received by RFC Editor",
+                    ),
+                    ("requested_publication", "Publication at RFC Editor requested"),
+                    (
+                        "sync_from_rfc_editor",
+                        "Received updated information from RFC Editor",
+                    ),
+                    ("requested_review", "Requested review"),
+                    ("assigned_review_request", "Assigned review request"),
+                    ("closed_review_request", "Closed review request"),
+                    ("closed_review_assignment", "Closed review assignment"),
+                    ("downref_approved", "Downref approved"),
+                    ("posted_related_ipr", "Posted related IPR"),
+                    ("removed_related_ipr", "Removed related IPR"),
+                    (
+                        "removed_objfalse_related_ipr",
+                        "Removed Objectively False related IPR",
+                    ),
+                    ("changed_editors", "Changed BOF Request editors"),
+                    ("published_statement", "Published statement"),
+                    ("approved_slides", "Slides approved"),
+                ],
+                max_length=50,
+            ),
+        ),
+    ]

ietf/doc/models.py

@@ -1288,6 +1288,9 @@ EVENT_TYPES = [
     # Statement events
     ("published_statement", "Published statement"),
 
+    # Slide events
+    ("approved_slides", "Slides approved"),
+    
     ]
 
 class DocEvent(models.Model):

ietf/doc/templatetags/ietf_filters.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2007-2020, All Rights Reserved
+# Copyright The IETF Trust 2007-2023, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -410,9 +410,9 @@ def startswith(x, y):
     return str(x).startswith(y)
 
 
-@register.filter(name='removesuffix', is_safe=False)
-def removesuffix(value, suffix):
-    """Remove an exact-match suffix
+@register.filter(name='removeprefix', is_safe=False)
+def removeprefix(value, prefix):
+    """Remove an exact-match prefix
     
     The is_safe flag is False because indiscriminate use of this could result in non-safe output.
     See https://docs.djangoproject.com/en/2.2/howto/custom-template-tags/#filters-and-auto-escaping
@@ -420,8 +420,8 @@ def removesuffix(value, suffix):
     HTML-unsafe output.
     """
     base = str(value)
-    if base.endswith(suffix):
-        return base[:-len(suffix)]
+    if base.startswith(prefix):
+        return base[len(prefix):]
     else:
         return base
 

ietf/doc/tests.py

@@ -40,11 +40,10 @@ from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactor
     ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, 
     IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, 
     BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory,
-    StatusChangeFactory, BofreqFactory, DocExtResourceFactory, RgDraftFactory)
+    StatusChangeFactory, DocExtResourceFactory, RgDraftFactory)
 from ietf.doc.forms import NotifyForm
 from ietf.doc.fields import SearchableDocumentsField
 from ietf.doc.utils import create_ballot_if_not_open, uppercase_std_abbreviated_name
-from ietf.doc.views_search import ad_dashboard_group, ad_dashboard_group_type, shorten_group_name # TODO: red flag that we're importing from views in tests. Move these to utils.
 from ietf.group.models import Group, Role
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.ipr.factories import HolderIprDisclosureFactory
@@ -60,6 +59,7 @@ from ietf.utils.test_utils import login_testing_unauthorized, unicontent
 from ietf.utils.test_utils import TestCase
 from ietf.utils.text import normalize_text
 from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO
+from ietf.doc.utils_search import AD_WORKLOAD
 
 
 class SearchTests(TestCase):
@@ -281,43 +281,61 @@ class SearchTests(TestCase):
         self.assertContains(r, "Document Search")
 
     def test_ad_workload(self):
-        Role.objects.filter(name_id='ad').delete()
-        ad = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active',person__name='Example Areadirector').person
-        doc_type_names = ['bofreq', 'charter', 'conflrev', 'draft', 'statchg']
-        expected = defaultdict(lambda :0)
-        for doc_type_name in doc_type_names:
-            if doc_type_name=='draft':
-                states = State.objects.filter(type='draft-iesg', used=True).exclude(slug="rfc").values_list('slug', flat=True)
-            else:
-                states = State.objects.filter(type=doc_type_name, used=True).values_list('slug', flat=True)
-
-            for state in states:
-                target_num = random.randint(0,2)
+        Role.objects.filter(name_id="ad").delete()
+        ad = RoleFactory(
+            name_id="ad",
+            group__type_id="area",
+            group__state_id="active",
+            person__name="Example Areadirector",
+        ).person
+        expected = defaultdict(lambda: 0)
+        for doc_type_slug in AD_WORKLOAD:
+            for state in AD_WORKLOAD[doc_type_slug]:
+                target_num = random.randint(0, 2)
                 for _ in range(target_num):
-                    if doc_type_name == 'draft':
-                        doc = IndividualDraftFactory(ad=ad,states=[('draft-iesg', state),('draft','rfc' if state=='pub' else 'active')])
-                    elif doc_type_name == 'charter':
-                        doc = CharterFactory(ad=ad, states=[(doc_type_name, state)])
-                    elif doc_type_name == 'bofreq':
-                        # Note that the view currently doesn't handle bofreqs
-                        doc = BofreqFactory(states=[(doc_type_name, state)], bofreqresponsibledocevent__responsible=[ad])
-                    elif doc_type_name == 'conflrev':
-                        doc = ConflictReviewFactory(ad=ad, states=State.objects.filter(type_id=doc_type_name, slug=state))
-                    elif doc_type_name == 'statchg':
-                        doc = StatusChangeFactory(ad=ad, states=State.objects.filter(type_id=doc_type_name, slug=state))
-                    else:
-                        # Currently unreachable
-                        doc = DocumentFactory(type_id=doc_type_name, ad=ad, states=[(doc_type_name, state)])
+                    if (
+                        doc_type_slug == "draft"
+                        or doc_type_slug == "rfc"
+                        and state == "rfcqueue"
+                    ):
+                        IndividualDraftFactory(
+                            ad=ad,
+                            states=[
+                                ("draft-iesg", state),
+                                ("draft", "rfc" if state == "pub" else "active"),
+                            ],
+                        )
+                    elif doc_type_slug == "rfc":
+                        WgRfcFactory.create(
+                            states=[("draft", "rfc"), ("draft-iesg", "pub")]
+                        )
 
-                    if not slugify(ad_dashboard_group_type(doc)) in ('document', 'none'):
-                        expected[(slugify(ad_dashboard_group_type(doc)), slugify(ad.full_name_as_key()), slugify(shorten_group_name(ad_dashboard_group(doc))))] += 1
-        
-        url = urlreverse('ietf.doc.views_search.ad_workload')
+                    elif doc_type_slug == "charter":
+                        CharterFactory(ad=ad, states=[(doc_type_slug, state)])
+                    elif doc_type_slug == "conflrev":
+                        ConflictReviewFactory(
+                            ad=ad,
+                            states=State.objects.filter(
+                                type_id=doc_type_slug, slug=state
+                            ),
+                        )
+                    elif doc_type_slug == "statchg":
+                        StatusChangeFactory(
+                            ad=ad,
+                            states=State.objects.filter(
+                                type_id=doc_type_slug, slug=state
+                            ),
+                        )
+        self.client.login(username="ad", password="ad+password")
+        url = urlreverse("ietf.doc.views_search.ad_workload")
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         for group_type, ad, group in expected:
-            self.assertEqual(int(q(f'#{group_type}-{ad}-{group}').text()),expected[(group_type, ad, group)])
+            self.assertEqual(
+                int(q(f"#{group_type}-{ad}-{group}").text()),
+                expected[(group_type, ad, group)],
+            )
 
     def test_docs_for_ad(self):
         ad = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active').person

ietf/doc/urls_review.py

@@ -9,6 +9,7 @@ urlpatterns = [
     url(r'^(?P<request_id>[0-9]+)/$', views_review.review_request),
     url(r'^(?P<request_id>[0-9]+)/login/$', views_review.review_request_forced_login),
     url(r'^(?P<request_id>[0-9]+)/close/$', views_review.close_request),
+    url(r'^(?P<request_id>[0-9]+)/addrequestcomment/$', views_review.add_request_comment),
     url(r'^(?P<request_id>[0-9]+)/assignreviewer/$', views_review.assign_reviewer),
     url(r'^(?P<assignment_id>[0-9]+)/rejectreviewerassignment/$', views_review.reject_reviewer_assignment),
     url(r'^(?P<assignment_id>[0-9]+)/complete/$', views_review.complete_review),

ietf/doc/utils_search.py

@@ -9,7 +9,7 @@ from zoneinfo import ZoneInfo
 
 from django.conf import settings
 
-from ietf.doc.models import Document, RelatedDocument, DocEvent, TelechatDocEvent, BallotDocEvent
+from ietf.doc.models import Document, RelatedDocument, DocEvent, TelechatDocEvent, BallotDocEvent, DocTypeName
 from ietf.doc.expire import expirable_drafts
 from ietf.doc.utils import augment_docs_and_user_with_user_info
 from ietf.meeting.models import SessionPresentation, Meeting, Session
@@ -284,3 +284,86 @@ def prepare_document_table(request, docs, query=None, max_results=200):
             h["sort_url"] = "?" + d.urlencode()
 
     return (docs, meta)
+
+
+# The document types and state slugs to include in the AD dashboard
+# and AD doc list, in the order they should be shown.
+#
+# "rfc" is a custom subset of "draft" that we special-case in the code
+# to break out these docs into a separate table.
+#
+AD_WORKLOAD = {
+    "draft": [
+        "pub-req",
+        "ad-eval",
+        "lc-req",
+        "lc",
+        "writeupw",
+        # "defer",  # probably not a useful state to show, since it's rare
+        "iesg-eva",
+        "goaheadw",
+        "approved",
+        "ann",
+    ],
+    "rfc": [
+        "rfcqueue",
+        "rfc",
+    ],
+    "conflrev": [
+        "needshep",
+        "adrev",
+        "iesgeval",
+        "approved",  # synthesized state for all the "appr-" states
+        # "withdraw",  # probably not a useful state to show
+    ],
+    "statchg": [
+        "needshep",
+        "adrev",
+        "lc-req",
+        "in-lc",
+        "iesgeval",
+        "goahead",
+        "appr-sent",
+        # "dead",  # probably not a useful state to show
+    ],
+    "charter": [
+        "notrev",
+        "infrev",
+        "intrev",
+        "extrev",
+        "iesgrev",
+        "approved",
+        # "replaced",  # probably not a useful state to show
+    ],
+}
+
+
+def doc_type(doc):
+    dt = doc.type.slug
+    if (
+        doc.get_state_slug("draft") == "rfc"
+        or doc.get_state_slug("draft-iesg") == "rfcqueue"
+    ):
+        dt = "rfc"
+    return dt
+
+
+def doc_state(doc):
+    dt = doc.type.slug
+    ds = doc.get_state(dt)
+    if dt == "draft":
+        dis = doc.get_state("draft-iesg")
+        if ds.slug == "active" and dis:
+            return dis.slug
+    elif dt == "conflrev":
+        if ds.slug.startswith("appr"):
+            return "approved"
+    return ds.slug
+
+
+def doc_type_name(doc_type):
+    if doc_type == "rfc":
+        return "RFC"
+    if doc_type == "draft":
+        return "Internet-Draft"
+    return DocTypeName.objects.get(slug=doc_type).name