ci: merge main to release (#7551)

ci: merge main to release

bin/monthly

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-# Weekly datatracker jobs.
-# 
-# This script is expected to be triggered by cron from
-# /etc/cron.d/datatracker
-export LANG=en_US.UTF-8
-export PYTHONIOENCODING=utf-8
-
-DTDIR=/a/www/ietf-datatracker/web
-cd $DTDIR/
-
-# Set up the virtual environment
-source $DTDIR/env/bin/activate
-
-logger -p user.info -t cron "Running $DTDIR/bin/monthly"
-

dev/build/celery-start.sh

@@ -5,6 +5,14 @@
 echo "Running Datatracker checks..."
 ./ietf/manage.py check
 
+if ! ietf/manage.py migrate --skip-checks --check ; then
+    echo "Unapplied migrations found, waiting to start..."
+    sleep 5
+    while ! ietf/manage.py migrate --skip-checks --check ; do 
+        sleep 5
+    done
+fi
+
 cleanup () {
   # Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit.
   if [[ -n "${celery_pid}" ]]; then

dev/build/datatracker-start.sh

@@ -4,14 +4,31 @@ echo "Running Datatracker checks..."
 ./ietf/manage.py check
 
 echo "Running Datatracker migrations..."
-./ietf/manage.py migrate --settings=settings_local
+./ietf/manage.py migrate --skip-checks --settings=settings_local
 
 echo "Starting Datatracker..."
 
+# trap TERM and shut down gunicorn
+cleanup () {
+    if [[ -n "${gunicorn_pid}" ]]; then
+        echo "Terminating gunicorn..."
+        kill -TERM "${gunicorn_pid}"
+        wait "${gunicorn_pid}"
+    fi
+}
+
+trap 'trap "" TERM; cleanup' TERM
+
+# start gunicorn in the background so we can trap the TERM signal
 gunicorn \
           --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \
           --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \
           --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \
           --bind :8000 \
           --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \
-          ietf.wsgi:application
+          --capture-output \
+          --access-logfile -\
+          ${DATATRACKER_GUNICORN_EXTRA_ARGS} \
+          ietf.wsgi:application &
+gunicorn_pid=$!
+wait "${gunicorn_pid}"

ietf/api/tests.py

@@ -10,6 +10,7 @@ import sys
 
 from importlib import import_module
 from pathlib import Path
+from random import randrange
 
 from django.apps import apps
 from django.conf import settings
@@ -1072,8 +1073,20 @@ class CustomApiTests(TestCase):
         self.assertEqual(r.status_code, 400)
         self.assertFalse(any(m.called for m in mocks))
 
-        # test that valid requests call handlers appropriately
+        # bad destination
         message_b64 = base64.b64encode(b"This is a message").decode()
+        r = self.client.post(
+            url,
+            {"dest": "not-a-destination", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "valid-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
+        self.assertFalse(any(m.called for m in mocks))
+
+        # test that valid requests call handlers appropriately
         r = self.client.post(
             url,
             {"dest": "iana-review", "message": message_b64},
@@ -1081,6 +1094,8 @@ class CustomApiTests(TestCase):
             headers={"X-Api-Key": "valid-token"},
         )
         self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "ok"})
         self.assertTrue(mock_iana_ingest.called)
         self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
         self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
@@ -1093,20 +1108,44 @@ class CustomApiTests(TestCase):
             headers={"X-Api-Key": "valid-token"},
         )
         self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "ok"})
         self.assertTrue(mock_ipr_ingest.called)
         self.assertEqual(mock_ipr_ingest.call_args, mock.call(b"This is a message"))
         self.assertFalse(any(m.called for m in (mocks - {mock_ipr_ingest})))
         mock_ipr_ingest.reset_mock()
 
+        # bad nomcom-feedback dest
+        for bad_nomcom_dest in [
+            "nomcom-feedback",  # no suffix
+            "nomcom-feedback-",  # no year
+            "nomcom-feedback-squid",  # not a year,
+            "nomcom-feedback-2024-2025",  # also not a year
+        ]:
+            r = self.client.post(
+                url,
+                {"dest": bad_nomcom_dest, "message": message_b64},
+                content_type="application/json",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 200)
+            self.assertEqual(r.headers["Content-Type"], "application/json")
+            self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
+            self.assertFalse(any(m.called for m in mocks))
+
+        # good nomcom-feedback dest
+        random_year = randrange(100000)
         r = self.client.post(
             url,
-            {"dest": "nomcom-feedback", "message": message_b64, "year": 2024},  # arbitrary year
+            {"dest": f"nomcom-feedback-{random_year}", "message": message_b64},
             content_type="application/json",
             headers={"X-Api-Key": "valid-token"},
         )
         self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "ok"})
         self.assertTrue(mock_nomcom_ingest.called)
-        self.assertEqual(mock_nomcom_ingest.call_args, mock.call(b"This is a message", 2024))
+        self.assertEqual(mock_nomcom_ingest.call_args, mock.call(b"This is a message", random_year))
         self.assertFalse(any(m.called for m in (mocks - {mock_nomcom_ingest})))
         mock_nomcom_ingest.reset_mock()
 
@@ -1118,7 +1157,9 @@ class CustomApiTests(TestCase):
             content_type="application/json",
             headers={"X-Api-Key": "valid-token"},
         )
-        self.assertEqual(r.status_code, 400)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "bad_msg"})
         self.assertTrue(mock_iana_ingest.called)
         self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
         self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
@@ -1138,7 +1179,9 @@ class CustomApiTests(TestCase):
                 content_type="application/json",
                 headers={"X-Api-Key": "valid-token"},
             )
-        self.assertEqual(r.status_code, 400)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "bad_msg"})
         self.assertTrue(mock_iana_ingest.called)
         self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
         self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
@@ -1167,7 +1210,9 @@ class CustomApiTests(TestCase):
                 content_type="application/json",
                 headers={"X-Api-Key": "valid-token"},
             )
-        self.assertEqual(r.status_code, 400)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "bad_msg"})
         self.assertTrue(mock_iana_ingest.called)
         self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
         self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
@@ -1192,7 +1237,9 @@ class CustomApiTests(TestCase):
                 content_type="application/json",
                 headers={"X-Api-Key": "valid-token"},
             )
-        self.assertEqual(r.status_code, 400)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "bad_msg"})
         self.assertTrue(mock_iana_ingest.called)
         self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
         self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))

ietf/api/views.py

@@ -8,6 +8,7 @@ import jsonschema
 import pytz
 import re
 
+from contextlib import suppress
 from django.conf import settings
 from django.contrib.auth import authenticate
 from django.contrib.auth.decorators import login_required
@@ -533,32 +534,13 @@ _response_email_json_validator = jsonschema.Draft202012Validator(
         "type": "object",
         "properties": {
             "dest": {
-                "enum": [
+                "type": "string",
-                    "iana-review",
-                    "ipr-response",
-                    "nomcom-feedback",
-                ]
             },
             "message": {
                 "type": "string",  # base64-encoded mail message
             },
         },
         "required": ["dest", "message"],
-        "if": {
-            # If dest == "nomcom-feedback"...
-            "properties": {
-                "dest": {"const": "nomcom-feedback"},
-            }
-        },
-        "then": {
-            # ... then also require year, an integer, be present
-            "properties": {
-                "year": {
-                    "type": "integer",
-                },
-            },
-            "required": ["year"],
-        },
     }
 )
 
@@ -630,49 +612,63 @@ class EmailIngestionError(Exception):
 @requires_api_token
 @csrf_exempt
 def ingest_email(request):
+    """Ingest incoming email
+    
+    Returns a 4xx or 5xx status code if the HTTP request was invalid or something went
+    wrong while processing it. If the request was valid, returns a 200. This may or may
+    not indicate that the message was accepted.
+    """
 
-    def _err(code, text):
+    def _http_err(code, text):
         return HttpResponse(text, status=code, content_type="text/plain")
 
+    def _api_response(result):
+        return JsonResponse(data={"result": result})
+
     if request.method != "POST":
-        return _err(405, "Method not allowed")
+        return _http_err(405, "Method not allowed")
 
     if request.content_type != "application/json":
-        return _err(415, "Content-Type must be application/json")
+        return _http_err(415, "Content-Type must be application/json")
 
     # Validate
     try:
         payload = json.loads(request.body)
         _response_email_json_validator.validate(payload)
     except json.decoder.JSONDecodeError as err:
-        return _err(400, f"JSON parse error at line {err.lineno} col {err.colno}: {err.msg}")
+        return _http_err(400, f"JSON parse error at line {err.lineno} col {err.colno}: {err.msg}")
     except jsonschema.exceptions.ValidationError as err:
-        return _err(400, f"JSON schema error at {err.json_path}: {err.message}")
+        return _http_err(400, f"JSON schema error at {err.json_path}: {err.message}")
     except Exception:
-        return _err(400, "Invalid request format")
+        return _http_err(400, "Invalid request format")
 
     try:
         message = base64.b64decode(payload["message"], validate=True)
     except binascii.Error:
-        return _err(400, "Invalid message: bad base64 encoding")
+        return _http_err(400, "Invalid message: bad base64 encoding")
 
     dest = payload["dest"]
+    valid_dest = False
     try:
         if dest == "iana-review":
+            valid_dest = True
             iana_ingest_review_email(message)
         elif dest == "ipr-response":
+            valid_dest = True
             ipr_ingest_response_email(message)
-        elif dest == "nomcom-feedback":
+        elif dest.startswith("nomcom-feedback-"):
-            year = payload["year"]
+            maybe_year = dest[len("nomcom-feedback-"):]
-            nomcom_ingest_feedback_email(message, year)
+            if maybe_year.isdecimal():
-        else:
+                valid_dest = True
-            # Should never get here - json schema validation should enforce the enum
+                nomcom_ingest_feedback_email(message, int(maybe_year))
-            log.unreachable(date="2024-04-04")
-            return _err(400, "Invalid dest")  # return something reasonable if we got here unexpectedly
     except EmailIngestionError as err:
         error_email = err.as_emailmessage()
         if error_email is not None:
-            send_smtp(error_email)
+            with suppress(Exception): # send_smtp logs its own exceptions, ignore them here
-        return _err(400, err.msg)
+                send_smtp(error_email)
+        return _api_response("bad_msg")
 
-    return HttpResponse(status=200)
+    if not valid_dest:
+        return _api_response("bad_dest")
+
+    return _api_response("ok")

ietf/group/tasks.py

@@ -14,6 +14,7 @@ from ietf.utils import log
 
 from .models import Group
 from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles
+from .views import extract_last_name, roles
 
 
 @shared_task
@@ -59,3 +60,40 @@ def generate_wg_charters_files_task():
                 log.log(
                     f"Error copying {charters_by_acronym_file} to {charter_copy_dest}: {err}"
                 )
+
+
+@shared_task
+def generate_wg_summary_files_task():
+    # Active WGs (all should have a parent, but filter to be sure)
+    groups = (
+        Group.objects.filter(type="wg", state="active")
+        .exclude(parent=None)
+        .order_by("acronym")
+    )
+    # Augment groups with chairs list
+    for group in groups:
+        group.chairs = sorted(roles(group, "chair"), key=extract_last_name)
+
+    # Active areas with one or more active groups in them 
+    areas = Group.objects.filter(
+        type="area",
+        state="active",
+        group__in=groups,
+    ).distinct().order_by("name")
+    # Augment areas with their groups
+    for area in areas:
+        area.groups = [g for g in groups if g.parent_id == area.pk]
+    summary_path = Path(settings.GROUP_SUMMARY_PATH)
+    summary_file = summary_path / "1wg-summary.txt"
+    summary_file.write_text(
+        render_to_string("group/1wg-summary.txt", {"areas": areas}),
+        encoding="utf8",
+    )
+    summary_by_acronym_file = summary_path / "1wg-summary-by-acronym.txt"
+    summary_by_acronym_file.write_text(
+        render_to_string(
+            "group/1wg-summary-by-acronym.txt",
+            {"areas": areas, "groups": groups},
+        ),
+        encoding="utf8",
+    )

ietf/group/tests_info.py

@@ -8,7 +8,7 @@ import datetime
 import io
 import bleach
 
-from unittest.mock import patch
+from unittest.mock import call, patch
 from pathlib import Path
 from pyquery import PyQuery
 from tempfile import NamedTemporaryFile
@@ -16,6 +16,7 @@ from tempfile import NamedTemporaryFile
 import debug                            # pyflakes:ignore
 
 from django.conf import settings
+from django.http import Http404, HttpResponse
 from django.test import RequestFactory
 from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
@@ -35,7 +36,8 @@ from ietf.group.factories import (GroupFactory, RoleFactory, GroupEventFactory,
     DatedGroupMilestoneFactory, DatelessGroupMilestoneFactory)
 from ietf.group.forms import GroupForm
 from ietf.group.models import Group, GroupEvent, GroupMilestone, GroupStateTransitions, Role
-from ietf.group.tasks import generate_wg_charters_files_task
+from ietf.group.tasks import generate_wg_charters_files_task, generate_wg_summary_files_task
+from ietf.group.views import response_from_file
 from ietf.group.utils import save_group_in_history, setup_default_community_list_for_group
 from ietf.meeting.factories import SessionFactory
 from ietf.name.models import DocTagName, GroupStateName, GroupTypeName, ExtResourceName, RoleName
@@ -58,7 +60,11 @@ def pklist(docs):
     return [ str(doc.pk) for doc in docs.all() ]
 
 class GroupPagesTests(TestCase):
-    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CHARTER_PATH', 'CHARTER_COPY_PATH']
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [
+        "CHARTER_PATH",
+        "CHARTER_COPY_PATH",
+        "GROUP_SUMMARY_PATH",
+    ]
 
     def test_active_groups(self):
         area = GroupFactory.create(type_id='area')
@@ -112,63 +118,90 @@ class GroupPagesTests(TestCase):
             self.assertContains(r, draft.name)
             self.assertContains(r, draft.title)
 
-    def test_wg_summaries(self):
+    def test_response_from_file(self):
-        group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area')).group
+        # n.b., GROUP_SUMMARY_PATH is a temp dir that will be cleaned up automatically
-        RoleFactory(group=group,name_id='chair',person=PersonFactory())
+        fp = Path(settings.GROUP_SUMMARY_PATH) / "some-file.txt"
-        RoleFactory(group=group,name_id='ad',person=PersonFactory())
+        fp.write_text("This is a charters file with an é")
-
+        r = response_from_file(fp)
-        chair = Email.objects.filter(role__group=group, role__name="chair")[0]
-
-        url = urlreverse('ietf.group.views.wg_summary_area', kwargs=dict(group_type="wg"))
-        r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        self.assertContains(r, group.parent.name)
+        self.assertEqual(r.headers["Content-Type"], "text/plain; charset=utf-8")
-        self.assertContains(r, group.acronym)
-        self.assertContains(r, group.name)
-        self.assertContains(r, chair.address)
-
-        url = urlreverse('ietf.group.views.wg_summary_acronym', kwargs=dict(group_type="wg"))
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 200)
-        self.assertContains(r, group.acronym)
-        self.assertContains(r, group.name)
-        self.assertContains(r, chair.address)
-        
-    def test_wg_charters(self):
-        # file does not exist = 404
-        url = urlreverse("ietf.group.views.wg_charters", kwargs=dict(group_type="wg"))
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 404)
-
-        # should return expected file with expected encoding
-        wg_path = Path(settings.CHARTER_PATH) / "1wg-charters.txt"
-        wg_path.write_text("This is a charters file with an é")
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 200)
-        self.assertEqual(r.charset, "UTF-8")
         self.assertEqual(r.content.decode("utf8"), "This is a charters file with an é")
+        # now try with a nonexistent file
+        fp.unlink()
+        with self.assertRaises(Http404):
+            response_from_file(fp)
 
-        # non-wg request = 404 even if the file exists
+    @patch("ietf.group.views.response_from_file")
-        url = urlreverse("ietf.group.views.wg_charters", kwargs=dict(group_type="rg"))
+    def test_wg_summary_area(self, mock):
-        r = self.client.get(url)
+        r = self.client.get(
+            urlreverse("ietf.group.views.wg_summary_area", kwargs={"group_type": "rg"})
+        )  # not wg
         self.assertEqual(r.status_code, 404)
-
+        self.assertFalse(mock.called)
-    def test_wg_charters_by_acronym(self):
+        mock.return_value = HttpResponse("yay")
-        url = urlreverse("ietf.group.views.wg_charters_by_acronym", kwargs=dict(group_type="wg"))
+        r = self.client.get(
-        r = self.client.get(url)
+            urlreverse("ietf.group.views.wg_summary_area", kwargs={"group_type": "wg"})
-        self.assertEqual(r.status_code, 404)
+        )
-
-        wg_path = Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt"
-        wg_path.write_text("This is a charters file with an é")
-        r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        self.assertEqual(r.charset, "UTF-8")
+        self.assertEqual(r.content.decode(), "yay")
-        self.assertEqual(r.content.decode("utf8"), "This is a charters file with an é")
+        self.assertEqual(mock.call_args, call(Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary.txt"))
 
-        # non-wg request = 404 even if the file exists
+    @patch("ietf.group.views.response_from_file")
-        url = urlreverse("ietf.group.views.wg_charters_by_acronym", kwargs=dict(group_type="rg"))
+    def test_wg_summary_acronym(self, mock):
-        r = self.client.get(url)
+        r = self.client.get(
+            urlreverse(
+                "ietf.group.views.wg_summary_acronym", kwargs={"group_type": "rg"}
+            )
+        )  # not wg
         self.assertEqual(r.status_code, 404)
+        self.assertFalse(mock.called)
+        mock.return_value = HttpResponse("yay")
+        r = self.client.get(
+            urlreverse(
+                "ietf.group.views.wg_summary_acronym", kwargs={"group_type": "wg"}
+            )
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "yay")
+        self.assertEqual(
+            mock.call_args, call(Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary-by-acronym.txt")
+        )
+
+    @patch("ietf.group.views.response_from_file")
+    def test_wg_charters(self, mock):
+        r = self.client.get(
+            urlreverse("ietf.group.views.wg_charters", kwargs={"group_type": "rg"})
+        )  # not wg
+        self.assertEqual(r.status_code, 404)
+        self.assertFalse(mock.called)
+        mock.return_value = HttpResponse("yay")
+        r = self.client.get(
+            urlreverse("ietf.group.views.wg_charters", kwargs={"group_type": "wg"})
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "yay")
+        self.assertEqual(mock.call_args, call(Path(settings.CHARTER_PATH) / "1wg-charters.txt"))
+
+    @patch("ietf.group.views.response_from_file")
+    def test_wg_charters_by_acronym(self, mock):
+        r = self.client.get(
+            urlreverse(
+                "ietf.group.views.wg_charters_by_acronym", kwargs={"group_type": "rg"}
+            )
+        )  # not wg
+        self.assertEqual(r.status_code, 404)
+        self.assertFalse(mock.called)
+        mock.return_value = HttpResponse("yay")
+        r = self.client.get(
+            urlreverse(
+                "ietf.group.views.wg_charters_by_acronym", kwargs={"group_type": "wg"}
+            )
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "yay")
+        self.assertEqual(
+            mock.call_args, call(Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt")
+        )
 
     def test_generate_wg_charters_files_task(self):
         group = CharterFactory(
@@ -254,6 +287,30 @@ class GroupPagesTests(TestCase):
         )
         self.assertEqual(not_a_dir.read_text(), "Not a dir")
 
+    def test_generate_wg_summary_files_task(self):
+        group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area')).group
+        RoleFactory(group=group,name_id='chair',person=PersonFactory())
+        RoleFactory(group=group,name_id='ad',person=PersonFactory())
+
+        chair = Email.objects.filter(role__group=group, role__name="chair")[0]
+
+        generate_wg_summary_files_task()
+
+        summary_by_area_contents = (
+            Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary.txt"
+        ).read_text(encoding="utf8")
+        self.assertIn(group.parent.name, summary_by_area_contents)
+        self.assertIn(group.acronym, summary_by_area_contents)
+        self.assertIn(group.name, summary_by_area_contents)
+        self.assertIn(chair.address, summary_by_area_contents)
+
+        summary_by_acronym_contents = (
+            Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary-by-acronym.txt"
+        ).read_text(encoding="utf8")
+        self.assertIn(group.acronym, summary_by_acronym_contents)
+        self.assertIn(group.name, summary_by_acronym_contents)
+        self.assertIn(chair.address, summary_by_acronym_contents)
+
     def test_chartering_groups(self):
         group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area'),states=[('charter','intrev')]).group
 

ietf/group/views.py

@@ -152,56 +152,39 @@ def check_group_email_aliases():
     return False
 
 
+def response_from_file(fpath: Path) -> HttpResponse:
+    """Helper to shovel a file back in an HttpResponse"""
+    try:
+        content = fpath.read_bytes()
+    except IOError:
+        raise Http404
+    return HttpResponse(content, content_type="text/plain; charset=utf-8")
+
+
 # --- View functions ---------------------------------------------------
 
 def wg_summary_area(request, group_type):
     if group_type != "wg":
         raise Http404
-    areas = Group.objects.filter(type="area", state="active").order_by("name")
+    return response_from_file(Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary.txt")
-    for area in areas:
-        area.groups = Group.objects.filter(parent=area, type="wg", state="active").order_by("acronym")
-        for group in area.groups:
-            group.chairs = sorted(roles(group, "chair"), key=extract_last_name)
 
-    areas = [a for a in areas if a.groups]
-
-    return render(request, 'group/1wg-summary.txt',
-                  { 'areas': areas },
-                  content_type='text/plain; charset=UTF-8')
 
 def wg_summary_acronym(request, group_type):
     if group_type != "wg":
         raise Http404
-    areas = Group.objects.filter(type="area", state="active").order_by("name")
+    return response_from_file(Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary-by-acronym.txt")
-    groups = Group.objects.filter(type="wg", state="active").order_by("acronym").select_related("parent")
-    for group in groups:
-        group.chairs = sorted(roles(group, "chair"), key=extract_last_name)
-    return render(request, 'group/1wg-summary-by-acronym.txt',
-                  { 'areas': areas,
-                    'groups': groups },
-                  content_type='text/plain; charset=UTF-8')
 
 
 def wg_charters(request, group_type):
     if group_type != "wg":
         raise Http404
-    fpath = Path(settings.CHARTER_PATH) / "1wg-charters.txt" 
+    return response_from_file(Path(settings.CHARTER_PATH) / "1wg-charters.txt") 
-    try:
-        content = fpath.read_bytes()
-    except IOError:
-        raise Http404
-    return HttpResponse(content, content_type="text/plain; charset=UTF-8")
 
 
 def wg_charters_by_acronym(request, group_type):
     if group_type != "wg":
         raise Http404
-    fpath = Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt" 
+    return response_from_file(Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt") 
-    try:
-        content = fpath.read_bytes()
-    except IOError:
-        raise Http404
-    return HttpResponse(content, content_type="text/plain; charset=UTF-8")
 
 
 def active_groups(request, group_type=None):

ietf/settings.py

@@ -671,6 +671,7 @@ INTERNET_DRAFT_PDF_PATH = '/a/www/ietf-datatracker/pdf/'
 RFC_PATH = '/a/www/ietf-ftp/rfc/'
 CHARTER_PATH = '/a/ietfdata/doc/charter/'
 CHARTER_COPY_PATH = '/a/www/ietf-ftp/ietf'  # copy 1wg-charters files here if set
+GROUP_SUMMARY_PATH = '/a/www/ietf-ftp/ietf'
 BOFREQ_PATH = '/a/ietfdata/doc/bofreq/'
 CONFLICT_REVIEW_PATH = '/a/ietfdata/doc/conflict-review'
 STATUS_CHANGE_PATH = '/a/ietfdata/doc/status-change'

ietf/utils/management/commands/periodic_tasks.py

@@ -231,6 +231,16 @@ class Command(BaseCommand):
             ),
         )
 
+        PeriodicTask.objects.get_or_create(
+            name="Generate WG summary files",
+            task="ietf.group.tasks.generate_wg_summary_files_task",
+            defaults=dict(
+                enabled=False,
+                crontab=self.crontabs["hourly"],
+                description="Update 1wg-summary.txt and 1wg-summary-by-acronym.txt",
+            ),
+        )
+
         PeriodicTask.objects.get_or_create(
             name="Generate I-D bibxml files",
             task="ietf.doc.tasks.generate_draft_bibxml_files_task",

k8s/beat.yaml

@@ -1,61 +1,63 @@
-apiVersion: apps/v1
+apiVersion: apps/v1
-kind: Deployment
+kind: Deployment
-metadata:
+metadata:
-  name: beat
+  name: beat
-spec:
+  labels:
-  replicas: 1
+    deleteBeforeUpgrade: yes
-  revisionHistoryLimit: 2
+spec:
-  selector:
+  replicas: 1
-    matchLabels:
+  revisionHistoryLimit: 2
-      app: beat
+  selector:
-  strategy:
+    matchLabels:
-    type: Recreate
+      app: beat
-  template:
+  strategy:
-    metadata:
+    type: Recreate
-      labels:
+  template:
-        app: beat
+    metadata:
-    spec:
+      labels:
-      securityContext:
+        app: beat
-        runAsNonRoot: true
+    spec:
-      containers:
+      securityContext:
-        - name: beat
+        runAsNonRoot: true
-          image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
+      containers:
-          imagePullPolicy: Always
+        - name: beat
-          ports:
+          image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
-            - containerPort: 8000
+          imagePullPolicy: Always
-              name: http
+          ports:
-              protocol: TCP
+            - containerPort: 8000
-          volumeMounts:
+              name: http
-            - name: dt-vol
+              protocol: TCP
-              mountPath: /a
+          volumeMounts:
-            - name: dt-tmp
+            - name: dt-vol
-              mountPath: /tmp
+              mountPath: /a
-            - name: dt-cfg
+            - name: dt-tmp
-              mountPath: /workspace/ietf/settings_local.py
+              mountPath: /tmp
-              subPath: settings_local.py
+            - name: dt-cfg
-          env:
+              mountPath: /workspace/ietf/settings_local.py
-            - name: "CONTAINER_ROLE"
+              subPath: settings_local.py
-              value: "beat"
+          env:
-          envFrom:
+            - name: "CONTAINER_ROLE"
-            - configMapRef:
+              value: "beat"
-                name: django-config
+          envFrom:
-          securityContext:
+            - configMapRef:
-            allowPrivilegeEscalation: false
+                name: django-config
-            capabilities:
+          securityContext:
-              drop:
+            allowPrivilegeEscalation: false
-              - ALL
+            capabilities:
-            readOnlyRootFilesystem: true
+              drop:
-            runAsUser: 1000
+              - ALL
-            runAsGroup: 1000
+            readOnlyRootFilesystem: true
-      volumes:
+            runAsUser: 1000
-        # To be overriden with the actual shared volume
+            runAsGroup: 1000
-        - name: dt-vol
+      volumes:
-        - name: dt-tmp
+        # To be overriden with the actual shared volume
-          emptyDir:
+        - name: dt-vol
-            sizeLimit: "2Gi"
+        - name: dt-tmp
-        - name: dt-cfg
+          emptyDir:
-          configMap:
+            sizeLimit: "2Gi"
-            name: files-cfgmap
+        - name: dt-cfg
-      dnsPolicy: ClusterFirst
+          configMap:
-      restartPolicy: Always
+            name: files-cfgmap
-      terminationGracePeriodSeconds: 30
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 600

k8s/celery.yaml

@@ -1,80 +1,91 @@
-apiVersion: apps/v1
+apiVersion: apps/v1
-kind: Deployment
+kind: Deployment
-metadata:
+metadata:
-  name: celery
+  name: celery
-spec:
+  labels:
-  replicas: 1
+    deleteBeforeUpgrade: yes
-  revisionHistoryLimit: 2
+spec:
-  selector:
+  replicas: 1
-    matchLabels:
+  revisionHistoryLimit: 2
-      app: celery
+  selector:
-  strategy:
+    matchLabels:
-    type: Recreate
+      app: celery
-  template:
+  strategy:
-    metadata:
+    type: Recreate
-      labels:
+  template:
-        app: celery
+    metadata:
-    spec:
+      labels:
-      securityContext:
+        app: celery
-        runAsNonRoot: true
+    spec:
-      containers:
+      securityContext:
-        # -----------------------------------------------------
+        runAsNonRoot: true
-        # ScoutAPM Container
+      containers:
-        # -----------------------------------------------------
+        # -----------------------------------------------------
-        - name: scoutapm
+        # ScoutAPM Container
-          image: "scoutapp/scoutapm:version-1.4.0"
+        # -----------------------------------------------------
-          imagePullPolicy: IfNotPresent
+        - name: scoutapm
-          livenessProbe:
+          image: "scoutapp/scoutapm:version-1.4.0"
-            exec:
+          imagePullPolicy: IfNotPresent
-              command:
+          # Replace command with one that will shut down on a TERM signal
-                - "sh"
+          # The ./core-agent start command line is from the scoutapm docker image
-                - "-c"
+          command:
-                - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
+            - "sh"
-          securityContext:
+            - "-c"
-            readOnlyRootFilesystem: true
+            - >-
-            runAsUser: 65534 # "nobody" user by default
+              trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
-            runAsGroup: 65534  # "nogroup" group by default
+              ./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
-        # -----------------------------------------------------
+              wait $!
-        # Celery Container
+          livenessProbe:
-        # -----------------------------------------------------
+            exec:
-        - name: celery
+              command:
-          image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
+                - "sh"
-          imagePullPolicy: Always
+                - "-c"
-          ports:
+                - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
-            - containerPort: 8000
+          securityContext:
-              name: http
+            readOnlyRootFilesystem: true
-              protocol: TCP
+            runAsUser: 65534 # "nobody" user by default
-          volumeMounts:
+            runAsGroup: 65534  # "nogroup" group by default
-            - name: dt-vol
+        # -----------------------------------------------------
-              mountPath: /a
+        # Celery Container
-            - name: dt-tmp
+        # -----------------------------------------------------
-              mountPath: /tmp
+        - name: celery
-            - name: dt-cfg
+          image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
-              mountPath: /workspace/ietf/settings_local.py
+          imagePullPolicy: Always
-              subPath: settings_local.py
+          ports:
-          env:
+            - containerPort: 8000
-            - name: "CONTAINER_ROLE"
+              name: http
-              value: "celery"
+              protocol: TCP
-          envFrom:
+          volumeMounts:
-            - configMapRef:
+            - name: dt-vol
-                name: django-config
+              mountPath: /a
-          securityContext:
+            - name: dt-tmp
-            allowPrivilegeEscalation: false
+              mountPath: /tmp
-            capabilities:
+            - name: dt-cfg
-              drop:
+              mountPath: /workspace/ietf/settings_local.py
-              - ALL
+              subPath: settings_local.py
-            readOnlyRootFilesystem: true
+          env:
-            runAsUser: 1000
+            - name: "CONTAINER_ROLE"
-            runAsGroup: 1000
+              value: "celery"
-      volumes:
+          envFrom:
-        # To be overriden with the actual shared volume
+            - configMapRef:
-        - name: dt-vol
+                name: django-config
-        - name: dt-tmp
+          securityContext:
-          emptyDir:
+            allowPrivilegeEscalation: false
-            sizeLimit: "2Gi"
+            capabilities:
-        - name: dt-cfg
+              drop:
-          configMap:
+              - ALL
-            name: files-cfgmap
+            readOnlyRootFilesystem: true
-      dnsPolicy: ClusterFirst
+            runAsUser: 1000
-      restartPolicy: Always
+            runAsGroup: 1000
-      terminationGracePeriodSeconds: 30
+      volumes:
+        # To be overriden with the actual shared volume
+        - name: dt-vol
+        - name: dt-tmp
+          emptyDir:
+            sizeLimit: "2Gi"
+        - name: dt-cfg
+          configMap:
+            name: files-cfgmap
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 600

k8s/datatracker.yaml

@@ -24,6 +24,15 @@ spec:
         - name: scoutapm
           image: "scoutapp/scoutapm:version-1.4.0"
           imagePullPolicy: IfNotPresent
+          # Replace command with one that will shut down on a TERM signal
+          # The ./core-agent start command line is from the scoutapm docker image
+          command:
+            - "sh"
+            - "-c"
+            - >-
+              trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
+              ./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
+              wait $!
           livenessProbe:
             exec:
               command:
@@ -77,7 +86,7 @@ spec:
             name: files-cfgmap
       dnsPolicy: ClusterFirst
       restartPolicy: Always
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
 ---
 apiVersion: v1
 kind: Service

k8s/settings_local.py

@@ -262,3 +262,9 @@ CACHES = {
 _csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS")
 if _csrf_trusted_origins_str is not None:
     CSRF_TRUSTED_ORIGINS = _multiline_to_list(_csrf_trusted_origins_str)
+
+# Send logs to console instead of debug_console when running in kubernetes
+LOGGING["loggers"]["django"]["handlers"] = ["console", "mail_admins"]
+LOGGING["loggers"]["django.security"]["handlers"] = ["console"]
+LOGGING["loggers"]["datatracker"]["handlers"] = ["console"]
+LOGGING["loggers"]["celery"]["handlers"] = ["console"]