refactor: idnits2 mgmt cmds -> tasks (#7421)

* feat: tasks for generate_idnits2_rfc* mgmt cmds

* chore: create periodic tasks

* chore: remove mgmt cmds from bin/hourly

* test: test new tasks

* chore: remove now-unused scripts

* refactor: unitize Idnits2SupportTests

bin/hourly

@@ -24,9 +24,6 @@ ID=/a/ietfdata/doc/draft/repository
 DERIVED=/a/ietfdata/derived
 DOWNLOAD=/a/www/www6s/download
 
-$DTDIR/ietf/manage.py generate_idnits2_rfc_status
-$DTDIR/ietf/manage.py generate_idnits2_rfcs_obsoleted
-
 CHARTER=/a/www/ietf-ftp/charter
 wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt
 wget -q https://datatracker.ietf.org/wg/1wg-charters.txt            -O $CHARTER/1wg-charters.txt

ietf/doc/management/commands/generate_idnits2_rfc_status.py

@@ -1,23 +0,0 @@
-# Copyright The IETF Trust 2021 All Rights Reserved
-
-import os
-
-from django.conf import settings
-from django.core.management.base import BaseCommand
-
-from ietf.doc.utils import generate_idnits2_rfc_status
-from ietf.utils.log import log
-
-class Command(BaseCommand):
-    help = ('Generate the rfc_status blob used by idnits2')
-
-    def handle(self, *args, **options):
-        filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status')
-        blob = generate_idnits2_rfc_status()
-        try:
-            bytes = blob.encode('utf-8')
-            with open(filename,'wb') as f:
-                f.write(bytes)
-        except Exception as e:
-            log('failed to write idnits2-rfc-status: '+str(e))
-            raise e

ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py

@@ -1,23 +0,0 @@
-# Copyright The IETF Trust 2021 All Rights Reserved
-
-import os
-
-from django.conf import settings
-from django.core.management.base import BaseCommand
-
-from ietf.doc.utils import generate_idnits2_rfcs_obsoleted
-from ietf.utils.log import log
-
-class Command(BaseCommand):
-    help = ('Generate the rfcs-obsoleted file used by idnits2')
-
-    def handle(self, *args, **options):
-        filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted')
-        blob = generate_idnits2_rfcs_obsoleted()
-        try:
-            bytes = blob.encode('utf-8')
-            with open(filename,'wb') as f:
-                f.write(bytes)
-        except Exception as e:
-            log('failed to write idnits2-rfcs-obsoleted: '+str(e))
-            raise e

ietf/doc/tasks.py

@@ -6,6 +6,9 @@ import datetime
 import debug  # pyflakes:ignore
 
 from celery import shared_task
+from pathlib import Path
+
+from django.conf import settings
 
 from ietf.utils import log
 from ietf.utils.timezone import datetime_today
@@ -21,6 +24,7 @@ from .expire import (
     send_expire_warning_for_draft,
 )
 from .models import Document
+from .utils import generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted
 
 
 @shared_task
@@ -54,3 +58,23 @@ def expire_ids_task():
 def notify_expirations_task(notify_days=14):
     for doc in get_soon_to_expire_drafts(notify_days):
         send_expire_warning_for_draft(doc)
+
+
+@shared_task
+def generate_idnits2_rfc_status_task():
+    outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status"
+    blob = generate_idnits2_rfc_status()
+    try:
+        outpath.write_text(blob, encoding="utf8")
+    except Exception as e:
+        log.log(f"failed to write idnits2-rfc-status: {e}")
+
+
+@shared_task
+def generate_idnits2_rfcs_obsoleted_task():
+    outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted"
+    blob = generate_idnits2_rfcs_obsoleted()
+    try:
+        outpath.write_text(blob, encoding="utf8")
+    except Exception as e:
+        log.log(f"failed to write idnits2-rfcs-obsoleted: {e}")

ietf/doc/tests.py

@@ -20,7 +20,6 @@ from tempfile import NamedTemporaryFile
 from collections import defaultdict
 from zoneinfo import ZoneInfo
 
-from django.core.management import call_command
 from django.urls import reverse as urlreverse
 from django.conf import settings
 from django.forms import Form
@@ -45,7 +44,14 @@ from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactor
     StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory)
 from ietf.doc.forms import NotifyForm
 from ietf.doc.fields import SearchableDocumentsField
-from ietf.doc.utils import create_ballot_if_not_open, investigate_fragment, uppercase_std_abbreviated_name, DraftAliasGenerator
+from ietf.doc.utils import (
+    create_ballot_if_not_open,
+    investigate_fragment,
+    uppercase_std_abbreviated_name,
+    DraftAliasGenerator,
+    generate_idnits2_rfc_status,
+    generate_idnits2_rfcs_obsoleted,
+)
 from ietf.group.models import Group, Role
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.ipr.factories import HolderIprDisclosureFactory
@@ -2831,32 +2837,40 @@ class MaterialsTests(TestCase):
 class Idnits2SupportTests(TestCase):
     settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR']
 
-    def test_obsoleted(self):
+    def test_generate_idnits2_rfcs_obsoleted(self):
         rfc = WgRfcFactory(rfc_number=1001)
         WgRfcFactory(rfc_number=1003,relations=[('obs',rfc)])
         rfc = WgRfcFactory(rfc_number=1005)
         WgRfcFactory(rfc_number=1007,relations=[('obs',rfc)])
+        blob = generate_idnits2_rfcs_obsoleted()
+        self.assertEqual(blob, b'1001 1003\n1005 1007\n'.decode("utf8"))
 
+    def test_obsoleted(self):
         url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted')
         r = self.client.get(url)
         self.assertEqual(r.status_code, 404)
-        call_command('generate_idnits2_rfcs_obsoleted')
+        # value written is arbitrary, expect it to be passed through
+        (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").write_bytes(b'1001 1003\n1005 1007\n')
         url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted')
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.content, b'1001 1003\n1005 1007\n')
 
-    def test_rfc_status(self):
+    def test_generate_idnits2_rfc_status(self):
         for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'):
             WgRfcFactory(std_level_id=slug)
+        blob = generate_idnits2_rfc_status().replace("\n", "")
+        self.assertEqual(blob[6312-1], "O")
+
+    def test_rfc_status(self):
         url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status')
         r = self.client.get(url)
         self.assertEqual(r.status_code,404)
-        call_command('generate_idnits2_rfc_status')
+        # value written is arbitrary, expect it to be passed through
+        (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").write_bytes(b'1001 1003\n1005 1007\n')
         r = self.client.get(url)
         self.assertEqual(r.status_code,200)
-        blob = unicontent(r).replace('\n','')
+        self.assertEqual(r.content, b'1001 1003\n1005 1007\n')
-        self.assertEqual(blob[6312-1],'O')
 
     def test_idnits2_state(self):
         rfc = WgRfcFactory()

ietf/doc/tests_tasks.py

@@ -1,15 +1,24 @@
 # Copyright The IETF Trust 2024, All Rights Reserved
 import mock
 
+from pathlib import Path
+
+from django.conf import settings
+
 from ietf.utils.test_utils import TestCase
 from ietf.utils.timezone import datetime_today
 
 from .factories import DocumentFactory
 from .models import Document
-from .tasks import expire_ids_task, notify_expirations_task
+from .tasks import (
-
+    expire_ids_task,
+    generate_idnits2_rfcs_obsoleted_task,
+    generate_idnits2_rfc_status_task,
+    notify_expirations_task,
+)
 
 class TaskTests(TestCase):
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ["DERIVED_DIR"]
 
     @mock.patch("ietf.doc.tasks.in_draft_expire_freeze")
     @mock.patch("ietf.doc.tasks.get_expired_drafts")
@@ -35,10 +44,10 @@ class TaskTests(TestCase):
             Document.objects.filter(pk=doc.pk),
             Document.objects.filter(pk=other_doc.pk),
         ]
-        
+
         # call task
         expire_ids_task()
-        
+
         # check results
         self.assertTrue(in_draft_expire_freeze_mock.called)
         self.assertEqual(expirable_drafts_mock.call_count, 2)
@@ -50,7 +59,7 @@ class TaskTests(TestCase):
 
         # test that an exception is raised
         in_draft_expire_freeze_mock.side_effect = RuntimeError
-        with self.assertRaises(RuntimeError):(
+        with self.assertRaises(RuntimeError): (
             expire_ids_task())
 
     @mock.patch("ietf.doc.tasks.send_expire_warning_for_draft")
@@ -61,3 +70,24 @@ class TaskTests(TestCase):
         notify_expirations_task()
         self.assertEqual(send_warning_mock.call_count, 1)
         self.assertEqual(send_warning_mock.call_args[0], ("sentinel",))
+
+    @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status")
+    def test_generate_idnits2_rfc_status_task(self, mock_generate):
+        mock_generate.return_value = "dåtå"
+        generate_idnits2_rfc_status_task()
+        self.assertEqual(mock_generate.call_count, 1)
+        self.assertEqual(
+            "dåtå".encode("utf8"),
+            (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(),
+        )
+    
+    @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted")
+    def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate):
+        mock_generate.return_value = "dåtå"
+        generate_idnits2_rfcs_obsoleted_task()
+        self.assertEqual(mock_generate.call_count, 1)
+        self.assertEqual(
+            "dåtå".encode("utf8"),
+            (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(),
+        )
+    

ietf/utils/management/commands/periodic_tasks.py

@@ -181,6 +181,26 @@ class Command(BaseCommand):
             )
         )
 
+        PeriodicTask.objects.get_or_create(
+            name="Generate idnits2 rfcs-obsoleted blob",
+            task="ietf.doc.tasks.generate_idnits2_rfcs_obsoleted_task",
+            defaults=dict(
+                enabled=False,
+                crontab=self.crontabs["hourly"],
+                description="Generate the rfcs-obsoleted file used by idnits",
+            ),
+        )
+
+        PeriodicTask.objects.get_or_create(
+            name="Generate idnits2 rfc-status blob",
+            task="ietf.doc.tasks.generate_idnits2_rfc_status_task",
+            defaults=dict(
+                enabled=False,
+                crontab=self.crontabs["hourly"],
+                description="Generate the rfc_status blob used by idnits",
+            ),
+        )
+
     def show_tasks(self):
         for label, crontab in self.crontabs.items():
             tasks = PeriodicTask.objects.filter(crontab=crontab).order_by(