diff --git a/bin/hourly b/bin/hourly index 9478bec11..81638fa54 100755 --- a/bin/hourly +++ b/bin/hourly @@ -24,9 +24,6 @@ ID=/a/ietfdata/doc/draft/repository DERIVED=/a/ietfdata/derived DOWNLOAD=/a/www/www6s/download -$DTDIR/ietf/manage.py generate_idnits2_rfc_status -$DTDIR/ietf/manage.py generate_idnits2_rfcs_obsoleted - CHARTER=/a/www/ietf-ftp/charter wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt diff --git a/ietf/doc/management/commands/generate_idnits2_rfc_status.py b/ietf/doc/management/commands/generate_idnits2_rfc_status.py deleted file mode 100644 index 45be18801..000000000 --- a/ietf/doc/management/commands/generate_idnits2_rfc_status.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright The IETF Trust 2021 All Rights Reserved - -import os - -from django.conf import settings -from django.core.management.base import BaseCommand - -from ietf.doc.utils import generate_idnits2_rfc_status -from ietf.utils.log import log - -class Command(BaseCommand): - help = ('Generate the rfc_status blob used by idnits2') - - def handle(self, *args, **options): - filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status') - blob = generate_idnits2_rfc_status() - try: - bytes = blob.encode('utf-8') - with open(filename,'wb') as f: - f.write(bytes) - except Exception as e: - log('failed to write idnits2-rfc-status: '+str(e)) - raise e diff --git a/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py b/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py deleted file mode 100644 index 8bd122e87..000000000 --- a/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright The IETF Trust 2021 All Rights Reserved - -import os - -from django.conf import settings -from django.core.management.base import BaseCommand - -from ietf.doc.utils import generate_idnits2_rfcs_obsoleted -from ietf.utils.log import log - -class Command(BaseCommand): - help = ('Generate the rfcs-obsoleted file used by idnits2') - - def handle(self, *args, **options): - filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted') - blob = generate_idnits2_rfcs_obsoleted() - try: - bytes = blob.encode('utf-8') - with open(filename,'wb') as f: - f.write(bytes) - except Exception as e: - log('failed to write idnits2-rfcs-obsoleted: '+str(e)) - raise e diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index a2e83e9e2..b189a6827 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -6,6 +6,9 @@ import datetime import debug # pyflakes:ignore from celery import shared_task +from pathlib import Path + +from django.conf import settings from ietf.utils import log from ietf.utils.timezone import datetime_today @@ -21,6 +24,7 @@ from .expire import ( send_expire_warning_for_draft, ) from .models import Document +from .utils import generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted @shared_task @@ -54,3 +58,23 @@ def expire_ids_task(): def notify_expirations_task(notify_days=14): for doc in get_soon_to_expire_drafts(notify_days): send_expire_warning_for_draft(doc) + + +@shared_task +def generate_idnits2_rfc_status_task(): + outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status" + blob = generate_idnits2_rfc_status() + try: + outpath.write_text(blob, encoding="utf8") + except Exception as e: + log.log(f"failed to write idnits2-rfc-status: {e}") + + +@shared_task +def generate_idnits2_rfcs_obsoleted_task(): + outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted" + blob = generate_idnits2_rfcs_obsoleted() + try: + outpath.write_text(blob, encoding="utf8") + except Exception as e: + log.log(f"failed to write idnits2-rfcs-obsoleted: {e}") diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 4f5492e6c..e6a50937a 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -20,7 +20,6 @@ from tempfile import NamedTemporaryFile from collections import defaultdict from zoneinfo import ZoneInfo -from django.core.management import call_command from django.urls import reverse as urlreverse from django.conf import settings from django.forms import Form @@ -45,7 +44,14 @@ from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactor StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory) from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField -from ietf.doc.utils import create_ballot_if_not_open, investigate_fragment, uppercase_std_abbreviated_name, DraftAliasGenerator +from ietf.doc.utils import ( + create_ballot_if_not_open, + investigate_fragment, + uppercase_std_abbreviated_name, + DraftAliasGenerator, + generate_idnits2_rfc_status, + generate_idnits2_rfcs_obsoleted, +) from ietf.group.models import Group, Role from ietf.group.factories import GroupFactory, RoleFactory from ietf.ipr.factories import HolderIprDisclosureFactory @@ -2831,32 +2837,40 @@ class MaterialsTests(TestCase): class Idnits2SupportTests(TestCase): settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] - def test_obsoleted(self): + def test_generate_idnits2_rfcs_obsoleted(self): rfc = WgRfcFactory(rfc_number=1001) WgRfcFactory(rfc_number=1003,relations=[('obs',rfc)]) rfc = WgRfcFactory(rfc_number=1005) WgRfcFactory(rfc_number=1007,relations=[('obs',rfc)]) + blob = generate_idnits2_rfcs_obsoleted() + self.assertEqual(blob, b'1001 1003\n1005 1007\n'.decode("utf8")) + def test_obsoleted(self): url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted') r = self.client.get(url) self.assertEqual(r.status_code, 404) - call_command('generate_idnits2_rfcs_obsoleted') + # value written is arbitrary, expect it to be passed through + (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").write_bytes(b'1001 1003\n1005 1007\n') url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted') r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r.content, b'1001 1003\n1005 1007\n') - def test_rfc_status(self): + def test_generate_idnits2_rfc_status(self): for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'): WgRfcFactory(std_level_id=slug) + blob = generate_idnits2_rfc_status().replace("\n", "") + self.assertEqual(blob[6312-1], "O") + + def test_rfc_status(self): url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status') r = self.client.get(url) self.assertEqual(r.status_code,404) - call_command('generate_idnits2_rfc_status') + # value written is arbitrary, expect it to be passed through + (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").write_bytes(b'1001 1003\n1005 1007\n') r = self.client.get(url) self.assertEqual(r.status_code,200) - blob = unicontent(r).replace('\n','') - self.assertEqual(blob[6312-1],'O') + self.assertEqual(r.content, b'1001 1003\n1005 1007\n') def test_idnits2_state(self): rfc = WgRfcFactory() diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 931ed438d..8915a6c5a 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,15 +1,24 @@ # Copyright The IETF Trust 2024, All Rights Reserved import mock +from pathlib import Path + +from django.conf import settings + from ietf.utils.test_utils import TestCase from ietf.utils.timezone import datetime_today from .factories import DocumentFactory from .models import Document -from .tasks import expire_ids_task, notify_expirations_task - +from .tasks import ( + expire_ids_task, + generate_idnits2_rfcs_obsoleted_task, + generate_idnits2_rfc_status_task, + notify_expirations_task, +) class TaskTests(TestCase): + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ["DERIVED_DIR"] @mock.patch("ietf.doc.tasks.in_draft_expire_freeze") @mock.patch("ietf.doc.tasks.get_expired_drafts") @@ -35,10 +44,10 @@ class TaskTests(TestCase): Document.objects.filter(pk=doc.pk), Document.objects.filter(pk=other_doc.pk), ] - + # call task expire_ids_task() - + # check results self.assertTrue(in_draft_expire_freeze_mock.called) self.assertEqual(expirable_drafts_mock.call_count, 2) @@ -50,7 +59,7 @@ class TaskTests(TestCase): # test that an exception is raised in_draft_expire_freeze_mock.side_effect = RuntimeError - with self.assertRaises(RuntimeError):( + with self.assertRaises(RuntimeError): ( expire_ids_task()) @mock.patch("ietf.doc.tasks.send_expire_warning_for_draft") @@ -61,3 +70,24 @@ class TaskTests(TestCase): notify_expirations_task() self.assertEqual(send_warning_mock.call_count, 1) self.assertEqual(send_warning_mock.call_args[0], ("sentinel",)) + + @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status") + def test_generate_idnits2_rfc_status_task(self, mock_generate): + mock_generate.return_value = "dåtå" + generate_idnits2_rfc_status_task() + self.assertEqual(mock_generate.call_count, 1) + self.assertEqual( + "dåtå".encode("utf8"), + (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(), + ) + + @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") + def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): + mock_generate.return_value = "dåtå" + generate_idnits2_rfcs_obsoleted_task() + self.assertEqual(mock_generate.call_count, 1) + self.assertEqual( + "dåtå".encode("utf8"), + (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(), + ) + diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index e35938283..429cb4e14 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -181,6 +181,26 @@ class Command(BaseCommand): ) ) + PeriodicTask.objects.get_or_create( + name="Generate idnits2 rfcs-obsoleted blob", + task="ietf.doc.tasks.generate_idnits2_rfcs_obsoleted_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["hourly"], + description="Generate the rfcs-obsoleted file used by idnits", + ), + ) + + PeriodicTask.objects.get_or_create( + name="Generate idnits2 rfc-status blob", + task="ietf.doc.tasks.generate_idnits2_rfc_status_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["hourly"], + description="Generate the rfc_status blob used by idnits", + ), + ) + def show_tasks(self): for label, crontab in self.crontabs.items(): tasks = PeriodicTask.objects.filter(crontab=crontab).order_by(