altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

refactor: idnits2 mgmt cmds -> tasks (#7421)

Browse source 
* feat: tasks for generate_idnits2_rfc* mgmt cmds

* chore: create periodic tasks

* chore: remove mgmt cmds from bin/hourly

* test: test new tasks

* chore: remove now-unused scripts

* refactor: unitize Idnits2SupportTests
This commit is contained in:
Jennifer Richards 2024-05-14 20:46:12 -03:00 committed by GitHub
parent c0a12fa8b2
commit 235ac8b2a6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 101 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
bin/hourly
View file

@ -24,9 +24,6 @@ ID=/a/ietfdata/doc/draft/repository
DERIVED=/a/ietfdata/derived
DOWNLOAD=/a/www/www6s/download
$DTDIR/ietf/manage.py generate_idnits2_rfc_status
$DTDIR/ietf/manage.py generate_idnits2_rfcs_obsoleted
CHARTER=/a/www/ietf-ftp/charter
wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt
wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt

23
ietf/doc/management/commands/generate_idnits2_rfc_status.py
View file

@ -1,23 +0,0 @@
# Copyright The IETF Trust 2021 All Rights Reserved
import os
from django.conf import settings
from django.core.management.base import BaseCommand
from ietf.doc.utils import generate_idnits2_rfc_status
from ietf.utils.log import log
class Command(BaseCommand):
help = ('Generate the rfc_status blob used by idnits2')
def handle(self, *args, **options):
filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status')
blob = generate_idnits2_rfc_status()
try:
bytes = blob.encode('utf-8')
with open(filename,'wb') as f:
f.write(bytes)
except Exception as e:
log('failed to write idnits2-rfc-status: '+str(e))
raise e

23
ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py
View file

@ -1,23 +0,0 @@
# Copyright The IETF Trust 2021 All Rights Reserved
import os
from django.conf import settings
from django.core.management.base import BaseCommand
from ietf.doc.utils import generate_idnits2_rfcs_obsoleted
from ietf.utils.log import log
class Command(BaseCommand):
help = ('Generate the rfcs-obsoleted file used by idnits2')
def handle(self, *args, **options):
filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted')
blob = generate_idnits2_rfcs_obsoleted()
try:
bytes = blob.encode('utf-8')
with open(filename,'wb') as f:
f.write(bytes)
except Exception as e:
log('failed to write idnits2-rfcs-obsoleted: '+str(e))
raise e

24
ietf/doc/tasks.py
View file

@ -6,6 +6,9 @@ import datetime
import debug # pyflakes:ignore
from celery import shared_task
from pathlib import Path
from django.conf import settings
from ietf.utils import log
from ietf.utils.timezone import datetime_today
@ -21,6 +24,7 @@ from .expire import (
send_expire_warning_for_draft,
)
from .models import Document
from .utils import generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted
@shared_task
@ -54,3 +58,23 @@ def expire_ids_task():
def notify_expirations_task(notify_days=14):
for doc in get_soon_to_expire_drafts(notify_days):
send_expire_warning_for_draft(doc)
@shared_task
def generate_idnits2_rfc_status_task():
outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status"
blob = generate_idnits2_rfc_status()
try:
outpath.write_text(blob, encoding="utf8")
except Exception as e:
log.log(f"failed to write idnits2-rfc-status: {e}")
@shared_task
def generate_idnits2_rfcs_obsoleted_task():
outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted"
blob = generate_idnits2_rfcs_obsoleted()
try:
outpath.write_text(blob, encoding="utf8")
except Exception as e:
log.log(f"failed to write idnits2-rfcs-obsoleted: {e}")

30
ietf/doc/tests.py
View file

@ -20,7 +20,6 @@ from tempfile import NamedTemporaryFile
from collections import defaultdict
from zoneinfo import ZoneInfo
from django.core.management import call_command
from django.urls import reverse as urlreverse
from django.conf import settings
from django.forms import Form
@ -45,7 +44,14 @@ from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactor
StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory)
from ietf.doc.forms import NotifyForm
from ietf.doc.fields import SearchableDocumentsField
from ietf.doc.utils import create_ballot_if_not_open, investigate_fragment, uppercase_std_abbreviated_name, DraftAliasGenerator
from ietf.doc.utils import (
create_ballot_if_not_open,
investigate_fragment,
uppercase_std_abbreviated_name,
DraftAliasGenerator,
generate_idnits2_rfc_status,
generate_idnits2_rfcs_obsoleted,
)
from ietf.group.models import Group, Role
from ietf.group.factories import GroupFactory, RoleFactory
from ietf.ipr.factories import HolderIprDisclosureFactory
@ -2831,32 +2837,40 @@ class MaterialsTests(TestCase):
class Idnits2SupportTests(TestCase):
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR']
def test_obsoleted(self):
def test_generate_idnits2_rfcs_obsoleted(self):
rfc = WgRfcFactory(rfc_number=1001)
WgRfcFactory(rfc_number=1003,relations=[('obs',rfc)])
rfc = WgRfcFactory(rfc_number=1005)
WgRfcFactory(rfc_number=1007,relations=[('obs',rfc)])
blob = generate_idnits2_rfcs_obsoleted()
self.assertEqual(blob, b'1001 1003\n1005 1007\n'.decode("utf8"))
def test_obsoleted(self):
url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted')
r = self.client.get(url)
self.assertEqual(r.status_code, 404)
call_command('generate_idnits2_rfcs_obsoleted')
# value written is arbitrary, expect it to be passed through
(Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").write_bytes(b'1001 1003\n1005 1007\n')
url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted')
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.content, b'1001 1003\n1005 1007\n')
def test_rfc_status(self):
def test_generate_idnits2_rfc_status(self):
for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'):
WgRfcFactory(std_level_id=slug)
blob = generate_idnits2_rfc_status().replace("\n", "")
self.assertEqual(blob[6312-1], "O")
def test_rfc_status(self):
url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status')
r = self.client.get(url)
self.assertEqual(r.status_code,404)
call_command('generate_idnits2_rfc_status')
# value written is arbitrary, expect it to be passed through
(Path(settings.DERIVED_DIR) / "idnits2-rfc-status").write_bytes(b'1001 1003\n1005 1007\n')
r = self.client.get(url)
self.assertEqual(r.status_code,200)
blob = unicontent(r).replace('\n','')
self.assertEqual(blob[6312-1],'O')
self.assertEqual(r.content, b'1001 1003\n1005 1007\n')
def test_idnits2_state(self):
rfc = WgRfcFactory()

40
ietf/doc/tests_tasks.py
View file

@ -1,15 +1,24 @@
# Copyright The IETF Trust 2024, All Rights Reserved
import mock
from pathlib import Path
from django.conf import settings
from ietf.utils.test_utils import TestCase
from ietf.utils.timezone import datetime_today
from .factories import DocumentFactory
from .models import Document
from .tasks import expire_ids_task, notify_expirations_task
from .tasks import (
expire_ids_task,
generate_idnits2_rfcs_obsoleted_task,
generate_idnits2_rfc_status_task,
notify_expirations_task,
)
class TaskTests(TestCase):
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ["DERIVED_DIR"]
@mock.patch("ietf.doc.tasks.in_draft_expire_freeze")
@mock.patch("ietf.doc.tasks.get_expired_drafts")
@ -35,10 +44,10 @@ class TaskTests(TestCase):
Document.objects.filter(pk=doc.pk),
Document.objects.filter(pk=other_doc.pk),
]
# call task
expire_ids_task()
# check results
self.assertTrue(in_draft_expire_freeze_mock.called)
self.assertEqual(expirable_drafts_mock.call_count, 2)
@ -50,7 +59,7 @@ class TaskTests(TestCase):
# test that an exception is raised
in_draft_expire_freeze_mock.side_effect = RuntimeError
with self.assertRaises(RuntimeError):(
with self.assertRaises(RuntimeError): (
expire_ids_task())
@mock.patch("ietf.doc.tasks.send_expire_warning_for_draft")
@ -61,3 +70,24 @@ class TaskTests(TestCase):
notify_expirations_task()
self.assertEqual(send_warning_mock.call_count, 1)
self.assertEqual(send_warning_mock.call_args[0], ("sentinel",))
@mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status")
def test_generate_idnits2_rfc_status_task(self, mock_generate):
mock_generate.return_value = "dåtå"
generate_idnits2_rfc_status_task()
self.assertEqual(mock_generate.call_count, 1)
self.assertEqual(
"dåtå".encode("utf8"),
(Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(),
)
@mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted")
def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate):
mock_generate.return_value = "dåtå"
generate_idnits2_rfcs_obsoleted_task()
self.assertEqual(mock_generate.call_count, 1)
self.assertEqual(
"dåtå".encode("utf8"),
(Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(),
)

20
ietf/utils/management/commands/periodic_tasks.py
View file

@ -181,6 +181,26 @@ class Command(BaseCommand):
)
)
PeriodicTask.objects.get_or_create(
name="Generate idnits2 rfcs-obsoleted blob",
task="ietf.doc.tasks.generate_idnits2_rfcs_obsoleted_task",
defaults=dict(
enabled=False,
crontab=self.crontabs["hourly"],
description="Generate the rfcs-obsoleted file used by idnits",
),
)
PeriodicTask.objects.get_or_create(
name="Generate idnits2 rfc-status blob",
task="ietf.doc.tasks.generate_idnits2_rfc_status_task",
defaults=dict(
enabled=False,
crontab=self.crontabs["hourly"],
description="Generate the rfc_status blob used by idnits",
),
)
def show_tasks(self):
for label, crontab in self.crontabs.items():
tasks = PeriodicTask.objects.filter(crontab=crontab).order_by(