altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

feat: re-run yang checks via celery (#7558)

Browse source 
* refactor: yang checks -> task

* chore: add periodic task

* chore: remove run_yang_model_checks.py

* test: add tests

* refactor: populate_yang_model_dirs -> task

* chore: remove populate_yang_model_dirs.py

* chore: remove python setup from bin/daily
This commit is contained in:
Jennifer Richards 2024-06-18 12:42:13 -03:00 committed by GitHub
parent 0ac2ae12dc
commit 92784f9c31
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 178 additions and 269 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
bin/daily
View file

@ -5,7 +5,6 @@
# This script is expected to be triggered by cron from
# /etc/cron.d/datatracker
export LANG=en_US.UTF-8
export PYTHONIOENCODING=utf-8
# Make sure we stop if something goes wrong:
program=${0##*/}
@ -17,10 +16,6 @@ cd $DTDIR/
logger -p user.info -t cron "Running $DTDIR/bin/daily"
# Set up the virtual environment
source $DTDIR/env/bin/activate
# Get IANA-registered yang models
#YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
# Hardcode the rsync target to avoid any unwanted deletes:
@ -30,9 +25,3 @@ rsync -avzq --delete /a/www/ietf-ftp/iana/yang-parameters/ /a/www/ietf-ftp/yang/
# Get Yang models from Yangcatalog.
#rsync -avzq rsync://rsync.yangcatalog.org:10873/yangdeps /a/www/ietf-ftp/yang/catalogmod/
/a/www/ietf-datatracker/scripts/sync_to_yangcatalog
# Populate the yang repositories
$DTDIR/ietf/manage.py populate_yang_model_dirs -v0
# Re-run yang checks on active documents
$DTDIR/ietf/manage.py run_yang_model_checks -v0

9
ietf/submit/tasks.py
View file

@ -10,7 +10,8 @@ from django.utils import timezone
from ietf.submit.models import Submission
from ietf.submit.utils import (cancel_submission, create_submission_event, process_uploaded_submission,
process_and_accept_uploaded_submission)
process_and_accept_uploaded_submission, run_all_yang_model_checks,
populate_yang_model_dirs)
from ietf.utils import log
@ -66,6 +67,12 @@ def cancel_stale_submissions():
create_submission_event(None, subm, 'Submission canceled: expired without being posted')
@shared_task
def run_yang_model_checks_task():
populate_yang_model_dirs()
run_all_yang_model_checks()
@shared_task(bind=True)
def poke(self):
log.log(f'Poked {self.name}, request id {self.request.id}')

26
ietf/submit/tests.py
View file

@ -49,6 +49,7 @@ from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactor
from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm
from ietf.submit.models import Submission, Preapproval, SubmissionExtResource
from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task
from ietf.submit.utils import apply_yang_checker_to_draft, run_all_yang_model_checks
from ietf.utils import tool_version
from ietf.utils.accesstoken import generate_access_token
from ietf.utils.mail import outbox, get_payload_text
@ -3487,3 +3488,28 @@ class SubmissionStatusTests(BaseSubmitTestCase):
"Your Internet-Draft failed at least one submission check.",
status_code=200,
)
class YangCheckerTests(TestCase):
@mock.patch("ietf.submit.utils.apply_yang_checker_to_draft")
def test_run_all_yang_model_checks(self, mock_apply):
active_drafts = WgDraftFactory.create_batch(3)
WgDraftFactory(states=[("draft", "expired")])
run_all_yang_model_checks()
self.assertEqual(mock_apply.call_count, 3)
self.assertCountEqual(
[args[0][1] for args in mock_apply.call_args_list],
active_drafts,
)
def test_apply_yang_checker_to_draft(self):
draft = WgDraftFactory()
submission = SubmissionFactory(name=draft.name, rev=draft.rev)
submission.checks.create(checker="my-checker")
checker = mock.Mock()
checker.name = "my-checker"
checker.symbol = "X"
checker.check_file_txt.return_value = (True, "whee", None, None, {})
apply_yang_checker_to_draft(checker, draft)
self.assertEqual(checker.check_file_txt.call_args, mock.call(draft.get_file_name()))

134
ietf/submit/utils.py
View file

@ -4,9 +4,11 @@
import datetime
import io
import json
import os
import pathlib
import re
import sys
import time
import traceback
import xml2rfc
@ -15,6 +17,7 @@ from pathlib import Path
from shutil import move
from typing import Optional, Union # pyflakes:ignore
from unidecode import unidecode
from xym import xym
from django.conf import settings
from django.core.exceptions import ValidationError
@ -43,6 +46,7 @@ from ietf.person.models import Person, Email
from ietf.community.utils import update_name_contains_indexes_with_new_doc
from ietf.submit.mail import ( announce_to_lists, announce_new_version, announce_to_authors,
send_approval_request, send_submission_confirmation, announce_new_wg_00, send_manual_post_request )
from ietf.submit.checkers import DraftYangChecker
from ietf.submit.models import ( Submission, SubmissionEvent, Preapproval, DraftSubmissionStateName,
SubmissionCheck, SubmissionExtResource )
from ietf.utils import log
@ -1431,3 +1435,133 @@ def process_uploaded_submission(submission):
submission.state_id = "uploaded"
submission.save()
create_submission_event(None, submission, desc="Completed submission validation checks")
def apply_yang_checker_to_draft(checker, draft):
submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first()
if submission:
check = submission.checks.filter(checker=checker.name).order_by('-id').first()
if check:
result = checker.check_file_txt(draft.get_file_name())
passed, message, errors, warnings, items = result
items = json.loads(json.dumps(items))
new_res = (passed, errors, warnings, message)
old_res = (check.passed, check.errors, check.warnings, check.message) if check else ()
if new_res != old_res:
log.log(f"Saving new yang checker results for {draft.name}-{draft.rev}")
qs = submission.checks.filter(checker=checker.name).order_by('time')
submission.checks.filter(checker=checker.name).exclude(pk=qs.first().pk).delete()
submission.checks.create(submission=submission, checker=checker.name, passed=passed,
message=message, errors=errors, warnings=warnings, items=items,
symbol=checker.symbol)
else:
log.log(f"Could not run yang checker for {draft.name}-{draft.rev}: missing submission object")
def run_all_yang_model_checks():
checker = DraftYangChecker()
for draft in Document.objects.filter(
type_id="draft",
states=State.objects.get(type="draft", slug="active"),
):
apply_yang_checker_to_draft(checker, draft)
def populate_yang_model_dirs():
"""Update the yang model dirs
* All yang modules from published RFCs should be extracted and be
available in an rfc-yang repository.
* All valid yang modules from active, not replaced, Internet-Drafts
should be extracted and be available in a draft-valid-yang repository.
* All, valid and invalid, yang modules from active, not replaced,
Internet-Drafts should be available in a draft-all-yang repository.
(Actually, given precedence ordering, it would be enough to place
non-validating modules in a draft-invalid-yang repository instead).
* In all cases, example modules should be excluded.
* Precedence is established by the search order of the repository as
provided to pyang.
* As drafts expire, models should be removed in order to catch cases
where a module being worked on depends on one which has slipped out
of the work queue.
"""
def extract_from(file, dir, strict=True):
saved_stdout = sys.stdout
saved_stderr = sys.stderr
xymerr = io.StringIO()
xymout = io.StringIO()
sys.stderr = xymerr
sys.stdout = xymout
model_list = []
try:
model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=-2)
for name in model_list:
modfile = moddir / name
mtime = file.stat().st_mtime
os.utime(str(modfile), (mtime, mtime))
if '"' in name:
name = name.replace('"', '')
modfile.rename(str(moddir / name))
model_list = [n.replace('"', '') for n in model_list]
except Exception as e:
log.log("Error when extracting from %s: %s" % (file, str(e)))
finally:
sys.stdout = saved_stdout
sys.stderr = saved_stderr
return model_list
# Extract from new RFCs
rfcdir = Path(settings.RFC_PATH)
moddir = Path(settings.SUBMIT_YANG_RFC_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
latest = 0
for item in moddir.iterdir():
if item.stat().st_mtime > latest:
latest = item.stat().st_mtime
log.log(f"Extracting RFC Yang models to {moddir} ...")
for item in rfcdir.iterdir():
if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit():
if item.stat().st_mtime > latest:
model_list = extract_from(item, moddir)
for name in model_list:
if not (name.startswith('ietf') or name.startswith('iana')):
modfile = moddir / name
modfile.unlink()
# Extract valid modules from drafts
six_months_ago = time.time() - 6 * 31 * 24 * 60 * 60
def active(dirent):
return dirent.stat().st_mtime > six_months_ago
draftdir = Path(settings.INTERNET_DRAFT_PATH)
moddir = Path(settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
log.log(f"Emptying {moddir} ...")
for item in moddir.iterdir():
item.unlink()
log.log(f"Extracting draft Yang models to {moddir} ...")
for item in draftdir.iterdir():
try:
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir, strict=False)
for name in model_list:
if name.startswith('example'):
modfile = moddir / name
modfile.unlink()
except UnicodeDecodeError as e:
log.log(f"Error processing {item.name}: {e}")

10
ietf/utils/management/commands/periodic_tasks.py
View file

@ -273,6 +273,16 @@ class Command(BaseCommand):
),
)
PeriodicTask.objects.get_or_create(
name="Run Yang model checks",
task="ietf.submit.tasks.run_yang_model_checks_task",
defaults=dict(
enabled=False,
crontab=self.crontabs["daily"],
description="Re-run Yang model checks on all active drafts",
),
)
def show_tasks(self):
for label, crontab in self.crontabs.items():
tasks = PeriodicTask.objects.filter(crontab=crontab).order_by(

172
ietf/utils/management/commands/populate_yang_model_dirs.py
View file

@ -1,172 +0,0 @@
# Copyright The IETF Trust 2016-2020, All Rights Reserved
# -*- coding: utf-8 -*-
import io
import os
import sys
import time
from pathlib import Path
from textwrap import dedent
from xym import xym
from django.conf import settings
from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
class Command(BaseCommand):
"""
Populate the yang module repositories from drafts and RFCs.
Extracts yang models from RFCs (found in settings.RFC_PATH and places
them in settings.SUBMIT_YANG_RFC_MODEL_DIR, and from active drafts, placed in
settings.SUBMIT_YANG_DRAFT_MODEL_DIR.
"""
help = dedent(__doc__).strip()
def add_arguments(self, parser):
parser.add_argument('--clean',
action='store_true', dest='clean', default=False,
help='Remove the current directory content before writing new models.')
def handle(self, *filenames, **options):
"""
* All yang modules from published RFCs should be extracted and be
available in an rfc-yang repository.
* All valid yang modules from active, not replaced, Internet-Drafts
should be extracted and be available in a draft-valid-yang repository.
* All, valid and invalid, yang modules from active, not replaced,
Internet-Drafts should be available in a draft-all-yang repository.
(Actually, given precedence ordering, it would be enough to place
non-validating modules in a draft-invalid-yang repository instead).
* In all cases, example modules should be excluded.
* Precedence is established by the search order of the repository as
provided to pyang.
* As drafts expire, models should be removed in order to catch cases
where a module being worked on depends on one which has slipped out
of the work queue.
"""
verbosity = int(options.get('verbosity'))
def extract_from(file, dir, strict=True):
saved_stdout = sys.stdout
saved_stderr = sys.stderr
xymerr = io.StringIO()
xymout = io.StringIO()
sys.stderr = xymerr
sys.stdout = xymout
model_list = []
try:
model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=verbosity-2)
for name in model_list:
modfile = moddir / name
mtime = file.stat().st_mtime
os.utime(str(modfile), (mtime, mtime))
if '"' in name:
name = name.replace('"', '')
modfile.rename(str(moddir/name))
model_list = [ n.replace('"','') for n in model_list ]
except Exception as e:
self.stderr.write("** Error when extracting from %s: %s" % (file, str(e)))
finally:
sys.stdout = saved_stdout
sys.stderr = saved_stderr
#
if verbosity > 1:
outmsg = xymout.getvalue()
if outmsg.strip():
self.stdout.write(outmsg)
if verbosity>2:
errmsg = xymerr.getvalue()
if errmsg.strip():
self.stderr.write(errmsg)
return model_list
# Extract from new RFCs
rfcdir = Path(settings.RFC_PATH)
moddir = Path(settings.SUBMIT_YANG_RFC_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
latest = 0
for item in moddir.iterdir():
if item.stat().st_mtime > latest:
latest = item.stat().st_mtime
if verbosity > 0:
self.stdout.write("Extracting to %s ..." % moddir)
for item in rfcdir.iterdir():
if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit():
if item.stat().st_mtime > latest:
model_list = extract_from(item, moddir)
for name in model_list:
if name.startswith('ietf') or name.startswith('iana'):
if verbosity > 1:
self.stdout.write(" Extracted from %s: %s" % (item, name))
elif verbosity > 0:
self.stdout.write('.', ending='')
self.stdout.flush()
else:
modfile = moddir / name
modfile.unlink()
if verbosity > 1:
self.stdout.write(" Skipped module from %s: %s" % (item, name))
if verbosity > 0:
self.stdout.write("")
# Extract valid modules from drafts
six_months_ago = time.time() - 6*31*24*60*60
def active(item):
return item.stat().st_mtime > six_months_ago
draftdir = Path(settings.INTERNET_DRAFT_PATH)
moddir = Path(settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
if verbosity > 0:
self.stdout.write("Emptying %s ..." % moddir)
for item in moddir.iterdir():
item.unlink()
if verbosity > 0:
self.stdout.write("Extracting to %s ..." % moddir)
for item in draftdir.iterdir():
try:
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir, strict=False)
for name in model_list:
if not name.startswith('example'):
if verbosity > 1:
self.stdout.write(" Extracted module from %s: %s" % (item, name))
elif verbosity > 0:
self.stdout.write('.', ending='')
self.stdout.flush()
else:
modfile = moddir / name
modfile.unlink()
if verbosity > 1:
self.stdout.write(" Skipped module from %s: %s" % (item, name))
except UnicodeDecodeError as e:
self.stderr.write('\nError: %s' % (e, ))
self.stderr.write(item.name)
self.stderr.write('')
if verbosity > 0:
self.stdout.write('')

85
ietf/utils/management/commands/run_yang_model_checks.py
View file

@ -1,85 +0,0 @@
# Copyright The IETF Trust 2017-2020, All Rights Reserved
# -*- coding: utf-8 -*-
import json
from textwrap import dedent
from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
from ietf.doc.models import Document, State
from ietf.submit.models import Submission
from ietf.submit.checkers import DraftYangChecker
class Command(BaseCommand):
"""
Run yang model checks on active drafts.
Repeats the yang checks in ietf/submit/checkers.py for active drafts, in
order to catch changes in status due to new modules becoming available in
the module directories.
"""
help = dedent(__doc__).strip()
def add_arguments(self, parser):
parser.add_argument('draftnames', nargs="*", help="drafts to check, or none to check all active yang drafts")
parser.add_argument('--clean',
action='store_true', dest='clean', default=False,
help='Remove the current directory content before writing new models.')
def check_yang(self, checker, draft, force=False):
if self.verbosity > 1:
self.stdout.write("Checking %s-%s" % (draft.name, draft.rev))
elif self.verbosity > 0:
self.stderr.write('.', ending='')
submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first()
if submission or force:
check = submission.checks.filter(checker=checker.name).order_by('-id').first()
if check or force:
result = checker.check_file_txt(draft.get_file_name())
passed, message, errors, warnings, items = result
if self.verbosity > 2:
self.stdout.write(" Errors: %s\n"
" Warnings: %s\n"
" Message:\n%s\n" % (errors, warnings, message))
items = json.loads(json.dumps(items))
new_res = (passed, errors, warnings, message)
old_res = (check.passed, check.errors, check.warnings, check.message) if check else ()
if new_res != old_res:
if self.verbosity > 1:
self.stdout.write(" Saving new yang checker results for %s-%s" % (draft.name, draft.rev))
qs = submission.checks.filter(checker=checker.name).order_by('time')
submission.checks.filter(checker=checker.name).exclude(pk=qs.first().pk).delete()
submission.checks.create(submission=submission, checker=checker.name, passed=passed,
message=message, errors=errors, warnings=warnings, items=items,
symbol=checker.symbol)
else:
self.stderr.write("Error: did not find any submission object for %s-%s" % (draft.name, draft.rev))
def handle(self, *filenames, **options):
"""
"""
self.verbosity = int(options.get('verbosity'))
drafts = options.get('draftnames')
active_state = State.objects.get(type="draft", slug="active")
checker = DraftYangChecker()
if drafts:
for name in drafts:
parts = name.rsplit('-',1)
if len(parts)==2 and len(parts[1])==2 and parts[1].isdigit():
name = parts[0]
draft = Document.objects.get(name=name)
self.check_yang(checker, draft, force=True)
else:
for draft in Document.objects.filter(states=active_state, type_id='draft'):
self.check_yang(checker, draft)