feat: re-run yang checks via celery (#7558)

* refactor: yang checks -> task

* chore: add periodic task

* chore: remove run_yang_model_checks.py

* test: add tests

* refactor: populate_yang_model_dirs -> task

* chore: remove populate_yang_model_dirs.py

* chore: remove python setup from bin/daily

bin/daily

@@ -5,7 +5,6 @@
 # This script is expected to be triggered by cron from
 # /etc/cron.d/datatracker
 export LANG=en_US.UTF-8
-export PYTHONIOENCODING=utf-8
 
 # Make sure we stop if something goes wrong:
 program=${0##*/}
@@ -17,10 +16,6 @@ cd $DTDIR/
 
 logger -p user.info -t cron "Running $DTDIR/bin/daily"
 
-# Set up the virtual environment
-source $DTDIR/env/bin/activate
-
-
 # Get IANA-registered yang models
 #YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
 # Hardcode the rsync target to avoid any unwanted deletes:
@@ -30,9 +25,3 @@ rsync -avzq --delete /a/www/ietf-ftp/iana/yang-parameters/ /a/www/ietf-ftp/yang/
 # Get Yang models from Yangcatalog.
 #rsync -avzq rsync://rsync.yangcatalog.org:10873/yangdeps /a/www/ietf-ftp/yang/catalogmod/
 /a/www/ietf-datatracker/scripts/sync_to_yangcatalog
-
-# Populate the yang repositories
-$DTDIR/ietf/manage.py populate_yang_model_dirs -v0
-
-# Re-run yang checks on active documents
-$DTDIR/ietf/manage.py run_yang_model_checks -v0

ietf/submit/tasks.py

@@ -10,7 +10,8 @@ from django.utils import timezone
 
 from ietf.submit.models import Submission
 from ietf.submit.utils import (cancel_submission, create_submission_event, process_uploaded_submission,
-                               process_and_accept_uploaded_submission)
+                               process_and_accept_uploaded_submission, run_all_yang_model_checks,
+                               populate_yang_model_dirs)
 from ietf.utils import log
 
 
@@ -66,6 +67,12 @@ def cancel_stale_submissions():
         create_submission_event(None, subm, 'Submission canceled: expired without being posted')
 
 
+@shared_task
+def run_yang_model_checks_task():
+    populate_yang_model_dirs()
+    run_all_yang_model_checks()
+
+    
 @shared_task(bind=True)
 def poke(self):
     log.log(f'Poked {self.name}, request id {self.request.id}')

ietf/submit/tests.py

@@ -49,6 +49,7 @@ from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactor
 from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm
 from ietf.submit.models import Submission, Preapproval, SubmissionExtResource
 from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task
+from ietf.submit.utils import apply_yang_checker_to_draft, run_all_yang_model_checks
 from ietf.utils import tool_version
 from ietf.utils.accesstoken import generate_access_token
 from ietf.utils.mail import outbox, get_payload_text
@@ -3487,3 +3488,28 @@ class SubmissionStatusTests(BaseSubmitTestCase):
                 "Your Internet-Draft failed at least one submission check.",
                 status_code=200,
             )
+
+
+class YangCheckerTests(TestCase):
+    @mock.patch("ietf.submit.utils.apply_yang_checker_to_draft")
+    def test_run_all_yang_model_checks(self, mock_apply):
+        active_drafts = WgDraftFactory.create_batch(3)
+        WgDraftFactory(states=[("draft", "expired")])
+        run_all_yang_model_checks()
+        self.assertEqual(mock_apply.call_count, 3)
+        self.assertCountEqual(
+            [args[0][1] for args in mock_apply.call_args_list],
+            active_drafts,
+        )
+
+    def test_apply_yang_checker_to_draft(self):
+        draft = WgDraftFactory()
+        submission = SubmissionFactory(name=draft.name, rev=draft.rev)
+        submission.checks.create(checker="my-checker")
+        checker = mock.Mock()
+        checker.name = "my-checker"
+        checker.symbol = "X"
+        checker.check_file_txt.return_value = (True, "whee", None, None, {})
+        apply_yang_checker_to_draft(checker, draft)
+        self.assertEqual(checker.check_file_txt.call_args, mock.call(draft.get_file_name()))
+

ietf/submit/utils.py

@@ -4,9 +4,11 @@
 
 import datetime
 import io
+import json
 import os
 import pathlib
 import re
+import sys
 import time
 import traceback
 import xml2rfc
@@ -15,6 +17,7 @@ from pathlib import Path
 from shutil import move
 from typing import Optional, Union  # pyflakes:ignore
 from unidecode import unidecode
+from xym import xym
 
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -43,6 +46,7 @@ from ietf.person.models import Person, Email
 from ietf.community.utils import update_name_contains_indexes_with_new_doc
 from ietf.submit.mail import ( announce_to_lists, announce_new_version, announce_to_authors,
     send_approval_request, send_submission_confirmation, announce_new_wg_00, send_manual_post_request )
+from ietf.submit.checkers import DraftYangChecker
 from ietf.submit.models import ( Submission, SubmissionEvent, Preapproval, DraftSubmissionStateName,
     SubmissionCheck, SubmissionExtResource )
 from ietf.utils import log
@@ -1431,3 +1435,133 @@ def process_uploaded_submission(submission):
         submission.state_id = "uploaded"
         submission.save()
         create_submission_event(None, submission, desc="Completed submission validation checks")
+
+
+def apply_yang_checker_to_draft(checker, draft):
+    submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first()
+    if submission:
+        check = submission.checks.filter(checker=checker.name).order_by('-id').first()
+        if check:
+            result = checker.check_file_txt(draft.get_file_name())
+            passed, message, errors, warnings, items = result
+            items = json.loads(json.dumps(items))
+            new_res = (passed, errors, warnings, message)
+            old_res = (check.passed, check.errors, check.warnings, check.message) if check else ()
+            if new_res != old_res:
+                log.log(f"Saving new yang checker results for {draft.name}-{draft.rev}")
+                qs = submission.checks.filter(checker=checker.name).order_by('time')
+                submission.checks.filter(checker=checker.name).exclude(pk=qs.first().pk).delete()
+                submission.checks.create(submission=submission, checker=checker.name, passed=passed,
+                                         message=message, errors=errors, warnings=warnings, items=items,
+                                         symbol=checker.symbol)
+    else:
+        log.log(f"Could not run yang checker for {draft.name}-{draft.rev}: missing submission object")
+
+
+def run_all_yang_model_checks():
+    checker = DraftYangChecker()
+    for draft in Document.objects.filter(
+        type_id="draft",
+        states=State.objects.get(type="draft", slug="active"),
+    ):
+        apply_yang_checker_to_draft(checker, draft)
+
+
+def populate_yang_model_dirs():
+    """Update the yang model dirs
+
+     * All yang modules from published RFCs should be extracted and be
+       available in an rfc-yang repository.
+
+     * All valid yang modules from active, not replaced, Internet-Drafts
+       should be extracted and be available in a draft-valid-yang repository.
+
+     * All, valid and invalid, yang modules from active, not replaced,
+       Internet-Drafts should be available in a draft-all-yang repository.
+       (Actually, given precedence ordering, it would be enough to place
+       non-validating modules in a draft-invalid-yang repository instead).
+
+     * In all cases, example modules should be excluded.
+
+     * Precedence is established by the search order of the repository as
+       provided to pyang.
+
+     * As drafts expire, models should be removed in order to catch cases
+       where a module being worked on depends on one which has slipped out
+       of the work queue.
+
+    """
+    def extract_from(file, dir, strict=True):
+        saved_stdout = sys.stdout
+        saved_stderr = sys.stderr
+        xymerr = io.StringIO()
+        xymout = io.StringIO()
+        sys.stderr = xymerr
+        sys.stdout = xymout
+        model_list = []
+        try:
+            model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=-2)
+            for name in model_list:
+                modfile = moddir / name
+                mtime = file.stat().st_mtime
+                os.utime(str(modfile), (mtime, mtime))
+                if '"' in name:
+                    name = name.replace('"', '')
+                    modfile.rename(str(moddir / name))
+            model_list = [n.replace('"', '') for n in model_list]
+        except Exception as e:
+            log.log("Error when extracting from %s: %s" % (file, str(e)))
+        finally:
+            sys.stdout = saved_stdout
+            sys.stderr = saved_stderr
+        return model_list
+
+    # Extract from new RFCs
+
+    rfcdir = Path(settings.RFC_PATH)
+
+    moddir = Path(settings.SUBMIT_YANG_RFC_MODEL_DIR)
+    if not moddir.exists():
+        moddir.mkdir(parents=True)
+
+    latest = 0
+    for item in moddir.iterdir():
+        if item.stat().st_mtime > latest:
+            latest = item.stat().st_mtime
+
+    log.log(f"Extracting RFC Yang models to {moddir} ...")
+    for item in rfcdir.iterdir():
+        if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit():
+            if item.stat().st_mtime > latest:
+                model_list = extract_from(item, moddir)
+                for name in model_list:
+                    if not (name.startswith('ietf') or name.startswith('iana')):
+                        modfile = moddir / name
+                        modfile.unlink()
+
+    # Extract valid modules from drafts
+
+    six_months_ago = time.time() - 6 * 31 * 24 * 60 * 60
+
+    def active(dirent):
+        return dirent.stat().st_mtime > six_months_ago
+
+    draftdir = Path(settings.INTERNET_DRAFT_PATH)
+    moddir = Path(settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
+    if not moddir.exists():
+        moddir.mkdir(parents=True)
+    log.log(f"Emptying {moddir} ...")
+    for item in moddir.iterdir():
+        item.unlink()
+
+    log.log(f"Extracting draft Yang models to {moddir} ...")
+    for item in draftdir.iterdir():
+        try:
+            if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
+                model_list = extract_from(item, moddir, strict=False)
+                for name in model_list:
+                    if name.startswith('example'):
+                        modfile = moddir / name
+                        modfile.unlink()
+        except UnicodeDecodeError as e:
+            log.log(f"Error processing {item.name}: {e}")

ietf/utils/management/commands/periodic_tasks.py

@@ -273,6 +273,16 @@ class Command(BaseCommand):
             ),
         )
 
+        PeriodicTask.objects.get_or_create(
+            name="Run Yang model checks",
+            task="ietf.submit.tasks.run_yang_model_checks_task",
+            defaults=dict(
+                enabled=False,
+                crontab=self.crontabs["daily"],
+                description="Re-run Yang model checks on all active drafts",
+            ),
+        )
+
     def show_tasks(self):
         for label, crontab in self.crontabs.items():
             tasks = PeriodicTask.objects.filter(crontab=crontab).order_by(

ietf/utils/management/commands/populate_yang_model_dirs.py

@@ -1,172 +0,0 @@
-# Copyright The IETF Trust 2016-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
-
-
-import io
-import os
-import sys
-import time
-
-from pathlib import Path
-from textwrap import dedent
-from xym import xym
-
-from django.conf import settings
-from django.core.management.base import BaseCommand
-
-import debug                            # pyflakes:ignore
-
-class Command(BaseCommand):
-    """
-    Populate the yang module repositories from drafts and RFCs.
-
-    Extracts yang models from RFCs (found in settings.RFC_PATH and places
-    them in settings.SUBMIT_YANG_RFC_MODEL_DIR, and from active drafts, placed in
-    settings.SUBMIT_YANG_DRAFT_MODEL_DIR.
-
-    """
-
-    help = dedent(__doc__).strip()
-            
-    def add_arguments(self, parser):
-        parser.add_argument('--clean',
-            action='store_true', dest='clean', default=False,
-            help='Remove the current directory content before writing new models.')
-
-
-    def handle(self, *filenames, **options):
-        """
-
-         * All yang modules from published RFCs should be extracted and be
-           available in an rfc-yang repository.
-
-         * All valid yang modules from active, not replaced, Internet-Drafts
-           should be extracted and be available in a draft-valid-yang repository.
-
-         * All, valid and invalid, yang modules from active, not replaced,
-           Internet-Drafts should be available in a draft-all-yang repository.
-           (Actually, given precedence ordering, it would be enough to place
-           non-validating modules in a draft-invalid-yang repository instead).
-
-         * In all cases, example modules should be excluded.
-
-         * Precedence is established by the search order of the repository as
-           provided to pyang.
-
-         * As drafts expire, models should be removed in order to catch cases
-           where a module being worked on depends on one which has slipped out
-           of the work queue.
-
-        """
-
-        verbosity = int(options.get('verbosity'))
-
-        def extract_from(file, dir, strict=True):
-            saved_stdout = sys.stdout
-            saved_stderr = sys.stderr
-            xymerr = io.StringIO()
-            xymout = io.StringIO()            
-            sys.stderr = xymerr
-            sys.stdout = xymout
-            model_list = []
-            try:
-                model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=verbosity-2)
-                for name in model_list:
-                    modfile = moddir / name
-                    mtime = file.stat().st_mtime
-                    os.utime(str(modfile), (mtime, mtime))
-                    if '"' in name:
-                        name = name.replace('"', '')
-                        modfile.rename(str(moddir/name))
-                model_list = [ n.replace('"','') for n in model_list ]
-            except Exception as e:
-                self.stderr.write("** Error when extracting from %s: %s" % (file, str(e)))
-            finally:
-                sys.stdout = saved_stdout
-                sys.stderr = saved_stderr
-            #
-            if verbosity > 1:
-                outmsg = xymout.getvalue()
-                if outmsg.strip():
-                    self.stdout.write(outmsg)
-            if verbosity>2:            
-                errmsg = xymerr.getvalue()
-                if errmsg.strip():
-                    self.stderr.write(errmsg)
-            return model_list
-
-        # Extract from new RFCs
-
-        rfcdir = Path(settings.RFC_PATH)
-
-        moddir = Path(settings.SUBMIT_YANG_RFC_MODEL_DIR)
-        if not moddir.exists():
-            moddir.mkdir(parents=True)
-
-        latest = 0
-        for item in moddir.iterdir():
-            if item.stat().st_mtime > latest:
-                latest = item.stat().st_mtime
-
-        if verbosity > 0:
-            self.stdout.write("Extracting to %s ..." % moddir)
-        for item in rfcdir.iterdir():
-            if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and  item.name[3:-4].isdigit():
-                if item.stat().st_mtime > latest:
-                    model_list = extract_from(item, moddir)
-                    for name in model_list:
-                        if name.startswith('ietf') or name.startswith('iana'):
-                            if verbosity > 1:
-                                self.stdout.write("  Extracted from %s: %s" % (item, name))
-                            elif verbosity > 0:
-                                self.stdout.write('.', ending='')
-                                self.stdout.flush()
-                        else:
-                            modfile = moddir / name
-                            modfile.unlink()
-                            if verbosity > 1:
-                                self.stdout.write("  Skipped module from %s: %s" % (item, name))
-        if verbosity > 0:
-            self.stdout.write("")
-
-        # Extract valid modules from drafts
-
-        six_months_ago = time.time() - 6*31*24*60*60
-        def active(item):
-            return item.stat().st_mtime > six_months_ago
-
-        draftdir = Path(settings.INTERNET_DRAFT_PATH)
-
-        moddir = Path(settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
-        if not moddir.exists():
-            moddir.mkdir(parents=True)
-        if verbosity > 0:
-            self.stdout.write("Emptying %s ..." % moddir)
-        for item in moddir.iterdir():
-            item.unlink()
-
-        if verbosity > 0:
-            self.stdout.write("Extracting to %s ..." % moddir)
-        for item in draftdir.iterdir():
-            try:
-                if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
-                    model_list = extract_from(item, moddir, strict=False)
-                    for name in model_list:
-                        if not name.startswith('example'):
-                            if verbosity > 1:
-                                self.stdout.write("  Extracted module from %s: %s" % (item, name))
-                            elif verbosity > 0:
-                                self.stdout.write('.', ending='')
-                                self.stdout.flush()
-                        else:
-                            modfile = moddir / name
-                            modfile.unlink()
-                            if verbosity > 1:
-                                self.stdout.write("  Skipped module from %s: %s" % (item, name))
-            except UnicodeDecodeError as e:
-                self.stderr.write('\nError: %s' % (e, ))
-                self.stderr.write(item.name)
-                self.stderr.write('')
-        if verbosity > 0:
-            self.stdout.write('')
-

ietf/utils/management/commands/run_yang_model_checks.py

@@ -1,85 +0,0 @@
-# Copyright The IETF Trust 2017-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
-
-
-import json
-
-from textwrap import dedent
-
-from django.core.management.base import BaseCommand
-
-import debug                            # pyflakes:ignore
-
-from ietf.doc.models import Document, State
-from ietf.submit.models import Submission
-from ietf.submit.checkers import DraftYangChecker
-
-
-class Command(BaseCommand):
-    """
-    Run yang model checks on active drafts.
-
-    Repeats the yang checks in ietf/submit/checkers.py for active drafts, in
-    order to catch changes in status due to new modules becoming available in
-    the module directories.
-
-    """
-
-    help = dedent(__doc__).strip()
-            
-    def add_arguments(self, parser):
-        parser.add_argument('draftnames', nargs="*", help="drafts to check, or none to check all active yang drafts")
-        parser.add_argument('--clean',
-            action='store_true', dest='clean', default=False,
-            help='Remove the current directory content before writing new models.')
-
-
-    def check_yang(self, checker, draft, force=False):
-        if self.verbosity > 1:
-            self.stdout.write("Checking %s-%s" % (draft.name, draft.rev))
-        elif self.verbosity > 0:
-            self.stderr.write('.', ending='')
-        submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first()
-        if submission or force:
-            check = submission.checks.filter(checker=checker.name).order_by('-id').first()
-            if check or force:
-                result = checker.check_file_txt(draft.get_file_name())
-                passed, message, errors, warnings, items = result
-                if self.verbosity > 2:
-                    self.stdout.write("  Errors: %s\n"
-                                      "  Warnings: %s\n"
-                                      "  Message:\n%s\n" % (errors, warnings, message))
-                items = json.loads(json.dumps(items))
-                new_res = (passed, errors, warnings, message)
-                old_res = (check.passed, check.errors, check.warnings, check.message) if check else ()
-                if new_res != old_res:
-                    if self.verbosity > 1:
-                        self.stdout.write("  Saving new yang checker results for %s-%s" % (draft.name, draft.rev))
-                    qs = submission.checks.filter(checker=checker.name).order_by('time')
-                    submission.checks.filter(checker=checker.name).exclude(pk=qs.first().pk).delete()
-                    submission.checks.create(submission=submission, checker=checker.name, passed=passed,
-                                                message=message, errors=errors, warnings=warnings, items=items,
-                                                symbol=checker.symbol)
-        else:
-            self.stderr.write("Error: did not find any submission object for %s-%s" % (draft.name, draft.rev))
-
-    def handle(self, *filenames, **options):
-        """
-        """
-
-        self.verbosity = int(options.get('verbosity'))
-        drafts = options.get('draftnames')
-
-        active_state = State.objects.get(type="draft", slug="active")
-
-        checker = DraftYangChecker()
-        if drafts:
-            for name in drafts:
-                parts = name.rsplit('-',1)
-                if len(parts)==2 and len(parts[1])==2 and parts[1].isdigit():
-                    name = parts[0]
-                draft = Document.objects.get(name=name)
-                self.check_yang(checker, draft, force=True)
-        else:
-            for draft in Document.objects.filter(states=active_state, type_id='draft'):
-                self.check_yang(checker, draft)