diff --git a/.gitignore b/.gitignore index 925b66eed..e44bf00fe 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ /static /testresult /unix.tag +/tmp-nomcom-public-keys-dir diff --git a/bin/daily b/bin/daily index f868e933d..3764e4a94 100755 --- a/bin/daily +++ b/bin/daily @@ -18,9 +18,16 @@ logger -p user.info -t cron "Running $DTDIR/bin/daily" # Set up the virtual environment source $DTDIR/env/bin/activate + +# Update our information about the current version of some commands we use +$DTDIR/ietf/manage.py update_external_command_info + # Populate the yang repositories $DTDIR/ietf/manage.py populate_yang_model_dirs +# Re-run yang checks on active documents +$DTDIR/ietf/manage.py run_yang_model_checks + # Expire internet drafts # Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner: $DTDIR/ietf/bin/expire-ids @@ -36,5 +43,3 @@ $DTDIR/ietf/bin/rfc-editor-index-updates -d 1969-01-01 # Fetch meeting attendance data from ietf.org/registration/attendees $DTDIR/ietf/manage.py fetch_meeting_attendance --latest - - diff --git a/bin/mkdevbranch b/bin/mkdevbranch index 16d038b9f..2ab89e9f8 100755 --- a/bin/mkdevbranch +++ b/bin/mkdevbranch @@ -142,7 +142,7 @@ cd $progdir if [ "$who" ]; then mksvndir $who svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$who/$target/ -m "New branch for $target" - say "New branch: ^/personal/$who/$target" + print "New branch: ^/personal/$who/$target" else [ "$num" ] || num=$( < /www/tools.ietf.org/meta/current-ietf-number.txt) for n in $(seq $((num-3)) $num); do diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index b71be5aa5..96162b164 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -533,3 +533,6 @@ def comma_separated_list(seq, end_word="and"): def role_names(roles): return list(set([ "%s %s" % (r.group.name, r.name.name) for r in roles ])) +@register.filter() +def zaptmp(s): + return re.sub(r'/tmp/tmp[^/]+/', '', s) diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 6a6cd71e2..87ffc206b 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -9708,5 +9708,38 @@ }, "model": "name.topicaudiencename", "pk": "nominees" + }, + { + "fields": { + "command": "xym", + "switch": "--version", + "time": "2017-06-15T06:24:58.869", + "used": true, + "version": "xym 0.3.2" + }, + "model": "utils.versioninfo", + "pk": 1 + }, + { + "fields": { + "command": "pyang", + "switch": "--version", + "time": "2017-06-15T06:24:59.516", + "used": true, + "version": "pyang 1.7.2" + }, + "model": "utils.versioninfo", + "pk": 2 + }, + { + "fields": { + "command": "yanglint", + "switch": "--version", + "time": "2017-06-15T06:24:59.531", + "used": true, + "version": "yanglint 0.12.183" + }, + "model": "utils.versioninfo", + "pk": 3 } ] diff --git a/ietf/name/generate_fixtures.py b/ietf/name/generate_fixtures.py index 753f6b8ad..f90abcbda 100644 --- a/ietf/name/generate_fixtures.py +++ b/ietf/name/generate_fixtures.py @@ -47,5 +47,8 @@ import ietf.mailtrigger.models objects += ietf.mailtrigger.models.Recipient.objects.all() objects += ietf.mailtrigger.models.MailTrigger.objects.all() +import ietf.utils.models +objects += ietf.utils.models.VersionInfo.objects.all() + output("names", objects) diff --git a/ietf/settings.py b/ietf/settings.py index 5493189fc..c50b6753c 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -660,11 +660,19 @@ IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_01 = 13 IDSUBMIT_DEFAULT_CUTOFF_TIME_UTC = datetime.timedelta(hours=23, minutes=59, seconds=59) IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21) +# 14 Jun 2017: New convention: prefix settings with the app name to which +# they (mainly) belong. So here, SUBMIT_, rather than IDSUBMIT_ +SUBMIT_YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/' +SUBMIT_YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/' +SUBMIT_YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/' + IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/' IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/' IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits' -IDSUBMIT_PYANG_COMMAND = 'pyang -p %(modpath)s --verbose --ietf %(model)s' +SUBMIT_PYANG_COMMAND = 'pyang --verbose --ietf -p {libs} {model}' +SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {rfclib} -p {draftlib} {model}' +SUBMIT_YANGLINT_COMMAND = None # use the value above if you have yanglint installed IDSUBMIT_CHECKER_CLASSES = ( "ietf.submit.checkers.DraftIdnitsChecker", @@ -696,10 +704,6 @@ IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE = 450 # in MB IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000 IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB -YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/' -YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/' -YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/' - XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/" # === Meeting Related Settings ================================================= diff --git a/ietf/static/ietf/css/ietf.css b/ietf/static/ietf/css/ietf.css index 96fb06b6d..a0e48e2d3 100644 --- a/ietf/static/ietf/css/ietf.css +++ b/ietf/static/ietf/css/ietf.css @@ -865,3 +865,9 @@ blockquote { #debug-query-table .code .current { background-color: #ddd; } + +.checker-warning, +.checker-success { + line-height: 1.0; + cursor: pointer; +} diff --git a/ietf/submit/admin.py b/ietf/submit/admin.py index 194c0d1e0..5c5b6ff95 100644 --- a/ietf/submit/admin.py +++ b/ietf/submit/admin.py @@ -33,7 +33,7 @@ class SubmissionEventAdmin(admin.ModelAdmin): admin.site.register(SubmissionEvent, SubmissionEventAdmin) class SubmissionCheckAdmin(admin.ModelAdmin): - list_display = ['submission', 'time', 'checker', 'passed', 'errors', 'warnings', 'items'] + list_display = ['submission', 'time', 'checker', 'passed', 'errors', 'warnings', 'message'] raw_id_fields = ['submission'] search_fields = ['submission__name'] admin.site.register(SubmissionCheck, SubmissionCheckAdmin) diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index d1a1ae495..f600f0441 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -2,16 +2,19 @@ import os import re +import sys from xym import xym import shutil import tempfile +import StringIO from django.conf import settings import debug # pyflakes:ignore -from ietf.utils.pipe import pipe from ietf.utils.log import log +from ietf.utils.models import VersionInfo +from ietf.utils.pipe import pipe class DraftSubmissionChecker(): name = "" @@ -47,6 +50,9 @@ class DraftIdnitsChecker(object): # start using this when we provide more in the way of warnings during # submission checking: # symbol = '' + # symbol = u'\ua17d' # Yi syllable 'nit' + # symbol = u'\ub2e1' # Hangul syllable 'nit' + symbol = "" def __init__(self, options=["--submitcheck", "--nitcount", ]): @@ -123,39 +129,66 @@ class DraftYangChecker(object): def check_file_txt(self, path): name = os.path.basename(path) workdir = tempfile.mkdtemp() - errors = [] - warnings = [] - results = {} + errors = 0 + warnings = 0 + message = "" + results = [] + passed = True # Used by the submission tool. Yang checks always pass. - extractor = xym.YangModuleExtractor(path, workdir, strict=True, debug_level = 0) + extractor = xym.YangModuleExtractor(path, workdir, strict=True, strict_examples=False, debug_level=0) if not os.path.exists(path): return None, "%s: No such file or directory: '%s'"%(name.capitalize(), path), errors, warnings, results with open(path) as file: + out = "" + err = "" + code = 0 try: # This places the yang models as files in workdir + saved_stdout = sys.stdout + saved_stderr = sys.stderr + sys.stdout = StringIO.StringIO() + sys.stderr = StringIO.StringIO() extractor.extract_yang_model(file.readlines()) + out = sys.stdout.getvalue() + err = sys.stderr.getvalue() + sys.stdout = saved_stdout + sys.stderr = saved_stderr model_list = extractor.get_extracted_models() except Exception as exc: - passed = False - message = exc - errors = [ (name, None, None, exc) ] - warnings = [] - return passed, message, errors, warnings + code = 1 + err = '\n'.join( [ m for m in [out, err, exc] if m ] ) + if err: + code += 1 + command = "xym" + cmd_version = VersionInfo.objects.get(command=command).version + message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err) + + results.append({ + "name": name, + "passed": passed, + "message": message, + "warnings": 0, + "errors": code, + "items": [], + }) for model in model_list: path = os.path.join(workdir, model) + message = "" modpath = ':'.join([ workdir, - settings.YANG_RFC_MODEL_DIR, - settings.YANG_DRAFT_MODEL_DIR, - settings.YANG_INVAL_MODEL_DIR, + settings.SUBMIT_YANG_RFC_MODEL_DIR, + settings.SUBMIT_YANG_DRAFT_MODEL_DIR, + settings.SUBMIT_YANG_INVAL_MODEL_DIR, ]) with open(path) as file: text = file.readlines() - cmd = settings.IDSUBMIT_PYANG_COMMAND % {"modpath": modpath, "model": path, } + # pyang + cmd_template = settings.SUBMIT_PYANG_COMMAND + command = cmd_template.split()[0] + cmd_version = VersionInfo.objects.get(command=command).version + cmd = cmd_template.format(libs=modpath, model=path) code, out, err = pipe(cmd) - errors = 0 - warnings = 0 items = [] if code > 0: error_lines = err.splitlines() @@ -175,26 +208,54 @@ class DraftYangChecker(object): warnings += 1 except ValueError: pass - results[model] = { - "passed": code == 0, - "message": out+"No validation errors\n" if code == 0 else err, + #passed = passed and code == 0 # For the submission tool. Yang checks always pass + message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if code == 0 else err) + + # yanglint + cmd_template = settings.SUBMIT_YANGLINT_COMMAND + command = cmd_template.split()[0] + cmd_version = VersionInfo.objects.get(command=command).version + cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR) + code, out, err = pipe(cmd) + if code > 0: + error_lines = err.splitlines() + for line in error_lines: + if line.strip(): + try: + if 'err : ' in line: + errors += 1 + if 'warn: ' in line: + warnings += 1 + except ValueError: + pass + #passed = passed and code == 0 # For the submission tool. Yang checks always pass + message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if code == 0 else err) + + if errors==0 and warnings==0: + dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model) + shutil.move(path, dest) + else: + dest = os.path.join(settings.SUBMIT_YANG_INVAL_MODEL_DIR, model) + shutil.move(path, dest) + + # summary result + results.append({ + "name": model, + "passed": passed, + "message": message, "warnings": warnings, "errors": errors, "items": items, - } + }) + shutil.rmtree(workdir) - ## For now, never fail because of failed yang validation. - if len(model_list): - passed = True - else: - passed = None - #passed = all( res["passed"] for res in results.values() ) - message = "\n\n".join([ "\n".join([model+':', res["message"]]) for model, res in results.items() ]) - errors = sum(res["errors"] for res in results.values() ) - warnings = sum(res["warnings"] for res in results.values() ) - items = [ e for res in results.values() for e in res["items"] ] + passed = all( res["passed"] for res in results ) + message = "\n".join([ "\n".join([res['name']+':', res["message"]]) for res in results ]) + errors = sum(res["errors"] for res in results ) + warnings = sum(res["warnings"] for res in results ) + items = [ e for res in results for e in res["items"] ] return passed, message, errors, warnings, items diff --git a/ietf/submit/migrations/0010_data_set_submission_check_symbol.py b/ietf/submit/migrations/0010_data_set_submission_check_symbol.py index 43ca7fc82..8bce3a132 100644 --- a/ietf/submit/migrations/0010_data_set_submission_check_symbol.py +++ b/ietf/submit/migrations/0010_data_set_submission_check_symbol.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- from __future__ import unicode_literals from tqdm import tqdm diff --git a/ietf/submit/models.py b/ietf/submit/models.py index 90a6a1378..82ca0e067 100644 --- a/ietf/submit/models.py +++ b/ietf/submit/models.py @@ -65,6 +65,10 @@ class Submission(models.Model): def existing_document(self): return Document.objects.filter(name=self.name).first() + def latest_checks(self): + checks = [ self.checks.filter(checker=c, passed__in=[True,False]).latest('time') for c in self.checks.values_list('checker', flat=True).distinct() ] + return checks + class SubmissionCheck(models.Model): time = models.DateTimeField(auto_now=True) submission = models.ForeignKey(Submission, related_name='checks') diff --git a/ietf/submit/test_submission.nonascii b/ietf/submit/test_submission.nonascii index 02c044aa3..b6c6b4fe9 100644 --- a/ietf/submit/test_submission.nonascii +++ b/ietf/submit/test_submission.nonascii @@ -71,99 +71,90 @@ Table of Contents 2. Yang - file "ietf-mpls@2015-10-16.yang" + file "ietf-yang-metadata@2016-08-05.yang" - module ietf-mpls { + module ietf-yang-metadata { - namespace "urn:ietf:params:xml:ns:yang:ietf-mpls"; + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata"; - prefix "mpls"; + prefix "md"; - import ietf-routing { - prefix "rt"; - } + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; - import ietf-interfaces { - prefix "if"; - } + contact + "WG Web: - organization "TBD"; + WG List: - contact "TBD"; + WG Chair: Lou Berger + + + WG Chair: Kent Watsen + + + Editor: Ladislav Lhotka + "; description - "This YANG module defines the essential components for the - management of the MPLS subsystem."; + "This YANG module defines an 'extension' statement that allows + for defining metadata annotations. - revision "2015-10-16" { + Copyright (c) 2016 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject to + the license terms contained in, the Simplified BSD License set + forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 7952 + (http://www.rfc-editor.org/info/rfc7952); see the RFC itself + for full legal notices."; + + revision 2016-08-05 { description - "Initial revision"; - reference "RFC 3031: A YANG Data Model for base MPLS"; + "Initial revision."; + reference + "RFC 7952: Defining and Using Metadata with YANG"; } - typedef mpls-label { - type uint32 { - range "0..1048575"; - } + extension annotation { + argument name; description - "The MPLS label range"; - } + "This extension allows for defining metadata annotations in + YANG modules. The 'md:annotation' statement can appear only + at the top level of a YANG module or submodule, i.e., it + becomes a new alternative in the ABNF production rule for + 'body-stmts' (Section 14 in RFC 7950). - typedef percent { - type uint16 { - range "0 .. 100"; - } - description "Percentage"; - } + The argument of the 'md:annotation' statement defines the name + of the annotation. Syntactically, it is a YANG identifier as + defined in Section 6.2 of RFC 7950. - grouping interface-mpls { - description "MPLS interface properties grouping"; - leaf enabled { - type boolean; - description - "'true' if mpls encapsulation is enabled on the - interface. 'false' if mpls encapsulation is enabled - on the interface."; - } - } + An annotation defined with this 'extension' statement inherits + the namespace and other context from the YANG module in which + it is defined. - augment "/rt:routing/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features config, e.g. MPLS static LSP, MPLS - LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc."; + The data type of the annotation value is specified in the same + way as for a leaf data node using the 'type' statement. - list interface { - key "name"; - description "List of MPLS interfaces"; - leaf name { - type if:interface-ref; - description - "The name of a configured MPLS interface"; - } - container config { - description "Holds intended configuration"; - uses interface-mpls; - } - container state { - config false; - description "Holds inuse configuration"; - uses interface-mpls; - } - } - } - } + The semantics of the annotation and other documentation can be + specified using the following standard YANG substatements (all + are optional): 'description', 'if-feature', 'reference', + 'status', and 'units'. - augment "/rt:routing-state/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - config false; - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features state"; - } + A server announces support for a particular annotation by + including the module in which the annotation is defined among + the advertised YANG modules, e.g., in a NETCONF + message or in the YANG library (RFC 7950). The annotation can + then be attached to any instance of a data node defined in any + YANG module that is advertised by the server. + + XML encoding and JSON encoding of annotations are defined in + RFC 7952."; } } diff --git a/ietf/submit/test_submission.txt b/ietf/submit/test_submission.txt index b6551bf2d..30e063c47 100644 --- a/ietf/submit/test_submission.txt +++ b/ietf/submit/test_submission.txt @@ -73,99 +73,90 @@ Table of Contents 2. Yang - file "ietf-mpls@2015-10-16.yang" + file "ietf-yang-metadata@2016-08-05.yang" - module ietf-mpls { + module ietf-yang-metadata { - namespace "urn:ietf:params:xml:ns:yang:ietf-mpls"; + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata"; - prefix "mpls"; + prefix "md"; - import ietf-routing { - prefix "rt"; - } + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; - import ietf-interfaces { - prefix "if"; - } + contact + "WG Web: - organization "TBD"; + WG List: - contact "TBD"; + WG Chair: Lou Berger + + + WG Chair: Kent Watsen + + + Editor: Ladislav Lhotka + "; description - "This YANG module defines the essential components for the - management of the MPLS subsystem."; + "This YANG module defines an 'extension' statement that allows + for defining metadata annotations. - revision "2015-10-16" { + Copyright (c) 2016 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject to + the license terms contained in, the Simplified BSD License set + forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 7952 + (http://www.rfc-editor.org/info/rfc7952); see the RFC itself + for full legal notices."; + + revision 2016-08-05 { description - "Initial revision"; - reference "RFC 3031: A YANG Data Model for base MPLS"; + "Initial revision."; + reference + "RFC 7952: Defining and Using Metadata with YANG"; } - typedef mpls-label { - type uint32 { - range "0..1048575"; - } + extension annotation { + argument name; description - "The MPLS label range"; - } + "This extension allows for defining metadata annotations in + YANG modules. The 'md:annotation' statement can appear only + at the top level of a YANG module or submodule, i.e., it + becomes a new alternative in the ABNF production rule for + 'body-stmts' (Section 14 in RFC 7950). - typedef percent { - type uint16 { - range "0 .. 100"; - } - description "Percentage"; - } + The argument of the 'md:annotation' statement defines the name + of the annotation. Syntactically, it is a YANG identifier as + defined in Section 6.2 of RFC 7950. - grouping interface-mpls { - description "MPLS interface properties grouping"; - leaf enabled { - type boolean; - description - "'true' if mpls encapsulation is enabled on the - interface. 'false' if mpls encapsulation is enabled - on the interface."; - } - } + An annotation defined with this 'extension' statement inherits + the namespace and other context from the YANG module in which + it is defined. - augment "/rt:routing/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features config, e.g. MPLS static LSP, MPLS - LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc."; + The data type of the annotation value is specified in the same + way as for a leaf data node using the 'type' statement. - list interface { - key "name"; - description "List of MPLS interfaces"; - leaf name { - type if:interface-ref; - description - "The name of a configured MPLS interface"; - } - container config { - description "Holds intended configuration"; - uses interface-mpls; - } - container state { - config false; - description "Holds inuse configuration"; - uses interface-mpls; - } - } - } - } + The semantics of the annotation and other documentation can be + specified using the following standard YANG substatements (all + are optional): 'description', 'if-feature', 'reference', + 'status', and 'units'. - augment "/rt:routing-state/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - config false; - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features state"; - } + A server announces support for a particular annotation by + including the module in which the annotation is defined among + the advertised YANG modules, e.g., in a NETCONF + message or in the YANG library (RFC 7950). The annotation can + then be attached to any instance of a data node defined in any + YANG module that is advertised by the server. + + XML encoding and JSON encoding of annotations are defined in + RFC 7952."; } } diff --git a/ietf/submit/test_submission.xml b/ietf/submit/test_submission.xml index a8b6d9cde..4dbe41382 100644 --- a/ietf/submit/test_submission.xml +++ b/ietf/submit/test_submission.xml @@ -35,100 +35,91 @@
file "ietf-mpls@2015-10-16.yang" + file "ietf-yang-metadata@2016-08-05.yang" -module ietf-mpls { +module ietf-yang-metadata { - namespace "urn:ietf:params:xml:ns:yang:ietf-mpls"; + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata"; - prefix "mpls"; + prefix "md"; - import ietf-routing { - prefix "rt"; - } + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; - import ietf-interfaces { - prefix "if"; - } + contact + "WG Web: - organization "TBD"; + WG List: - contact "TBD"; + WG Chair: Lou Berger + - description - "This YANG module defines the essential components for the - management of the MPLS subsystem."; + WG Chair: Kent Watsen + - revision "2015-10-16" { - description - "Initial revision"; - reference "RFC 3031: A YANG Data Model for base MPLS"; - } + Editor: Ladislav Lhotka + "; - typedef mpls-label { - type uint32 { - range "0..1048575"; - } - description - "The MPLS label range"; - } + description + "This YANG module defines an 'extension' statement that allows + for defining metadata annotations. - typedef percent { - type uint16 { - range "0 .. 100"; - } - description "Percentage"; - } + Copyright (c) 2016 IETF Trust and the persons identified as + authors of the code. All rights reserved. - grouping interface-mpls { - description "MPLS interface properties grouping"; - leaf enabled { - type boolean; - description - "'true' if mpls encapsulation is enabled on the - interface. 'false' if mpls encapsulation is enabled - on the interface."; - } - } + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject to + the license terms contained in, the Simplified BSD License set + forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). - augment "/rt:routing/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features config, e.g. MPLS static LSP, MPLS - LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc."; + This version of this YANG module is part of RFC 7952 + (http://www.rfc-editor.org/info/rfc7952); see the RFC itself + for full legal notices."; - list interface { - key "name"; - description "List of MPLS interfaces"; - leaf name { - type if:interface-ref; - description - "The name of a configured MPLS interface"; - } - container config { - description "Holds intended configuration"; - uses interface-mpls; - } - container state { - config false; - description "Holds inuse configuration"; - uses interface-mpls; - } - } - } - } + revision 2016-08-05 { + description + "Initial revision."; + reference + "RFC 7952: Defining and Using Metadata with YANG"; + } - augment "/rt:routing-state/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - config false; - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features state"; - } - } + extension annotation { + argument name; + description + "This extension allows for defining metadata annotations in + YANG modules. The 'md:annotation' statement can appear only + at the top level of a YANG module or submodule, i.e., it + becomes a new alternative in the ABNF production rule for + 'body-stmts' (Section 14 in RFC 7950). + + The argument of the 'md:annotation' statement defines the name + of the annotation. Syntactically, it is a YANG identifier as + defined in Section 6.2 of RFC 7950. + + An annotation defined with this 'extension' statement inherits + the namespace and other context from the YANG module in which + it is defined. + + The data type of the annotation value is specified in the same + way as for a leaf data node using the 'type' statement. + + The semantics of the annotation and other documentation can be + specified using the following standard YANG substatements (all + are optional): 'description', 'if-feature', 'reference', + 'status', and 'units'. + + A server announces support for a particular annotation by + including the module in which the annotation is defined among + the advertised YANG modules, e.g., in a NETCONF + message or in the YANG library (RFC 7950). The annotation can + then be attached to any instance of a data node defined in any + YANG module that is advertised by the server. + + XML encoding and JSON encoding of annotations are defined in + RFC 7952."; + } } diff --git a/ietf/submit/test_submission_invalid_yang.txt b/ietf/submit/test_submission_invalid_yang.txt new file mode 100644 index 000000000..d3b9a9d3d --- /dev/null +++ b/ietf/submit/test_submission_invalid_yang.txt @@ -0,0 +1,213 @@ + + + + +Network Working Group A. Name +Internet-Draft Test Centre Inc. +Intended status: Informational %(month)s %(year)s +Expires: %(expiration)s + + + Testing Tests + %(name)s + +Abstract + + This document describes how to test tests. + +Status of This Memo + + This Internet-Draft is submitted in full conformance with the + provisions of BCP 78 and BCP 79. + + Internet-Drafts are working documents of the Internet Engineering + Task Force (IETF). Note that other groups may also distribute + working documents as Internet-Drafts. The list of current Internet- + Drafts is at http://datatracker.ietf.org/drafts/current/. + + Internet-Drafts are draft documents valid for a maximum of six months + and may be updated, replaced, or obsoleted by other documents at any + time. It is inappropriate to use Internet-Drafts as reference + material or to cite them other than as "work in progress." + + This Internet-Draft will expire on %(expiration)s. + +Copyright Notice + + Copyright (c) %(year)s IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + + + + + + +Name Expires %(expiration)s [Page 1] + +Internet-Draft Testing Tests %(month)s %(year)s + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 + 2. Yang . . . . . . . . . . . . . . . . . . . . . . . . . . . . 2 + 3. JSON example . . . . . . . . . . . . . . . . . . . . . . . . 2 + 4. Security Considerations . . . . . . . . . . . . . . . . . . . 2 + 5. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 2 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . . 2 + +1. Introduction + + This document describes a protocol for testing tests. + +2. Yang + + file "ietf-yang-metadata@2016-08-05.yang" + + module ietf-yang-metadata { + + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata"; + + prefix "md"; + + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; + + contact + "WG Web: + + WG List: + + WG Chair: Lou Berger + + + WG Chair: Kent Watsen + + + Editor: Ladislav Lhotka + "; + + description + "This YANG module defines an 'extension' statement that allows + for defining metadata annotations. + + Copyright (c) 2016 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject to + the license terms contained in, the Simplified BSD License set + forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 7952 + (http://www.rfc-editor.org/info/rfc7952); see the RFC itself + for full legal notices."; + + revision 2016-08-05 { + description + "Initial revision."; + reference + "RFC 7952: Defining and Using Metadata with YANG"; + } + + extension annotation { + argument name;; + description + "This extension allows for defining metadata annotations in + YANG modules. The 'md:annotation' statement can appear only + at the top level of a YANG module or submodule, i.e., it + becomes a new alternative in the ABNF production rule for + 'body-stmts' (Section 14 in RFC 7950). + + The argument of the 'md:annotation' statement defines the name + of the annotation. Syntactically, it is a YANG identifier as + defined in Section 6.2 of RFC 7950. + + An annotation defined with this 'extension' statement inherits + the namespace and other context from the YANG module in which + it is defined. + + The data type of the annotation value is specified in the same + way as for a leaf data node using the 'type' statement. + + The semantics of the annotation and other documentation can be + specified using the following standard YANG substatements (all + are optional): 'description', 'if-feature', 'reference', + 'status', and 'units'. + + A server announces support for a particular annotation by + including the module in which the annotation is defined among + the advertised YANG modules, e.g., in a NETCONF + message or in the YANG library (RFC 7950). The annotation can + then be attached to any instance of a data node defined in any + YANG module that is advertised by the server. + + XML encoding and JSON encoding of annotations are defined in + RFC 7952."; + } + } + + + +3. JSON example + + The JSON object should look like this: + + { + "test": 1234 + } + +4. Security Considerations + + There are none. + +5. IANA Considerations + + No new registrations for IANA. + +Author's Address + + Author Name + Test Centre Inc. + 42 Some Road + Some Where 12345 + UK + + Email: author@example.com + + + + + + + + + + + + + + + + + + + + + + + +Name Expires %(expiration)s [Page 2] diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index ce93184bb..4419d95e2 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -24,6 +24,7 @@ from ietf.person.factories import UserFactory, PersonFactory from ietf.submit.models import Submission, Preapproval from ietf.submit.mail import add_submission_email, process_response_email from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.models import VersionInfo from ietf.utils.test_data import make_test_data from ietf.utils.test_utils import login_testing_unauthorized, unicontent, TestCase @@ -68,17 +69,17 @@ class SubmitTests(TestCase): self.archive_dir = self.tempdir('submit-archive') settings.INTERNET_DRAFT_ARCHIVE_DIR = self.archive_dir - self.saved_yang_rfc_model_dir = settings.YANG_RFC_MODEL_DIR + self.saved_yang_rfc_model_dir = settings.SUBMIT_YANG_RFC_MODEL_DIR self.yang_rfc_model_dir = self.tempdir('yang-rfc-model') - settings.YANG_RFC_MODEL_DIR = self.yang_rfc_model_dir + settings.SUBMIT_YANG_RFC_MODEL_DIR = self.yang_rfc_model_dir - self.saved_yang_draft_model_dir = settings.YANG_DRAFT_MODEL_DIR + self.saved_yang_draft_model_dir = settings.SUBMIT_YANG_DRAFT_MODEL_DIR self.yang_draft_model_dir = self.tempdir('yang-draft-model') - settings.YANG_DRAFT_MODEL_DIR = self.yang_draft_model_dir + settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.yang_draft_model_dir - self.saved_yang_inval_model_dir = settings.YANG_INVAL_MODEL_DIR + self.saved_yang_inval_model_dir = settings.SUBMIT_YANG_INVAL_MODEL_DIR self.yang_inval_model_dir = self.tempdir('yang-inval-model') - settings.YANG_INVAL_MODEL_DIR = self.yang_inval_model_dir + settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.yang_inval_model_dir def tearDown(self): shutil.rmtree(self.staging_dir) @@ -91,9 +92,9 @@ class SubmitTests(TestCase): settings.INTERNET_DRAFT_PATH = self.saved_internet_draft_path settings.IDSUBMIT_REPOSITORY_PATH = self.saved_idsubmit_repository_path settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir - settings.YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir - settings.YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir - settings.YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir + settings.SUBMIT_YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir + settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir + settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir def do_submission(self, name, rev, group=None, formats=["txt",]): @@ -220,6 +221,11 @@ class SubmitTests(TestCase): r = self.client.get(status_url) self.assertEqual(r.status_code, 200) + + self.assertContains(r, 'xym') + self.assertContains(r, 'pyang') + self.assertContains(r, 'yanglint') + q = PyQuery(r.content) approve_button = q('[type=submit]:contains("Approve")') self.assertEqual(len(approve_button), 1) @@ -266,6 +272,15 @@ class SubmitTests(TestCase): self.assertTrue("ames-chairs@" in outbox[-1]["To"].lower()) self.assertTrue("mars-chairs@" in outbox[-1]["To"].lower()) + # fetch the document page + url = urlreverse('ietf.doc.views_doc.document_main', kwargs={'name':name}) + r = self.client.get(url) + self.assertContains(r, name) + self.assertContains(r, 'Active Internet-Draft') + self.assertContains(r, 'mars WG') + self.assertContains(r, 'Yang Validation') + self.assertContains(r, 'WG Document') + def test_submit_new_wg_txt(self): self.submit_new_wg(["txt"]) @@ -943,6 +958,38 @@ class SubmitTests(TestCase): self.assertIn('The idnits check returned 1 error', m) + def test_submit_invalid_yang(self): + make_test_data() + + name = "draft-yang-testing-invalid" + rev = "00" + group = None + + # get + url = urlreverse('ietf.submit.views.upload_submission') + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + + # submit + files = {"txt": submission_file(name, rev, group, "txt", "test_submission_invalid_yang.txt") } + + r = self.client.post(url, files) + self.assertEqual(r.status_code, 302) + status_url = r["Location"] + r = self.client.get(status_url) + q = PyQuery(r.content) + # + self.assertContains(r, u'The yang validation returned 1 error') + # + m = q('#yang-validation-message').text() + for command in ['xym', 'pyang', 'yanglint']: + version = VersionInfo.objects.get(command=command).version + self.assertIn(version, m) + self.assertIn("draft-yang-testing-invalid-00.txt", m) + self.assertIn("error: syntax error: illegal keyword: ;", m) + self.assertIn("No validation errors", m) + class ApprovalsTestCase(TestCase): def test_approvals(self): diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 743760116..23f122589 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -165,7 +165,9 @@ def upload_submission(request): def apply_check(submission, checker, method, fn): func = getattr(checker, method) passed, message, errors, warnings, items = func(fn) - check = SubmissionCheck(submission=submission, checker=checker.name, passed=passed, message=message, errors=errors, warnings=warnings, items=items, symbol=checker.symbol) + check = SubmissionCheck(submission=submission, checker=checker.name, passed=passed, + message=message, errors=errors, warnings=warnings, items=items, + symbol=checker.symbol) check.save() for checker_path in settings.IDSUBMIT_CHECKER_CLASSES: diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index 870e188a5..4925d992d 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -76,15 +76,6 @@ {% if latest_revision and latest_revision.time.date != doc.time.date %} (latest revision {{ latest_revision.time|date:"Y-m-d" }}) {% endif %} - {% for check in doc.submission.checks.all %} - {% if check.passed != None and check.symbol.strip %} - {% if check.errors or check.warnings %} - {{ check.symbol|safe }} - {% else %} - {{ check.symbol|safe }} - {% endif %} - {% endif %} - {% endfor %} @@ -192,6 +183,27 @@ + {% for check in doc.submission.latest_checks %} + {% if check.passed != None and check.symbol.strip %} + + + {{ check.checker|title }} + + + {% if check.errors or check.warnings %} + {{ check.symbol|safe }} + {% else %} + {{ check.symbol|safe }} + {% endif %} + + {{ check.errors }} errors, {{ check.warnings }} warnings. + + {% include "doc/yang-check-modal-overlay.html" %} + + + {% endif %} + {% endfor %} + {% if review_requests or can_request_review %} diff --git a/ietf/templates/doc/search/search_result_row.html b/ietf/templates/doc/search/search_result_row.html index 1ada48b26..f2adcb93f 100644 --- a/ietf/templates/doc/search/search_result_row.html +++ b/ietf/templates/doc/search/search_result_row.html @@ -56,13 +56,14 @@ {% if doc.latest_revision_date|timesince_days|new_enough:request and doc.get_state_slug != "rfc" %}{% endif %} - {% for check in doc.submission.checks.all %} + {% for check in doc.submission.latest_checks %} {% if check.passed != None and check.symbol.strip %} {% if check.errors or check.warnings %} - {{ check.symbol|safe }} + {{ check.symbol|safe }} {% else %} - {{ check.symbol|safe }} + {{ check.symbol|safe }} {% endif %} + {% include "doc/yang-check-modal-overlay.html" %} {% endif %} {% endfor %} diff --git a/ietf/templates/doc/yang-check-modal-overlay.html b/ietf/templates/doc/yang-check-modal-overlay.html new file mode 100644 index 000000000..10b3bfb36 --- /dev/null +++ b/ietf/templates/doc/yang-check-modal-overlay.html @@ -0,0 +1,17 @@ + {% load ietf_filters %}{% load origin %}{% origin %} + diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index 473829130..a4b9d5a8b 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -44,7 +44,7 @@ Your draft has NOT been verified to pass the submission checks. {% endif %}

- {% for check in submission.checks.all %} + {% for check in submission.latest_checks %} {% if check.errors %}

The {{check.checker}} returned {{ check.errors }} error{{ check.errors|pluralize }} @@ -58,7 +58,7 @@ {% endif %} {% endfor %} - {% for check in submission.checks.all %} + {% for check in submission.latest_checks %} {% if check.passed != None %} @@ -67,9 +67,9 @@

- +
-
+ 
{{ check.message }}
diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index bfce0b08a..73e166409 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -1,3 +1,5 @@ +from django.contrib import admin +from ietf.utils.models import VersionInfo def name(obj): if hasattr(obj, 'abbrev'): @@ -45,3 +47,6 @@ def admin_link(field, label=None, ordering="", display=name, suffix=""): _link.admin_order_field = ordering return _link +class VersionInfoAdmin(admin.ModelAdmin): + list_display = ['command', 'switch', 'version', 'time', ] +admin.site.register(VersionInfo, VersionInfoAdmin) diff --git a/ietf/utils/management/commands/populate_yang_model_dirs.py b/ietf/utils/management/commands/populate_yang_model_dirs.py index 5025f5a8e..8aabd56fe 100644 --- a/ietf/utils/management/commands/populate_yang_model_dirs.py +++ b/ietf/utils/management/commands/populate_yang_model_dirs.py @@ -1,8 +1,10 @@ +from __future__ import print_function, unicode_literals + import os import sys import time -from pathlib import Path +from pathlib2 import Path from StringIO import StringIO from textwrap import dedent from xym import xym @@ -10,9 +12,11 @@ from xym import xym from django.conf import settings from django.core.management.base import BaseCommand +import debug # pyflakes:ignore + class Command(BaseCommand): """ - Populate the yang models repositories from drafts and RFCs. + Populate the yang module repositories from drafts and RFCs. Extracts yang models from RFCs (found in settings.RFC_PATH and places them in settings.YANG_RFC_MODEL_DIR, and from active drafts, placed in @@ -57,22 +61,34 @@ class Command(BaseCommand): verbosity = int(options.get('verbosity')) def extract_from(file, dir, strict=True): + saved_stdout = sys.stdout saved_stderr = sys.stderr - sys.stderr = StringIO() + xymerr = StringIO() + xymout = StringIO() + sys.stderr = xymerr + sys.stdout = xymout model_list = [] try: - model_list = xym.xym(str(item), str(item.parent), str(dir), strict=strict, debug_level=(verbosity>1)) + model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=verbosity-2) for name in model_list: modfile = moddir / name - mtime = item.stat().st_mtime + mtime = file.stat().st_mtime os.utime(str(modfile), (mtime, mtime)) if '"' in name: name = name.replace('"', '') modfile.rename(str(moddir/name)) - model_list = [ n.replace('"','') for n in model_list ] + model_list = [ n.replace('"','') for n in model_list ] except Exception as e: - print("** Error when extracting from %s: %s" % (item, str(e))) + print("** Error when extracting from %s: %s" % (file, str(e))) + sys.stdout = saved_stdout sys.stderr = saved_stderr + # + if verbosity > 1: + outmsg = xymout.getvalue() + self.stdout.write(outmsg) + if verbosity>2: + errmsg = xymerr.getvalue() + self.stderr.write(errmsg) return model_list # Extract from new RFCs @@ -87,10 +103,10 @@ class Command(BaseCommand): for item in moddir.iterdir(): if item.stat().st_mtime > latest: latest = item.stat().st_mtime - + print("Extracting to %s ..." % moddir) for item in rfcdir.iterdir(): - if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt'): + if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit(): if item.stat().st_mtime > latest: model_list = extract_from(item, moddir) for name in model_list: @@ -124,20 +140,25 @@ class Command(BaseCommand): print("Extracting to %s ..." % moddir) for item in draftdir.iterdir(): - if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): - model_list = extract_from(item, moddir) - for name in model_list: - if not name.startswith('example'): - if verbosity > 1: - print(" Extracted valid module from %s: %s" % (item, name)) + try: + if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): + model_list = extract_from(item, moddir) + for name in model_list: + if not name.startswith('example'): + if verbosity > 1: + print(" Extracted valid module from %s: %s" % (item, name)) + else: + sys.stdout.write('.') + sys.stdout.flush() else: - sys.stdout.write('.') - sys.stdout.flush() - else: - modfile = moddir / name - modfile.unlink() - if verbosity > 1: - print(" Skipped module from %s: %s" % (item, name)) + modfile = moddir / name + modfile.unlink() + if verbosity > 1: + print(" Skipped module from %s: %s" % (item, name)) + except UnicodeDecodeError as e: + sys.stderr.write('\nError: %s\n' % (e, )) + sys.stderr.write(item.name) + sys.stderr.write('\n') print("") # Extract invalid modules from drafts @@ -151,22 +172,28 @@ class Command(BaseCommand): print("Extracting to %s ..." % moddir) for item in draftdir.iterdir(): - if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): - model_list = extract_from(item, moddir, strict=False) - for name in model_list: - modfile = moddir / name - if (valdir/name).exists(): - modfile.unlink() - if verbosity > 1: - print(" Skipped valid module from %s: %s" % (item, name)) - elif not name.startswith('example'): - if verbosity > 1: - print(" Extracted invalid module from %s: %s" % (item, name)) + try: + if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): + model_list = extract_from(item, moddir, strict=False) + for name in model_list: + modfile = moddir / name + if (valdir/name).exists(): + modfile.unlink() + if verbosity > 1: + print(" Skipped valid module from %s: %s" % (item, name)) + elif not name.startswith('example'): + if verbosity > 1: + print(" Extracted invalid module from %s: %s" % (item, name)) + else: + sys.stdout.write('.') + sys.stdout.flush() else: - sys.stdout.write('.') - sys.stdout.flush() - else: - modfile.unlink() - if verbosity > 1: - print(" Skipped module from %s: %s" % (item, name)) + modfile.unlink() + if verbosity > 1: + print(" Skipped module from %s: %s" % (item, name)) + except UnicodeDecodeError as e: + sys.stderr.write('\nError: %s\n' % (e, )) + sys.stderr.write(item.name) + sys.stderr.write('\n') + print("") diff --git a/ietf/utils/management/commands/run_yang_model_checks.py b/ietf/utils/management/commands/run_yang_model_checks.py new file mode 100644 index 000000000..4d44c145b --- /dev/null +++ b/ietf/utils/management/commands/run_yang_model_checks.py @@ -0,0 +1,82 @@ +from __future__ import print_function, unicode_literals + +import sys +import json + +from textwrap import dedent + +from django.core.management.base import BaseCommand + +import debug # pyflakes:ignore + +from ietf.doc.models import Document, State +from ietf.submit.models import Submission, SubmissionCheck +from ietf.submit.checkers import DraftYangChecker + + +class Command(BaseCommand): + """ + Run yang model checks on active drafts. + + Repeats the yang checks in ietf/submit/checkers.py for active drafts, in + order to catch changes in status due to new modules becoming available in + the module directories. + + """ + + help = dedent(__doc__).strip() + + def add_arguments(self, parser): + parser.add_argument('filenames', nargs="*") + parser.add_argument('--clean', + action='store_true', dest='clean', default=False, + help='Remove the current directory content before writing new models.') + + + def check_yang(self, checker, draft, force=False): + if self.verbosity > 1: + self.stdout.write("Checking %s-%s" % (draft.name, draft.rev)) + else: + sys.stderr.write('.') + submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first() + if submission or force: + check = submission.checks.filter(checker=checker.name).order_by('-id').first() + if check or force: + result = checker.check_file_txt(draft.get_file_name()) + passed, message, errors, warnings, items = result + if self.verbosity > 2: + self.stdout.write(" Errors: %s\n" + " Warnings: %s\n" + " Message:\n%s\n" % (errors, warnings, message)) + items = json.loads(json.dumps(items)) + new_res = (passed, errors, warnings, message) + old_res = (check.passed, check.errors, check.warnings, check.message) if check else () + if new_res != old_res: + if self.verbosity > 1: + self.stdout.write(" Saving new yang checker results for %s-%s" % (draft.name, draft.rev)) + SubmissionCheck.objects.create(submission=submission, checker=checker.name, passed=passed, + message=message, errors=errors, warnings=warnings, items=items, + symbol=checker.symbol) + else: + self.stderr.write("Error: did not find any submission object for %s-%s\n" % (draft.name, draft.rev)) + + def handle(self, *filenames, **options): + """ + """ + + self.verbosity = int(options.get('verbosity')) + filenames = options.get('filenames') + + active_state = State.objects.get(type="draft", slug="active") + + checker = DraftYangChecker() + if filenames: + for name in filenames: + parts = name.rsplit('-',1) + if len(parts)==2 and len(parts[1])==2 and parts[1].isdigit(): + name = parts[0] + draft = Document.objects.get(name=name) + self.check_yang(checker, draft, force=True) + else: + for draft in Document.objects.filter(states=active_state, type_id='draft'): + self.check_yang(checker, draft) diff --git a/ietf/utils/management/commands/update_external_command_info.py b/ietf/utils/management/commands/update_external_command_info.py new file mode 100644 index 000000000..04949951c --- /dev/null +++ b/ietf/utils/management/commands/update_external_command_info.py @@ -0,0 +1,37 @@ +from __future__ import print_function, unicode_literals + +import sys + +from textwrap import dedent + +from django.core.management.base import BaseCommand + +import debug # pyflakes:ignore + +from ietf.utils.models import VersionInfo +from ietf.utils.pipe import pipe + +class Command(BaseCommand): + """ + Update the version information for external commands used by the datatracker. + + Iterates through the entries in the VersionInfo table, runs the relevant + command, and updates the version string with the result. + + """ + + help = dedent(__doc__).strip() + + def handle(self, *filenames, **options): + for c in VersionInfo.objects.filter(used=True): + cmd = "%s %s" % (c.command, c.switch) + code, out, err = pipe(cmd) + if code != 0: + sys.stderr.write("Command '%s' retuned %s: \n%s\n%s\n" % (cmd, code, out, err)) + else: + c.version = (out.strip()+'\n'+err.strip()).strip() + if options.get('verbosity', 1) > 1: + sys.stdout.write( + "Command: %s\n" + " Version: %s\n" % (cmd, c.version)) + c.save() diff --git a/ietf/utils/migrations/0003_versioninfo.py b/ietf/utils/migrations/0003_versioninfo.py new file mode 100644 index 000000000..0c1dca17a --- /dev/null +++ b/ietf/utils/migrations/0003_versioninfo.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.7 on 2017-06-15 03:31 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('utils', '0002_dumpinfo_tz'), + ] + + operations = [ + migrations.CreateModel( + name='VersionInfo', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('time', models.DateTimeField(auto_now=True)), + ('command', models.CharField(max_length=32)), + ('switch', models.CharField(max_length=16)), + ('version', models.CharField(max_length=64)), + ('used', models.BooleanField(default=True)), + ], + ), + ] diff --git a/ietf/utils/models.py b/ietf/utils/models.py index a38d1057b..c176b3cab 100644 --- a/ietf/utils/models.py +++ b/ietf/utils/models.py @@ -7,3 +7,12 @@ class DumpInfo(models.Model): host = models.CharField(max_length=128) tz = models.CharField(max_length=32, default='UTC') +class VersionInfo(models.Model): + time = models.DateTimeField(auto_now=True) + command = models.CharField(max_length=32) + switch = models.CharField(max_length=16) + version = models.CharField(max_length=64) + used = models.BooleanField(default=True) + class Meta: + verbose_name_plural = 'VersionInfo' + diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index 4f27081a8..a35e972f0 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -8,7 +8,8 @@ from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from ietf import api -from ietf.utils.models import DumpInfo +from ietf.utils.models import DumpInfo, VersionInfo + class UserResource(ModelResource): username = CharField() @@ -17,6 +18,7 @@ class UserResource(ModelResource): queryset = User.objects.all() serializer = api.Serializer() + class ContentTypeResource(ModelResource): username = CharField() class Meta: @@ -24,6 +26,7 @@ class ContentTypeResource(ModelResource): queryset = ContentType.objects.all() serializer = api.Serializer() + class DumpInfoResource(ModelResource): class Meta: cache = SimpleCache() @@ -36,3 +39,19 @@ class DumpInfoResource(ModelResource): } api.utils.register(DumpInfoResource()) + +class VersionInfoResource(ModelResource): + class Meta: + queryset = VersionInfo.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'versioninfo' + filtering = { + "id": ALL, + "time": ALL, + "command": ALL, + "switch": ALL, + "version": ALL, + "used": ALL, + } +api.utils.register(VersionInfoResource()) diff --git a/requirements.txt b/requirements.txt index ad419e823..6b5a886ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,8 +31,9 @@ mimeparse>=0.1.3 # from TastyPie mock>=2.0.0 MySQL-python>=1.2.5 pathlib>=1.0 +pathlib2>=2.3.0 Pillow>=3.0 -pyang>=1.6 +pyang>=1.7.2 # Handled as an svn:externals component pyflakes>=0.8.1 pyquery>=1.2.13,!=1.2.14 # Pyqyery 1.2.14 fails on some selectors or stacked selectors python-dateutil>=2.2 @@ -50,6 +51,6 @@ Trac>=1.0.10,<1.2 Unidecode>=0.4.18 #wsgiref>=0.1.2 xml2rfc>=2.5.0,!=2.6.0 -xym>=0.1.2,!=0.3 +#xym>=0.1.2,!=0.3 # Handled as an svn:externals component #zxcvbn-python>=4.4.14 # Not needed until we do back-end password entropy validation