From d98054c10316d5844e9cc7aebd4ff0ad11ab263c Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Thu, 15 Jun 2017 16:09:28 +0000 Subject: [PATCH] Added a new yang checker, 'yanglint', to the existing Yang checker class, in addition to the existing 'pyang' checker. Added modal overlay displays showing the yang check results every place the yin/yang symbol is shown (red or green) to indicate the presencee and result of yang checks. Added a Yang Validation: line in the document meta-information section on the document's page in the datatracker. Added the result of the xym extaction to the yang check results, to make extration failures visible. Added the version of the used xym, pyang, and yanglint commands to the check results. Added an action to move successfully extracted and validated modules to the module library directories immediately on submission. Added the xym and pyang repositories as svn:external components, rather than listing them in requirements.txt, as there has been delays of many months between essential features in the repositories, and an actual release. We may get occasional buildbot failures if broken code is pulled in from the repository, but better that than the functionality failure of severely outdated componets. Added a new management command to re-run yang validation for active drafts for which yang modules were found at submission time, in order to pick up imported models which may have arrived in the model libraries after the draft's submission. Run daily from bin/daily. Added a table to hold version information for external commands. The yang checker output should include the version information of the used checkers, but seems unnecessary to run each command with its --version switch every time we check a module... Added a new management command to collect version information for external commands on demand. To be run daily from bin/daily. Added tests to verify that xym, pyang and yanglint information is available on the submission confirmation page, and updated the yang module contained in the test document to validate under both pyang and yanglint. Updated admin.py and resource.py files as needed. - Legacy-Id: 13630 --- .gitignore | 1 + bin/daily | 9 +- ietf/doc/templatetags/ietf_filters.py | 3 + ietf/doc/views_doc.py | 8 + ietf/name/fixtures/names.json | 33 ++++ ietf/name/generate_fixtures.py | 3 + ietf/settings.py | 14 +- ietf/static/ietf/css/ietf.css | 6 + ietf/submit/admin.py | 2 +- ietf/submit/checkers.py | 121 ++++++++++---- .../0010_data_set_submission_check_symbol.py | 2 +- ietf/submit/models.py | 4 + ietf/submit/test_submission.nonascii | 139 ++++++++-------- ietf/submit/test_submission.txt | 139 ++++++++-------- ietf/submit/test_submission.xml | 149 ++++++++---------- ietf/submit/tests.py | 25 +-- ietf/submit/views.py | 4 +- ietf/templates/doc/document_draft.html | 30 ++-- .../doc/search/search_result_row.html | 7 +- .../doc/yang-check-modal-overlay.html | 17 ++ ietf/templates/submit/submission_status.html | 4 +- ietf/utils/admin.py | 5 + .../commands/populate_yang_model_dirs.py | 105 +++++++----- .../commands/run_yang_model_checks.py | 82 ++++++++++ .../commands/update_external_command_info.py | 37 +++++ ietf/utils/migrations/0003_versioninfo.py | 26 +++ ietf/utils/models.py | 9 ++ ietf/utils/resources.py | 21 ++- requirements.txt | 1 + 29 files changed, 676 insertions(+), 330 deletions(-) create mode 100644 ietf/templates/doc/yang-check-modal-overlay.html create mode 100644 ietf/utils/management/commands/run_yang_model_checks.py create mode 100644 ietf/utils/management/commands/update_external_command_info.py create mode 100644 ietf/utils/migrations/0003_versioninfo.py diff --git a/.gitignore b/.gitignore index 925b66eed..e44bf00fe 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ /static /testresult /unix.tag +/tmp-nomcom-public-keys-dir diff --git a/bin/daily b/bin/daily index f868e933d..3764e4a94 100755 --- a/bin/daily +++ b/bin/daily @@ -18,9 +18,16 @@ logger -p user.info -t cron "Running $DTDIR/bin/daily" # Set up the virtual environment source $DTDIR/env/bin/activate + +# Update our information about the current version of some commands we use +$DTDIR/ietf/manage.py update_external_command_info + # Populate the yang repositories $DTDIR/ietf/manage.py populate_yang_model_dirs +# Re-run yang checks on active documents +$DTDIR/ietf/manage.py run_yang_model_checks + # Expire internet drafts # Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner: $DTDIR/ietf/bin/expire-ids @@ -36,5 +43,3 @@ $DTDIR/ietf/bin/rfc-editor-index-updates -d 1969-01-01 # Fetch meeting attendance data from ietf.org/registration/attendees $DTDIR/ietf/manage.py fetch_meeting_attendance --latest - - diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index b71be5aa5..96162b164 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -533,3 +533,6 @@ def comma_separated_list(seq, end_word="and"): def role_names(roles): return list(set([ "%s %s" % (r.group.name, r.name.name) for r in roles ])) +@register.filter() +def zaptmp(s): + return re.sub(r'/tmp/tmp[^/]+/', '', s) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index d53515c53..55dbd5776 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1277,3 +1277,11 @@ def all_presentations(request, name): 'in_progress': in_progress, 'past' : past, }) + +def document_checks(request, name): + doc = get_object_or_404(Document, docalias__name=name, type_id='draft') + + checks = doc.submission.latest_checks() + debug.show('checks') + + return render(request, 'doc/document_checks.html', {'doc': doc, 'checks': checks}) diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 6a6cd71e2..87ffc206b 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -9708,5 +9708,38 @@ }, "model": "name.topicaudiencename", "pk": "nominees" + }, + { + "fields": { + "command": "xym", + "switch": "--version", + "time": "2017-06-15T06:24:58.869", + "used": true, + "version": "xym 0.3.2" + }, + "model": "utils.versioninfo", + "pk": 1 + }, + { + "fields": { + "command": "pyang", + "switch": "--version", + "time": "2017-06-15T06:24:59.516", + "used": true, + "version": "pyang 1.7.2" + }, + "model": "utils.versioninfo", + "pk": 2 + }, + { + "fields": { + "command": "yanglint", + "switch": "--version", + "time": "2017-06-15T06:24:59.531", + "used": true, + "version": "yanglint 0.12.183" + }, + "model": "utils.versioninfo", + "pk": 3 } ] diff --git a/ietf/name/generate_fixtures.py b/ietf/name/generate_fixtures.py index 753f6b8ad..f90abcbda 100644 --- a/ietf/name/generate_fixtures.py +++ b/ietf/name/generate_fixtures.py @@ -47,5 +47,8 @@ import ietf.mailtrigger.models objects += ietf.mailtrigger.models.Recipient.objects.all() objects += ietf.mailtrigger.models.MailTrigger.objects.all() +import ietf.utils.models +objects += ietf.utils.models.VersionInfo.objects.all() + output("names", objects) diff --git a/ietf/settings.py b/ietf/settings.py index 5493189fc..c50b6753c 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -660,11 +660,19 @@ IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_01 = 13 IDSUBMIT_DEFAULT_CUTOFF_TIME_UTC = datetime.timedelta(hours=23, minutes=59, seconds=59) IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21) +# 14 Jun 2017: New convention: prefix settings with the app name to which +# they (mainly) belong. So here, SUBMIT_, rather than IDSUBMIT_ +SUBMIT_YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/' +SUBMIT_YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/' +SUBMIT_YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/' + IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/' IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/' IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits' -IDSUBMIT_PYANG_COMMAND = 'pyang -p %(modpath)s --verbose --ietf %(model)s' +SUBMIT_PYANG_COMMAND = 'pyang --verbose --ietf -p {libs} {model}' +SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {rfclib} -p {draftlib} {model}' +SUBMIT_YANGLINT_COMMAND = None # use the value above if you have yanglint installed IDSUBMIT_CHECKER_CLASSES = ( "ietf.submit.checkers.DraftIdnitsChecker", @@ -696,10 +704,6 @@ IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE = 450 # in MB IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000 IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB -YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/' -YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/' -YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/' - XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/" # === Meeting Related Settings ================================================= diff --git a/ietf/static/ietf/css/ietf.css b/ietf/static/ietf/css/ietf.css index 96fb06b6d..a0e48e2d3 100644 --- a/ietf/static/ietf/css/ietf.css +++ b/ietf/static/ietf/css/ietf.css @@ -865,3 +865,9 @@ blockquote { #debug-query-table .code .current { background-color: #ddd; } + +.checker-warning, +.checker-success { + line-height: 1.0; + cursor: pointer; +} diff --git a/ietf/submit/admin.py b/ietf/submit/admin.py index 194c0d1e0..5c5b6ff95 100644 --- a/ietf/submit/admin.py +++ b/ietf/submit/admin.py @@ -33,7 +33,7 @@ class SubmissionEventAdmin(admin.ModelAdmin): admin.site.register(SubmissionEvent, SubmissionEventAdmin) class SubmissionCheckAdmin(admin.ModelAdmin): - list_display = ['submission', 'time', 'checker', 'passed', 'errors', 'warnings', 'items'] + list_display = ['submission', 'time', 'checker', 'passed', 'errors', 'warnings', 'message'] raw_id_fields = ['submission'] search_fields = ['submission__name'] admin.site.register(SubmissionCheck, SubmissionCheckAdmin) diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index d1a1ae495..f600f0441 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -2,16 +2,19 @@ import os import re +import sys from xym import xym import shutil import tempfile +import StringIO from django.conf import settings import debug # pyflakes:ignore -from ietf.utils.pipe import pipe from ietf.utils.log import log +from ietf.utils.models import VersionInfo +from ietf.utils.pipe import pipe class DraftSubmissionChecker(): name = "" @@ -47,6 +50,9 @@ class DraftIdnitsChecker(object): # start using this when we provide more in the way of warnings during # submission checking: # symbol = '' + # symbol = u'\ua17d' # Yi syllable 'nit' + # symbol = u'\ub2e1' # Hangul syllable 'nit' + symbol = "" def __init__(self, options=["--submitcheck", "--nitcount", ]): @@ -123,39 +129,66 @@ class DraftYangChecker(object): def check_file_txt(self, path): name = os.path.basename(path) workdir = tempfile.mkdtemp() - errors = [] - warnings = [] - results = {} + errors = 0 + warnings = 0 + message = "" + results = [] + passed = True # Used by the submission tool. Yang checks always pass. - extractor = xym.YangModuleExtractor(path, workdir, strict=True, debug_level = 0) + extractor = xym.YangModuleExtractor(path, workdir, strict=True, strict_examples=False, debug_level=0) if not os.path.exists(path): return None, "%s: No such file or directory: '%s'"%(name.capitalize(), path), errors, warnings, results with open(path) as file: + out = "" + err = "" + code = 0 try: # This places the yang models as files in workdir + saved_stdout = sys.stdout + saved_stderr = sys.stderr + sys.stdout = StringIO.StringIO() + sys.stderr = StringIO.StringIO() extractor.extract_yang_model(file.readlines()) + out = sys.stdout.getvalue() + err = sys.stderr.getvalue() + sys.stdout = saved_stdout + sys.stderr = saved_stderr model_list = extractor.get_extracted_models() except Exception as exc: - passed = False - message = exc - errors = [ (name, None, None, exc) ] - warnings = [] - return passed, message, errors, warnings + code = 1 + err = '\n'.join( [ m for m in [out, err, exc] if m ] ) + if err: + code += 1 + command = "xym" + cmd_version = VersionInfo.objects.get(command=command).version + message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err) + + results.append({ + "name": name, + "passed": passed, + "message": message, + "warnings": 0, + "errors": code, + "items": [], + }) for model in model_list: path = os.path.join(workdir, model) + message = "" modpath = ':'.join([ workdir, - settings.YANG_RFC_MODEL_DIR, - settings.YANG_DRAFT_MODEL_DIR, - settings.YANG_INVAL_MODEL_DIR, + settings.SUBMIT_YANG_RFC_MODEL_DIR, + settings.SUBMIT_YANG_DRAFT_MODEL_DIR, + settings.SUBMIT_YANG_INVAL_MODEL_DIR, ]) with open(path) as file: text = file.readlines() - cmd = settings.IDSUBMIT_PYANG_COMMAND % {"modpath": modpath, "model": path, } + # pyang + cmd_template = settings.SUBMIT_PYANG_COMMAND + command = cmd_template.split()[0] + cmd_version = VersionInfo.objects.get(command=command).version + cmd = cmd_template.format(libs=modpath, model=path) code, out, err = pipe(cmd) - errors = 0 - warnings = 0 items = [] if code > 0: error_lines = err.splitlines() @@ -175,26 +208,54 @@ class DraftYangChecker(object): warnings += 1 except ValueError: pass - results[model] = { - "passed": code == 0, - "message": out+"No validation errors\n" if code == 0 else err, + #passed = passed and code == 0 # For the submission tool. Yang checks always pass + message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if code == 0 else err) + + # yanglint + cmd_template = settings.SUBMIT_YANGLINT_COMMAND + command = cmd_template.split()[0] + cmd_version = VersionInfo.objects.get(command=command).version + cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR) + code, out, err = pipe(cmd) + if code > 0: + error_lines = err.splitlines() + for line in error_lines: + if line.strip(): + try: + if 'err : ' in line: + errors += 1 + if 'warn: ' in line: + warnings += 1 + except ValueError: + pass + #passed = passed and code == 0 # For the submission tool. Yang checks always pass + message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if code == 0 else err) + + if errors==0 and warnings==0: + dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model) + shutil.move(path, dest) + else: + dest = os.path.join(settings.SUBMIT_YANG_INVAL_MODEL_DIR, model) + shutil.move(path, dest) + + # summary result + results.append({ + "name": model, + "passed": passed, + "message": message, "warnings": warnings, "errors": errors, "items": items, - } + }) + shutil.rmtree(workdir) - ## For now, never fail because of failed yang validation. - if len(model_list): - passed = True - else: - passed = None - #passed = all( res["passed"] for res in results.values() ) - message = "\n\n".join([ "\n".join([model+':', res["message"]]) for model, res in results.items() ]) - errors = sum(res["errors"] for res in results.values() ) - warnings = sum(res["warnings"] for res in results.values() ) - items = [ e for res in results.values() for e in res["items"] ] + passed = all( res["passed"] for res in results ) + message = "\n".join([ "\n".join([res['name']+':', res["message"]]) for res in results ]) + errors = sum(res["errors"] for res in results ) + warnings = sum(res["warnings"] for res in results ) + items = [ e for res in results for e in res["items"] ] return passed, message, errors, warnings, items diff --git a/ietf/submit/migrations/0010_data_set_submission_check_symbol.py b/ietf/submit/migrations/0010_data_set_submission_check_symbol.py index 43ca7fc82..8bce3a132 100644 --- a/ietf/submit/migrations/0010_data_set_submission_check_symbol.py +++ b/ietf/submit/migrations/0010_data_set_submission_check_symbol.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- from __future__ import unicode_literals from tqdm import tqdm diff --git a/ietf/submit/models.py b/ietf/submit/models.py index 90a6a1378..82ca0e067 100644 --- a/ietf/submit/models.py +++ b/ietf/submit/models.py @@ -65,6 +65,10 @@ class Submission(models.Model): def existing_document(self): return Document.objects.filter(name=self.name).first() + def latest_checks(self): + checks = [ self.checks.filter(checker=c, passed__in=[True,False]).latest('time') for c in self.checks.values_list('checker', flat=True).distinct() ] + return checks + class SubmissionCheck(models.Model): time = models.DateTimeField(auto_now=True) submission = models.ForeignKey(Submission, related_name='checks') diff --git a/ietf/submit/test_submission.nonascii b/ietf/submit/test_submission.nonascii index 02c044aa3..b6c6b4fe9 100644 --- a/ietf/submit/test_submission.nonascii +++ b/ietf/submit/test_submission.nonascii @@ -71,99 +71,90 @@ Table of Contents 2. Yang - file "ietf-mpls@2015-10-16.yang" + file "ietf-yang-metadata@2016-08-05.yang" - module ietf-mpls { + module ietf-yang-metadata { - namespace "urn:ietf:params:xml:ns:yang:ietf-mpls"; + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata"; - prefix "mpls"; + prefix "md"; - import ietf-routing { - prefix "rt"; - } + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; - import ietf-interfaces { - prefix "if"; - } + contact + "WG Web: - organization "TBD"; + WG List: - contact "TBD"; + WG Chair: Lou Berger + + + WG Chair: Kent Watsen + + + Editor: Ladislav Lhotka + "; description - "This YANG module defines the essential components for the - management of the MPLS subsystem."; + "This YANG module defines an 'extension' statement that allows + for defining metadata annotations. - revision "2015-10-16" { + Copyright (c) 2016 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject to + the license terms contained in, the Simplified BSD License set + forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 7952 + (http://www.rfc-editor.org/info/rfc7952); see the RFC itself + for full legal notices."; + + revision 2016-08-05 { description - "Initial revision"; - reference "RFC 3031: A YANG Data Model for base MPLS"; + "Initial revision."; + reference + "RFC 7952: Defining and Using Metadata with YANG"; } - typedef mpls-label { - type uint32 { - range "0..1048575"; - } + extension annotation { + argument name; description - "The MPLS label range"; - } + "This extension allows for defining metadata annotations in + YANG modules. The 'md:annotation' statement can appear only + at the top level of a YANG module or submodule, i.e., it + becomes a new alternative in the ABNF production rule for + 'body-stmts' (Section 14 in RFC 7950). - typedef percent { - type uint16 { - range "0 .. 100"; - } - description "Percentage"; - } + The argument of the 'md:annotation' statement defines the name + of the annotation. Syntactically, it is a YANG identifier as + defined in Section 6.2 of RFC 7950. - grouping interface-mpls { - description "MPLS interface properties grouping"; - leaf enabled { - type boolean; - description - "'true' if mpls encapsulation is enabled on the - interface. 'false' if mpls encapsulation is enabled - on the interface."; - } - } + An annotation defined with this 'extension' statement inherits + the namespace and other context from the YANG module in which + it is defined. - augment "/rt:routing/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features config, e.g. MPLS static LSP, MPLS - LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc."; + The data type of the annotation value is specified in the same + way as for a leaf data node using the 'type' statement. - list interface { - key "name"; - description "List of MPLS interfaces"; - leaf name { - type if:interface-ref; - description - "The name of a configured MPLS interface"; - } - container config { - description "Holds intended configuration"; - uses interface-mpls; - } - container state { - config false; - description "Holds inuse configuration"; - uses interface-mpls; - } - } - } - } + The semantics of the annotation and other documentation can be + specified using the following standard YANG substatements (all + are optional): 'description', 'if-feature', 'reference', + 'status', and 'units'. - augment "/rt:routing-state/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - config false; - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features state"; - } + A server announces support for a particular annotation by + including the module in which the annotation is defined among + the advertised YANG modules, e.g., in a NETCONF + message or in the YANG library (RFC 7950). The annotation can + then be attached to any instance of a data node defined in any + YANG module that is advertised by the server. + + XML encoding and JSON encoding of annotations are defined in + RFC 7952."; } } diff --git a/ietf/submit/test_submission.txt b/ietf/submit/test_submission.txt index b6551bf2d..30e063c47 100644 --- a/ietf/submit/test_submission.txt +++ b/ietf/submit/test_submission.txt @@ -73,99 +73,90 @@ Table of Contents 2. Yang - file "ietf-mpls@2015-10-16.yang" + file "ietf-yang-metadata@2016-08-05.yang" - module ietf-mpls { + module ietf-yang-metadata { - namespace "urn:ietf:params:xml:ns:yang:ietf-mpls"; + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata"; - prefix "mpls"; + prefix "md"; - import ietf-routing { - prefix "rt"; - } + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; - import ietf-interfaces { - prefix "if"; - } + contact + "WG Web: - organization "TBD"; + WG List: - contact "TBD"; + WG Chair: Lou Berger + + + WG Chair: Kent Watsen + + + Editor: Ladislav Lhotka + "; description - "This YANG module defines the essential components for the - management of the MPLS subsystem."; + "This YANG module defines an 'extension' statement that allows + for defining metadata annotations. - revision "2015-10-16" { + Copyright (c) 2016 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject to + the license terms contained in, the Simplified BSD License set + forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). + + This version of this YANG module is part of RFC 7952 + (http://www.rfc-editor.org/info/rfc7952); see the RFC itself + for full legal notices."; + + revision 2016-08-05 { description - "Initial revision"; - reference "RFC 3031: A YANG Data Model for base MPLS"; + "Initial revision."; + reference + "RFC 7952: Defining and Using Metadata with YANG"; } - typedef mpls-label { - type uint32 { - range "0..1048575"; - } + extension annotation { + argument name; description - "The MPLS label range"; - } + "This extension allows for defining metadata annotations in + YANG modules. The 'md:annotation' statement can appear only + at the top level of a YANG module or submodule, i.e., it + becomes a new alternative in the ABNF production rule for + 'body-stmts' (Section 14 in RFC 7950). - typedef percent { - type uint16 { - range "0 .. 100"; - } - description "Percentage"; - } + The argument of the 'md:annotation' statement defines the name + of the annotation. Syntactically, it is a YANG identifier as + defined in Section 6.2 of RFC 7950. - grouping interface-mpls { - description "MPLS interface properties grouping"; - leaf enabled { - type boolean; - description - "'true' if mpls encapsulation is enabled on the - interface. 'false' if mpls encapsulation is enabled - on the interface."; - } - } + An annotation defined with this 'extension' statement inherits + the namespace and other context from the YANG module in which + it is defined. - augment "/rt:routing/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features config, e.g. MPLS static LSP, MPLS - LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc."; + The data type of the annotation value is specified in the same + way as for a leaf data node using the 'type' statement. - list interface { - key "name"; - description "List of MPLS interfaces"; - leaf name { - type if:interface-ref; - description - "The name of a configured MPLS interface"; - } - container config { - description "Holds intended configuration"; - uses interface-mpls; - } - container state { - config false; - description "Holds inuse configuration"; - uses interface-mpls; - } - } - } - } + The semantics of the annotation and other documentation can be + specified using the following standard YANG substatements (all + are optional): 'description', 'if-feature', 'reference', + 'status', and 'units'. - augment "/rt:routing-state/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - config false; - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features state"; - } + A server announces support for a particular annotation by + including the module in which the annotation is defined among + the advertised YANG modules, e.g., in a NETCONF + message or in the YANG library (RFC 7950). The annotation can + then be attached to any instance of a data node defined in any + YANG module that is advertised by the server. + + XML encoding and JSON encoding of annotations are defined in + RFC 7952."; } } diff --git a/ietf/submit/test_submission.xml b/ietf/submit/test_submission.xml index a8b6d9cde..4dbe41382 100644 --- a/ietf/submit/test_submission.xml +++ b/ietf/submit/test_submission.xml @@ -35,100 +35,91 @@
file "ietf-mpls@2015-10-16.yang" + file "ietf-yang-metadata@2016-08-05.yang" -module ietf-mpls { +module ietf-yang-metadata { - namespace "urn:ietf:params:xml:ns:yang:ietf-mpls"; + namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata"; - prefix "mpls"; + prefix "md"; - import ietf-routing { - prefix "rt"; - } + organization + "IETF NETMOD (NETCONF Data Modeling Language) Working Group"; - import ietf-interfaces { - prefix "if"; - } + contact + "WG Web: - organization "TBD"; + WG List: - contact "TBD"; + WG Chair: Lou Berger + - description - "This YANG module defines the essential components for the - management of the MPLS subsystem."; + WG Chair: Kent Watsen + - revision "2015-10-16" { - description - "Initial revision"; - reference "RFC 3031: A YANG Data Model for base MPLS"; - } + Editor: Ladislav Lhotka + "; - typedef mpls-label { - type uint32 { - range "0..1048575"; - } - description - "The MPLS label range"; - } + description + "This YANG module defines an 'extension' statement that allows + for defining metadata annotations. - typedef percent { - type uint16 { - range "0 .. 100"; - } - description "Percentage"; - } + Copyright (c) 2016 IETF Trust and the persons identified as + authors of the code. All rights reserved. - grouping interface-mpls { - description "MPLS interface properties grouping"; - leaf enabled { - type boolean; - description - "'true' if mpls encapsulation is enabled on the - interface. 'false' if mpls encapsulation is enabled - on the interface."; - } - } + Redistribution and use in source and binary forms, with or + without modification, is permitted pursuant to, and subject to + the license terms contained in, the Simplified BSD License set + forth in Section 4.c of the IETF Trust's Legal Provisions + Relating to IETF Documents + (http://trustee.ietf.org/license-info). - augment "/rt:routing/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features config, e.g. MPLS static LSP, MPLS - LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc."; + This version of this YANG module is part of RFC 7952 + (http://www.rfc-editor.org/info/rfc7952); see the RFC itself + for full legal notices."; - list interface { - key "name"; - description "List of MPLS interfaces"; - leaf name { - type if:interface-ref; - description - "The name of a configured MPLS interface"; - } - container config { - description "Holds intended configuration"; - uses interface-mpls; - } - container state { - config false; - description "Holds inuse configuration"; - uses interface-mpls; - } - } - } - } + revision 2016-08-05 { + description + "Initial revision."; + reference + "RFC 7952: Defining and Using Metadata with YANG"; + } - augment "/rt:routing-state/rt:routing-instance" { - description "MPLS augmentation."; - container mpls { - config false; - description - "MPLS container, to be used as an augmentation target node - other MPLS sub-features state"; - } - } + extension annotation { + argument name; + description + "This extension allows for defining metadata annotations in + YANG modules. The 'md:annotation' statement can appear only + at the top level of a YANG module or submodule, i.e., it + becomes a new alternative in the ABNF production rule for + 'body-stmts' (Section 14 in RFC 7950). + + The argument of the 'md:annotation' statement defines the name + of the annotation. Syntactically, it is a YANG identifier as + defined in Section 6.2 of RFC 7950. + + An annotation defined with this 'extension' statement inherits + the namespace and other context from the YANG module in which + it is defined. + + The data type of the annotation value is specified in the same + way as for a leaf data node using the 'type' statement. + + The semantics of the annotation and other documentation can be + specified using the following standard YANG substatements (all + are optional): 'description', 'if-feature', 'reference', + 'status', and 'units'. + + A server announces support for a particular annotation by + including the module in which the annotation is defined among + the advertised YANG modules, e.g., in a NETCONF + message or in the YANG library (RFC 7950). The annotation can + then be attached to any instance of a data node defined in any + YANG module that is advertised by the server. + + XML encoding and JSON encoding of annotations are defined in + RFC 7952."; + } } diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index ce93184bb..64657d539 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -68,17 +68,17 @@ class SubmitTests(TestCase): self.archive_dir = self.tempdir('submit-archive') settings.INTERNET_DRAFT_ARCHIVE_DIR = self.archive_dir - self.saved_yang_rfc_model_dir = settings.YANG_RFC_MODEL_DIR + self.saved_yang_rfc_model_dir = settings.SUBMIT_YANG_RFC_MODEL_DIR self.yang_rfc_model_dir = self.tempdir('yang-rfc-model') - settings.YANG_RFC_MODEL_DIR = self.yang_rfc_model_dir + settings.SUBMIT_YANG_RFC_MODEL_DIR = self.yang_rfc_model_dir - self.saved_yang_draft_model_dir = settings.YANG_DRAFT_MODEL_DIR + self.saved_yang_draft_model_dir = settings.SUBMIT_YANG_DRAFT_MODEL_DIR self.yang_draft_model_dir = self.tempdir('yang-draft-model') - settings.YANG_DRAFT_MODEL_DIR = self.yang_draft_model_dir + settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.yang_draft_model_dir - self.saved_yang_inval_model_dir = settings.YANG_INVAL_MODEL_DIR + self.saved_yang_inval_model_dir = settings.SUBMIT_YANG_INVAL_MODEL_DIR self.yang_inval_model_dir = self.tempdir('yang-inval-model') - settings.YANG_INVAL_MODEL_DIR = self.yang_inval_model_dir + settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.yang_inval_model_dir def tearDown(self): shutil.rmtree(self.staging_dir) @@ -91,9 +91,9 @@ class SubmitTests(TestCase): settings.INTERNET_DRAFT_PATH = self.saved_internet_draft_path settings.IDSUBMIT_REPOSITORY_PATH = self.saved_idsubmit_repository_path settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir - settings.YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir - settings.YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir - settings.YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir + settings.SUBMIT_YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir + settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir + settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir def do_submission(self, name, rev, group=None, formats=["txt",]): @@ -133,6 +133,8 @@ class SubmitTests(TestCase): self.assertEqual(author["affiliation"], "Test Centre Inc.") self.assertEqual(author["country"], "UK") + + return status_url def supply_extra_metadata(self, name, status_url, submitter_name, submitter_email, replaces): @@ -220,6 +222,11 @@ class SubmitTests(TestCase): r = self.client.get(status_url) self.assertEqual(r.status_code, 200) + + self.assertContains(r, 'xym') + self.assertContains(r, 'pyang') + self.assertContains(r, 'yanglint') + q = PyQuery(r.content) approve_button = q('[type=submit]:contains("Approve")') self.assertEqual(len(approve_button), 1) diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 743760116..23f122589 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -165,7 +165,9 @@ def upload_submission(request): def apply_check(submission, checker, method, fn): func = getattr(checker, method) passed, message, errors, warnings, items = func(fn) - check = SubmissionCheck(submission=submission, checker=checker.name, passed=passed, message=message, errors=errors, warnings=warnings, items=items, symbol=checker.symbol) + check = SubmissionCheck(submission=submission, checker=checker.name, passed=passed, + message=message, errors=errors, warnings=warnings, items=items, + symbol=checker.symbol) check.save() for checker_path in settings.IDSUBMIT_CHECKER_CLASSES: diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index 870e188a5..4925d992d 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -76,15 +76,6 @@ {% if latest_revision and latest_revision.time.date != doc.time.date %} (latest revision {{ latest_revision.time|date:"Y-m-d" }}) {% endif %} - {% for check in doc.submission.checks.all %} - {% if check.passed != None and check.symbol.strip %} - {% if check.errors or check.warnings %} - {{ check.symbol|safe }} - {% else %} - {{ check.symbol|safe }} - {% endif %} - {% endif %} - {% endfor %} @@ -192,6 +183,27 @@ + {% for check in doc.submission.latest_checks %} + {% if check.passed != None and check.symbol.strip %} + + + {{ check.checker|title }} + + + {% if check.errors or check.warnings %} + {{ check.symbol|safe }} + {% else %} + {{ check.symbol|safe }} + {% endif %} + + {{ check.errors }} errors, {{ check.warnings }} warnings. + + {% include "doc/yang-check-modal-overlay.html" %} + + + {% endif %} + {% endfor %} + {% if review_requests or can_request_review %} diff --git a/ietf/templates/doc/search/search_result_row.html b/ietf/templates/doc/search/search_result_row.html index 1ada48b26..f2adcb93f 100644 --- a/ietf/templates/doc/search/search_result_row.html +++ b/ietf/templates/doc/search/search_result_row.html @@ -56,13 +56,14 @@ {% if doc.latest_revision_date|timesince_days|new_enough:request and doc.get_state_slug != "rfc" %}{% endif %} - {% for check in doc.submission.checks.all %} + {% for check in doc.submission.latest_checks %} {% if check.passed != None and check.symbol.strip %} {% if check.errors or check.warnings %} - {{ check.symbol|safe }} + {{ check.symbol|safe }} {% else %} - {{ check.symbol|safe }} + {{ check.symbol|safe }} {% endif %} + {% include "doc/yang-check-modal-overlay.html" %} {% endif %} {% endfor %} diff --git a/ietf/templates/doc/yang-check-modal-overlay.html b/ietf/templates/doc/yang-check-modal-overlay.html new file mode 100644 index 000000000..10b3bfb36 --- /dev/null +++ b/ietf/templates/doc/yang-check-modal-overlay.html @@ -0,0 +1,17 @@ + {% load ietf_filters %}{% load origin %}{% origin %} + diff --git a/ietf/templates/submit/submission_status.html b/ietf/templates/submit/submission_status.html index 473829130..7a39804f2 100644 --- a/ietf/templates/submit/submission_status.html +++ b/ietf/templates/submit/submission_status.html @@ -44,7 +44,7 @@ Your draft has NOT been verified to pass the submission checks. {% endif %}

- {% for check in submission.checks.all %} + {% for check in submission.latest_checks %} {% if check.errors %}

The {{check.checker}} returned {{ check.errors }} error{{ check.errors|pluralize }} @@ -58,7 +58,7 @@ {% endif %} {% endfor %} - {% for check in submission.checks.all %} + {% for check in submission.latest_checks %} {% if check.passed != None %} diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index bfce0b08a..73e166409 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -1,3 +1,5 @@ +from django.contrib import admin +from ietf.utils.models import VersionInfo def name(obj): if hasattr(obj, 'abbrev'): @@ -45,3 +47,6 @@ def admin_link(field, label=None, ordering="", display=name, suffix=""): _link.admin_order_field = ordering return _link +class VersionInfoAdmin(admin.ModelAdmin): + list_display = ['command', 'switch', 'version', 'time', ] +admin.site.register(VersionInfo, VersionInfoAdmin) diff --git a/ietf/utils/management/commands/populate_yang_model_dirs.py b/ietf/utils/management/commands/populate_yang_model_dirs.py index 5025f5a8e..8aabd56fe 100644 --- a/ietf/utils/management/commands/populate_yang_model_dirs.py +++ b/ietf/utils/management/commands/populate_yang_model_dirs.py @@ -1,8 +1,10 @@ +from __future__ import print_function, unicode_literals + import os import sys import time -from pathlib import Path +from pathlib2 import Path from StringIO import StringIO from textwrap import dedent from xym import xym @@ -10,9 +12,11 @@ from xym import xym from django.conf import settings from django.core.management.base import BaseCommand +import debug # pyflakes:ignore + class Command(BaseCommand): """ - Populate the yang models repositories from drafts and RFCs. + Populate the yang module repositories from drafts and RFCs. Extracts yang models from RFCs (found in settings.RFC_PATH and places them in settings.YANG_RFC_MODEL_DIR, and from active drafts, placed in @@ -57,22 +61,34 @@ class Command(BaseCommand): verbosity = int(options.get('verbosity')) def extract_from(file, dir, strict=True): + saved_stdout = sys.stdout saved_stderr = sys.stderr - sys.stderr = StringIO() + xymerr = StringIO() + xymout = StringIO() + sys.stderr = xymerr + sys.stdout = xymout model_list = [] try: - model_list = xym.xym(str(item), str(item.parent), str(dir), strict=strict, debug_level=(verbosity>1)) + model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=verbosity-2) for name in model_list: modfile = moddir / name - mtime = item.stat().st_mtime + mtime = file.stat().st_mtime os.utime(str(modfile), (mtime, mtime)) if '"' in name: name = name.replace('"', '') modfile.rename(str(moddir/name)) - model_list = [ n.replace('"','') for n in model_list ] + model_list = [ n.replace('"','') for n in model_list ] except Exception as e: - print("** Error when extracting from %s: %s" % (item, str(e))) + print("** Error when extracting from %s: %s" % (file, str(e))) + sys.stdout = saved_stdout sys.stderr = saved_stderr + # + if verbosity > 1: + outmsg = xymout.getvalue() + self.stdout.write(outmsg) + if verbosity>2: + errmsg = xymerr.getvalue() + self.stderr.write(errmsg) return model_list # Extract from new RFCs @@ -87,10 +103,10 @@ class Command(BaseCommand): for item in moddir.iterdir(): if item.stat().st_mtime > latest: latest = item.stat().st_mtime - + print("Extracting to %s ..." % moddir) for item in rfcdir.iterdir(): - if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt'): + if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit(): if item.stat().st_mtime > latest: model_list = extract_from(item, moddir) for name in model_list: @@ -124,20 +140,25 @@ class Command(BaseCommand): print("Extracting to %s ..." % moddir) for item in draftdir.iterdir(): - if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): - model_list = extract_from(item, moddir) - for name in model_list: - if not name.startswith('example'): - if verbosity > 1: - print(" Extracted valid module from %s: %s" % (item, name)) + try: + if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): + model_list = extract_from(item, moddir) + for name in model_list: + if not name.startswith('example'): + if verbosity > 1: + print(" Extracted valid module from %s: %s" % (item, name)) + else: + sys.stdout.write('.') + sys.stdout.flush() else: - sys.stdout.write('.') - sys.stdout.flush() - else: - modfile = moddir / name - modfile.unlink() - if verbosity > 1: - print(" Skipped module from %s: %s" % (item, name)) + modfile = moddir / name + modfile.unlink() + if verbosity > 1: + print(" Skipped module from %s: %s" % (item, name)) + except UnicodeDecodeError as e: + sys.stderr.write('\nError: %s\n' % (e, )) + sys.stderr.write(item.name) + sys.stderr.write('\n') print("") # Extract invalid modules from drafts @@ -151,22 +172,28 @@ class Command(BaseCommand): print("Extracting to %s ..." % moddir) for item in draftdir.iterdir(): - if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): - model_list = extract_from(item, moddir, strict=False) - for name in model_list: - modfile = moddir / name - if (valdir/name).exists(): - modfile.unlink() - if verbosity > 1: - print(" Skipped valid module from %s: %s" % (item, name)) - elif not name.startswith('example'): - if verbosity > 1: - print(" Extracted invalid module from %s: %s" % (item, name)) + try: + if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): + model_list = extract_from(item, moddir, strict=False) + for name in model_list: + modfile = moddir / name + if (valdir/name).exists(): + modfile.unlink() + if verbosity > 1: + print(" Skipped valid module from %s: %s" % (item, name)) + elif not name.startswith('example'): + if verbosity > 1: + print(" Extracted invalid module from %s: %s" % (item, name)) + else: + sys.stdout.write('.') + sys.stdout.flush() else: - sys.stdout.write('.') - sys.stdout.flush() - else: - modfile.unlink() - if verbosity > 1: - print(" Skipped module from %s: %s" % (item, name)) + modfile.unlink() + if verbosity > 1: + print(" Skipped module from %s: %s" % (item, name)) + except UnicodeDecodeError as e: + sys.stderr.write('\nError: %s\n' % (e, )) + sys.stderr.write(item.name) + sys.stderr.write('\n') + print("") diff --git a/ietf/utils/management/commands/run_yang_model_checks.py b/ietf/utils/management/commands/run_yang_model_checks.py new file mode 100644 index 000000000..4d44c145b --- /dev/null +++ b/ietf/utils/management/commands/run_yang_model_checks.py @@ -0,0 +1,82 @@ +from __future__ import print_function, unicode_literals + +import sys +import json + +from textwrap import dedent + +from django.core.management.base import BaseCommand + +import debug # pyflakes:ignore + +from ietf.doc.models import Document, State +from ietf.submit.models import Submission, SubmissionCheck +from ietf.submit.checkers import DraftYangChecker + + +class Command(BaseCommand): + """ + Run yang model checks on active drafts. + + Repeats the yang checks in ietf/submit/checkers.py for active drafts, in + order to catch changes in status due to new modules becoming available in + the module directories. + + """ + + help = dedent(__doc__).strip() + + def add_arguments(self, parser): + parser.add_argument('filenames', nargs="*") + parser.add_argument('--clean', + action='store_true', dest='clean', default=False, + help='Remove the current directory content before writing new models.') + + + def check_yang(self, checker, draft, force=False): + if self.verbosity > 1: + self.stdout.write("Checking %s-%s" % (draft.name, draft.rev)) + else: + sys.stderr.write('.') + submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first() + if submission or force: + check = submission.checks.filter(checker=checker.name).order_by('-id').first() + if check or force: + result = checker.check_file_txt(draft.get_file_name()) + passed, message, errors, warnings, items = result + if self.verbosity > 2: + self.stdout.write(" Errors: %s\n" + " Warnings: %s\n" + " Message:\n%s\n" % (errors, warnings, message)) + items = json.loads(json.dumps(items)) + new_res = (passed, errors, warnings, message) + old_res = (check.passed, check.errors, check.warnings, check.message) if check else () + if new_res != old_res: + if self.verbosity > 1: + self.stdout.write(" Saving new yang checker results for %s-%s" % (draft.name, draft.rev)) + SubmissionCheck.objects.create(submission=submission, checker=checker.name, passed=passed, + message=message, errors=errors, warnings=warnings, items=items, + symbol=checker.symbol) + else: + self.stderr.write("Error: did not find any submission object for %s-%s\n" % (draft.name, draft.rev)) + + def handle(self, *filenames, **options): + """ + """ + + self.verbosity = int(options.get('verbosity')) + filenames = options.get('filenames') + + active_state = State.objects.get(type="draft", slug="active") + + checker = DraftYangChecker() + if filenames: + for name in filenames: + parts = name.rsplit('-',1) + if len(parts)==2 and len(parts[1])==2 and parts[1].isdigit(): + name = parts[0] + draft = Document.objects.get(name=name) + self.check_yang(checker, draft, force=True) + else: + for draft in Document.objects.filter(states=active_state, type_id='draft'): + self.check_yang(checker, draft) diff --git a/ietf/utils/management/commands/update_external_command_info.py b/ietf/utils/management/commands/update_external_command_info.py new file mode 100644 index 000000000..04949951c --- /dev/null +++ b/ietf/utils/management/commands/update_external_command_info.py @@ -0,0 +1,37 @@ +from __future__ import print_function, unicode_literals + +import sys + +from textwrap import dedent + +from django.core.management.base import BaseCommand + +import debug # pyflakes:ignore + +from ietf.utils.models import VersionInfo +from ietf.utils.pipe import pipe + +class Command(BaseCommand): + """ + Update the version information for external commands used by the datatracker. + + Iterates through the entries in the VersionInfo table, runs the relevant + command, and updates the version string with the result. + + """ + + help = dedent(__doc__).strip() + + def handle(self, *filenames, **options): + for c in VersionInfo.objects.filter(used=True): + cmd = "%s %s" % (c.command, c.switch) + code, out, err = pipe(cmd) + if code != 0: + sys.stderr.write("Command '%s' retuned %s: \n%s\n%s\n" % (cmd, code, out, err)) + else: + c.version = (out.strip()+'\n'+err.strip()).strip() + if options.get('verbosity', 1) > 1: + sys.stdout.write( + "Command: %s\n" + " Version: %s\n" % (cmd, c.version)) + c.save() diff --git a/ietf/utils/migrations/0003_versioninfo.py b/ietf/utils/migrations/0003_versioninfo.py new file mode 100644 index 000000000..0c1dca17a --- /dev/null +++ b/ietf/utils/migrations/0003_versioninfo.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.7 on 2017-06-15 03:31 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('utils', '0002_dumpinfo_tz'), + ] + + operations = [ + migrations.CreateModel( + name='VersionInfo', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('time', models.DateTimeField(auto_now=True)), + ('command', models.CharField(max_length=32)), + ('switch', models.CharField(max_length=16)), + ('version', models.CharField(max_length=64)), + ('used', models.BooleanField(default=True)), + ], + ), + ] diff --git a/ietf/utils/models.py b/ietf/utils/models.py index a38d1057b..c176b3cab 100644 --- a/ietf/utils/models.py +++ b/ietf/utils/models.py @@ -7,3 +7,12 @@ class DumpInfo(models.Model): host = models.CharField(max_length=128) tz = models.CharField(max_length=32, default='UTC') +class VersionInfo(models.Model): + time = models.DateTimeField(auto_now=True) + command = models.CharField(max_length=32) + switch = models.CharField(max_length=16) + version = models.CharField(max_length=64) + used = models.BooleanField(default=True) + class Meta: + verbose_name_plural = 'VersionInfo' + diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index 4f27081a8..a35e972f0 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -8,7 +8,8 @@ from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from ietf import api -from ietf.utils.models import DumpInfo +from ietf.utils.models import DumpInfo, VersionInfo + class UserResource(ModelResource): username = CharField() @@ -17,6 +18,7 @@ class UserResource(ModelResource): queryset = User.objects.all() serializer = api.Serializer() + class ContentTypeResource(ModelResource): username = CharField() class Meta: @@ -24,6 +26,7 @@ class ContentTypeResource(ModelResource): queryset = ContentType.objects.all() serializer = api.Serializer() + class DumpInfoResource(ModelResource): class Meta: cache = SimpleCache() @@ -36,3 +39,19 @@ class DumpInfoResource(ModelResource): } api.utils.register(DumpInfoResource()) + +class VersionInfoResource(ModelResource): + class Meta: + queryset = VersionInfo.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'versioninfo' + filtering = { + "id": ALL, + "time": ALL, + "command": ALL, + "switch": ALL, + "version": ALL, + "used": ALL, + } +api.utils.register(VersionInfoResource()) diff --git a/requirements.txt b/requirements.txt index ad419e823..f6c2e25ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,6 +31,7 @@ mimeparse>=0.1.3 # from TastyPie mock>=2.0.0 MySQL-python>=1.2.5 pathlib>=1.0 +pathlib2>=2.3.0 Pillow>=3.0 pyang>=1.6 pyflakes>=0.8.1