altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

Added a new yang checker, 'yanglint', to the existing Yang checker class, in

Browse source 
addition to the existing 'pyang' checker.

Added modal overlay displays showing the yang check results every place the
yin/yang symbol is shown (red or green) to indicate the presencee and result
of yang checks.  Added a Yang Validation: line in the document
meta-information section on the document's page in the datatracker.

Added the result of the xym extaction to the yang check results, to make
extration failures visible.

Added the version of the used xym, pyang, and yanglint commands to the check
results.

Added an action to move successfully extracted and validated modules to the
module library directories immediately on submission.

Added the xym and pyang repositories as svn:external components, rather than
listing them in requirements.txt, as there has been delays of many months
between essential features in the repositories, and an actual release.  We may
get occasional buildbot failures if broken code is pulled in from the
repository, but better that than the functionality failure of severely
outdated componets.

Added a new management command to re-run yang validation for active drafts for
which yang modules were found at submission time, in order to pick up imported
models which may have arrived in the model libraries after the draft's
submission.  Run daily from bin/daily.

Added a table to hold version information for external commands.  The yang
checker output should include the version information of the used checkers,
but seems unnecessary to run each command with its --version switch every
time we check a module...

Added a new management command to collect version information for external
commands on demand.  To be run daily from bin/daily.

Added tests to verify that xym, pyang and yanglint information is available
on the submission confirmation page, and updated the yang module contained in
the test document to validate under both pyang and yanglint.

Updated admin.py and resource.py files as needed.
 - Legacy-Id: 13630
This commit is contained in:
Henrik Levkowetz 2017-06-15 16:09:28 +00:00
parent c53e3784c3
commit d98054c103
29 changed files with 676 additions and 330 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -39,3 +39,4 @@
/static
/testresult
/unix.tag
/tmp-nomcom-public-keys-dir

9
bin/daily
View file

@ -18,9 +18,16 @@ logger -p user.info -t cron "Running $DTDIR/bin/daily"
# Set up the virtual environment
source $DTDIR/env/bin/activate
# Update our information about the current version of some commands we use
$DTDIR/ietf/manage.py update_external_command_info
# Populate the yang repositories
$DTDIR/ietf/manage.py populate_yang_model_dirs
# Re-run yang checks on active documents
$DTDIR/ietf/manage.py run_yang_model_checks
# Expire internet drafts
# Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner:
$DTDIR/ietf/bin/expire-ids
@ -36,5 +43,3 @@ $DTDIR/ietf/bin/rfc-editor-index-updates -d 1969-01-01
# Fetch meeting attendance data from ietf.org/registration/attendees
$DTDIR/ietf/manage.py fetch_meeting_attendance --latest

3
ietf/doc/templatetags/ietf_filters.py
View file

@ -533,3 +533,6 @@ def comma_separated_list(seq, end_word="and"):
def role_names(roles):
return list(set([ "%s %s" % (r.group.name, r.name.name) for r in roles ]))
@register.filter()
def zaptmp(s):
return re.sub(r'/tmp/tmp[^/]+/', '', s)

8
ietf/doc/views_doc.py
View file

@ -1277,3 +1277,11 @@ def all_presentations(request, name):
'in_progress': in_progress,
'past' : past,
})
def document_checks(request, name):
doc = get_object_or_404(Document, docalias__name=name, type_id='draft')
checks = doc.submission.latest_checks()
debug.show('checks')
return render(request, 'doc/document_checks.html', {'doc': doc, 'checks': checks})

33
ietf/name/fixtures/names.json
View file

@ -9708,5 +9708,38 @@
},
"model": "name.topicaudiencename",
"pk": "nominees"
},
{
"fields": {
"command": "xym",
"switch": "--version",
"time": "2017-06-15T06:24:58.869",
"used": true,
"version": "xym 0.3.2"
},
"model": "utils.versioninfo",
"pk": 1
},
{
"fields": {
"command": "pyang",
"switch": "--version",
"time": "2017-06-15T06:24:59.516",
"used": true,
"version": "pyang 1.7.2"
},
"model": "utils.versioninfo",
"pk": 2
},
{
"fields": {
"command": "yanglint",
"switch": "--version",
"time": "2017-06-15T06:24:59.531",
"used": true,
"version": "yanglint 0.12.183"
},
"model": "utils.versioninfo",
"pk": 3
}
]

3
ietf/name/generate_fixtures.py
View file

@ -47,5 +47,8 @@ import ietf.mailtrigger.models
objects += ietf.mailtrigger.models.Recipient.objects.all()
objects += ietf.mailtrigger.models.MailTrigger.objects.all()
import ietf.utils.models
objects += ietf.utils.models.VersionInfo.objects.all()
output("names", objects)

14
ietf/settings.py
View file

@ -660,11 +660,19 @@ IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_01 = 13
IDSUBMIT_DEFAULT_CUTOFF_TIME_UTC = datetime.timedelta(hours=23, minutes=59, seconds=59)
IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21)
# 14 Jun 2017: New convention: prefix settings with the app name to which
# they (mainly) belong. So here, SUBMIT_, rather than IDSUBMIT_
SUBMIT_YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
SUBMIT_YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
SUBMIT_YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/'
IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
IDSUBMIT_PYANG_COMMAND = 'pyang -p %(modpath)s --verbose --ietf %(model)s'
SUBMIT_PYANG_COMMAND = 'pyang --verbose --ietf -p {libs} {model}'
SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {rfclib} -p {draftlib} {model}'
SUBMIT_YANGLINT_COMMAND = None # use the value above if you have yanglint installed
IDSUBMIT_CHECKER_CLASSES = (
"ietf.submit.checkers.DraftIdnitsChecker",
@ -696,10 +704,6 @@ IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE = 450 # in MB
IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000
IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB
YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/'
XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/"
# === Meeting Related Settings =================================================

6
ietf/static/ietf/css/ietf.css
View file

@ -865,3 +865,9 @@ blockquote {
#debug-query-table .code .current {
background-color: #ddd;
}
.checker-warning,
.checker-success {
line-height: 1.0;
cursor: pointer;
}

2
ietf/submit/admin.py
View file

@ -33,7 +33,7 @@ class SubmissionEventAdmin(admin.ModelAdmin):
admin.site.register(SubmissionEvent, SubmissionEventAdmin)
class SubmissionCheckAdmin(admin.ModelAdmin):
list_display = ['submission', 'time', 'checker', 'passed', 'errors', 'warnings', 'items']
list_display = ['submission', 'time', 'checker', 'passed', 'errors', 'warnings', 'message']
raw_id_fields = ['submission']
search_fields = ['submission__name']
admin.site.register(SubmissionCheck, SubmissionCheckAdmin)

121
ietf/submit/checkers.py
View file

@ -2,16 +2,19 @@
import os
import re
import sys
from xym import xym
import shutil
import tempfile
import StringIO
from django.conf import settings
import debug # pyflakes:ignore
from ietf.utils.pipe import pipe
from ietf.utils.log import log
from ietf.utils.models import VersionInfo
from ietf.utils.pipe import pipe
class DraftSubmissionChecker():
name = ""
@ -47,6 +50,9 @@ class DraftIdnitsChecker(object):
# start using this when we provide more in the way of warnings during
# submission checking:
# symbol = '<span class="fa fa-check-square"></span>'
# symbol = u'<span class="large">\ua17d</span>' # Yi syllable 'nit'
# symbol = u'<span class="large">\ub2e1</span>' # Hangul syllable 'nit'
symbol = ""
def __init__(self, options=["--submitcheck", "--nitcount", ]):
@ -123,39 +129,66 @@ class DraftYangChecker(object):
def check_file_txt(self, path):
name = os.path.basename(path)
workdir = tempfile.mkdtemp()
errors = []
warnings = []
results = {}
errors = 0
warnings = 0
message = ""
results = []
passed = True # Used by the submission tool. Yang checks always pass.
extractor = xym.YangModuleExtractor(path, workdir, strict=True, debug_level = 0)
extractor = xym.YangModuleExtractor(path, workdir, strict=True, strict_examples=False, debug_level=0)
if not os.path.exists(path):
return None, "%s: No such file or directory: '%s'"%(name.capitalize(), path), errors, warnings, results
with open(path) as file:
out = ""
err = ""
code = 0
try:
# This places the yang models as files in workdir
saved_stdout = sys.stdout
saved_stderr = sys.stderr
sys.stdout = StringIO.StringIO()
sys.stderr = StringIO.StringIO()
extractor.extract_yang_model(file.readlines())
out = sys.stdout.getvalue()
err = sys.stderr.getvalue()
sys.stdout = saved_stdout
sys.stderr = saved_stderr
model_list = extractor.get_extracted_models()
except Exception as exc:
passed = False
message = exc
errors = [ (name, None, None, exc) ]
warnings = []
return passed, message, errors, warnings
code = 1
err = '\n'.join( [ m for m in [out, err, exc] if m ] )
if err:
code += 1
command = "xym"
cmd_version = VersionInfo.objects.get(command=command).version
message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err)
results.append({
"name": name,
"passed": passed,
"message": message,
"warnings": 0,
"errors": code,
"items": [],
})
for model in model_list:
path = os.path.join(workdir, model)
message = ""
modpath = ':'.join([
workdir,
settings.YANG_RFC_MODEL_DIR,
settings.YANG_DRAFT_MODEL_DIR,
settings.YANG_INVAL_MODEL_DIR,
settings.SUBMIT_YANG_RFC_MODEL_DIR,
settings.SUBMIT_YANG_DRAFT_MODEL_DIR,
settings.SUBMIT_YANG_INVAL_MODEL_DIR,
])
with open(path) as file:
text = file.readlines()
cmd = settings.IDSUBMIT_PYANG_COMMAND % {"modpath": modpath, "model": path, }
# pyang
cmd_template = settings.SUBMIT_PYANG_COMMAND
command = cmd_template.split()[0]
cmd_version = VersionInfo.objects.get(command=command).version
cmd = cmd_template.format(libs=modpath, model=path)
code, out, err = pipe(cmd)
errors = 0
warnings = 0
items = []
if code > 0:
error_lines = err.splitlines()
@ -175,26 +208,54 @@ class DraftYangChecker(object):
warnings += 1
except ValueError:
pass
results[model] = {
"passed": code == 0,
"message": out+"No validation errors\n" if code == 0 else err,
#passed = passed and code == 0 # For the submission tool. Yang checks always pass
message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if code == 0 else err)
# yanglint
cmd_template = settings.SUBMIT_YANGLINT_COMMAND
command = cmd_template.split()[0]
cmd_version = VersionInfo.objects.get(command=command).version
cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
code, out, err = pipe(cmd)
if code > 0:
error_lines = err.splitlines()
for line in error_lines:
if line.strip():
try:
if 'err : ' in line:
errors += 1
if 'warn: ' in line:
warnings += 1
except ValueError:
pass
#passed = passed and code == 0 # For the submission tool. Yang checks always pass
message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if code == 0 else err)
if errors==0 and warnings==0:
dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model)
shutil.move(path, dest)
else:
dest = os.path.join(settings.SUBMIT_YANG_INVAL_MODEL_DIR, model)
shutil.move(path, dest)
# summary result
results.append({
"name": model,
"passed": passed,
"message": message,
"warnings": warnings,
"errors": errors,
"items": items,
}
})
shutil.rmtree(workdir)
## For now, never fail because of failed yang validation.
if len(model_list):
passed = True
else:
passed = None
#passed = all( res["passed"] for res in results.values() )
message = "\n\n".join([ "\n".join([model+':', res["message"]]) for model, res in results.items() ])
errors = sum(res["errors"] for res in results.values() )
warnings = sum(res["warnings"] for res in results.values() )
items = [ e for res in results.values() for e in res["items"] ]
passed = all( res["passed"] for res in results )
message = "\n".join([ "\n".join([res['name']+':', res["message"]]) for res in results ])
errors = sum(res["errors"] for res in results )
warnings = sum(res["warnings"] for res in results )
items = [ e for res in results for e in res["items"] ]
return passed, message, errors, warnings, items

2
ietf/submit/migrations/0010_data_set_submission_check_symbol.py
View file

@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from tqdm import tqdm

4
ietf/submit/models.py
View file

@ -65,6 +65,10 @@ class Submission(models.Model):
def existing_document(self):
return Document.objects.filter(name=self.name).first()
def latest_checks(self):
checks = [ self.checks.filter(checker=c, passed__in=[True,False]).latest('time') for c in self.checks.values_list('checker', flat=True).distinct() ]
return checks
class SubmissionCheck(models.Model):
time = models.DateTimeField(auto_now=True)
submission = models.ForeignKey(Submission, related_name='checks')

139
ietf/submit/test_submission.nonascii
View file

@ -71,99 +71,90 @@ Table of Contents
2. Yang
<CODE BEGINS> file "ietf-mpls@2015-10-16.yang"
<CODE BEGINS> file "ietf-yang-metadata@2016-08-05.yang"
module ietf-mpls {
module ietf-yang-metadata {
namespace "urn:ietf:params:xml:ns:yang:ietf-mpls";
namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata";
prefix "mpls";
prefix "md";
import ietf-routing {
prefix "rt";
}
organization
"IETF NETMOD (NETCONF Data Modeling Language) Working Group";
import ietf-interfaces {
prefix "if";
}
contact
"WG Web: <https://datatracker.ietf.org/wg/netmod/>
organization "TBD";
WG List: <mailto:netmod@ietf.org>
contact "TBD";
WG Chair: Lou Berger
<mailto:lberger@labn.net>
WG Chair: Kent Watsen
<mailto:kwatsen@juniper.net>
Editor: Ladislav Lhotka
<mailto:lhotka@nic.cz>";
description
"This YANG module defines the essential components for the
management of the MPLS subsystem.";
"This YANG module defines an 'extension' statement that allows
for defining metadata annotations.
revision "2015-10-16" {
Copyright (c) 2016 IETF Trust and the persons identified as
authors of the code. All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, is permitted pursuant to, and subject to
the license terms contained in, the Simplified BSD License set
forth in Section 4.c of the IETF Trust's Legal Provisions
Relating to IETF Documents
(http://trustee.ietf.org/license-info).
This version of this YANG module is part of RFC 7952
(http://www.rfc-editor.org/info/rfc7952); see the RFC itself
for full legal notices.";
revision 2016-08-05 {
description
"Initial revision";
reference "RFC 3031: A YANG Data Model for base MPLS";
"Initial revision.";
reference
"RFC 7952: Defining and Using Metadata with YANG";
}
typedef mpls-label {
type uint32 {
range "0..1048575";
}
extension annotation {
argument name;
description
"The MPLS label range";
}
"This extension allows for defining metadata annotations in
YANG modules. The 'md:annotation' statement can appear only
at the top level of a YANG module or submodule, i.e., it
becomes a new alternative in the ABNF production rule for
'body-stmts' (Section 14 in RFC 7950).
typedef percent {
type uint16 {
range "0 .. 100";
}
description "Percentage";
}
The argument of the 'md:annotation' statement defines the name
of the annotation. Syntactically, it is a YANG identifier as
defined in Section 6.2 of RFC 7950.
grouping interface-mpls {
description "MPLS interface properties grouping";
leaf enabled {
type boolean;
description
"'true' if mpls encapsulation is enabled on the
interface. 'false' if mpls encapsulation is enabled
on the interface.";
}
}
An annotation defined with this 'extension' statement inherits
the namespace and other context from the YANG module in which
it is defined.
augment "/rt:routing/rt:routing-instance" {
description "MPLS augmentation.";
container mpls {
description
"MPLS container, to be used as an augmentation target node
other MPLS sub-features config, e.g. MPLS static LSP, MPLS
LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc.";
The data type of the annotation value is specified in the same
way as for a leaf data node using the 'type' statement.
list interface {
key "name";
description "List of MPLS interfaces";
leaf name {
type if:interface-ref;
description
"The name of a configured MPLS interface";
}
container config {
description "Holds intended configuration";
uses interface-mpls;
}
container state {
config false;
description "Holds inuse configuration";
uses interface-mpls;
}
}
}
}
The semantics of the annotation and other documentation can be
specified using the following standard YANG substatements (all
are optional): 'description', 'if-feature', 'reference',
'status', and 'units'.
augment "/rt:routing-state/rt:routing-instance" {
description "MPLS augmentation.";
container mpls {
config false;
description
"MPLS container, to be used as an augmentation target node
other MPLS sub-features state";
}
A server announces support for a particular annotation by
including the module in which the annotation is defined among
the advertised YANG modules, e.g., in a NETCONF <hello>
message or in the YANG library (RFC 7950). The annotation can
then be attached to any instance of a data node defined in any
YANG module that is advertised by the server.
XML encoding and JSON encoding of annotations are defined in
RFC 7952.";
}
}

139
ietf/submit/test_submission.txt
View file

@ -73,99 +73,90 @@ Table of Contents
2. Yang
<CODE BEGINS> file "ietf-mpls@2015-10-16.yang"
<CODE BEGINS> file "ietf-yang-metadata@2016-08-05.yang"
module ietf-mpls {
module ietf-yang-metadata {
namespace "urn:ietf:params:xml:ns:yang:ietf-mpls";
namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata";
prefix "mpls";
prefix "md";
import ietf-routing {
prefix "rt";
}
organization
"IETF NETMOD (NETCONF Data Modeling Language) Working Group";
import ietf-interfaces {
prefix "if";
}
contact
"WG Web: <https://datatracker.ietf.org/wg/netmod/>
organization "TBD";
WG List: <mailto:netmod@ietf.org>
contact "TBD";
WG Chair: Lou Berger
<mailto:lberger@labn.net>
WG Chair: Kent Watsen
<mailto:kwatsen@juniper.net>
Editor: Ladislav Lhotka
<mailto:lhotka@nic.cz>";
description
"This YANG module defines the essential components for the
management of the MPLS subsystem.";
"This YANG module defines an 'extension' statement that allows
for defining metadata annotations.
revision "2015-10-16" {
Copyright (c) 2016 IETF Trust and the persons identified as
authors of the code. All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, is permitted pursuant to, and subject to
the license terms contained in, the Simplified BSD License set
forth in Section 4.c of the IETF Trust's Legal Provisions
Relating to IETF Documents
(http://trustee.ietf.org/license-info).
This version of this YANG module is part of RFC 7952
(http://www.rfc-editor.org/info/rfc7952); see the RFC itself
for full legal notices.";
revision 2016-08-05 {
description
"Initial revision";
reference "RFC 3031: A YANG Data Model for base MPLS";
"Initial revision.";
reference
"RFC 7952: Defining and Using Metadata with YANG";
}
typedef mpls-label {
type uint32 {
range "0..1048575";
}
extension annotation {
argument name;
description
"The MPLS label range";
}
"This extension allows for defining metadata annotations in
YANG modules. The 'md:annotation' statement can appear only
at the top level of a YANG module or submodule, i.e., it
becomes a new alternative in the ABNF production rule for
'body-stmts' (Section 14 in RFC 7950).
typedef percent {
type uint16 {
range "0 .. 100";
}
description "Percentage";
}
The argument of the 'md:annotation' statement defines the name
of the annotation. Syntactically, it is a YANG identifier as
defined in Section 6.2 of RFC 7950.
grouping interface-mpls {
description "MPLS interface properties grouping";
leaf enabled {
type boolean;
description
"'true' if mpls encapsulation is enabled on the
interface. 'false' if mpls encapsulation is enabled
on the interface.";
}
}
An annotation defined with this 'extension' statement inherits
the namespace and other context from the YANG module in which
it is defined.
augment "/rt:routing/rt:routing-instance" {
description "MPLS augmentation.";
container mpls {
description
"MPLS container, to be used as an augmentation target node
other MPLS sub-features config, e.g. MPLS static LSP, MPLS
LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc.";
The data type of the annotation value is specified in the same
way as for a leaf data node using the 'type' statement.
list interface {
key "name";
description "List of MPLS interfaces";
leaf name {
type if:interface-ref;
description
"The name of a configured MPLS interface";
}
container config {
description "Holds intended configuration";
uses interface-mpls;
}
container state {
config false;
description "Holds inuse configuration";
uses interface-mpls;
}
}
}
}
The semantics of the annotation and other documentation can be
specified using the following standard YANG substatements (all
are optional): 'description', 'if-feature', 'reference',
'status', and 'units'.
augment "/rt:routing-state/rt:routing-instance" {
description "MPLS augmentation.";
container mpls {
config false;
description
"MPLS container, to be used as an augmentation target node
other MPLS sub-features state";
}
A server announces support for a particular annotation by
including the module in which the annotation is defined among
the advertised YANG modules, e.g., in a NETCONF <hello>
message or in the YANG library (RFC 7950). The annotation can
then be attached to any instance of a data node defined in any
YANG module that is advertised by the server.
XML encoding and JSON encoding of annotations are defined in
RFC 7952.";
}
}

149
ietf/submit/test_submission.xml
View file

@ -35,100 +35,91 @@
<figure>
<artwork>
<![CDATA[
<CODE BEGINS> file "ietf-mpls@2015-10-16.yang"
<CODE BEGINS> file "ietf-yang-metadata@2016-08-05.yang"
module ietf-mpls {
module ietf-yang-metadata {
namespace "urn:ietf:params:xml:ns:yang:ietf-mpls";
namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata";
prefix "mpls";
prefix "md";
import ietf-routing {
prefix "rt";
}
organization
"IETF NETMOD (NETCONF Data Modeling Language) Working Group";
import ietf-interfaces {
prefix "if";
}
contact
"WG Web: <https://datatracker.ietf.org/wg/netmod/>
organization "TBD";
WG List: <mailto:netmod@ietf.org>
contact "TBD";
WG Chair: Lou Berger
<mailto:lberger@labn.net>
description
"This YANG module defines the essential components for the
management of the MPLS subsystem.";
WG Chair: Kent Watsen
<mailto:kwatsen@juniper.net>
revision "2015-10-16" {
description
"Initial revision";
reference "RFC 3031: A YANG Data Model for base MPLS";
}
Editor: Ladislav Lhotka
<mailto:lhotka@nic.cz>";
typedef mpls-label {
type uint32 {
range "0..1048575";
}
description
"The MPLS label range";
}
description
"This YANG module defines an 'extension' statement that allows
for defining metadata annotations.
typedef percent {
type uint16 {
range "0 .. 100";
}
description "Percentage";
}
Copyright (c) 2016 IETF Trust and the persons identified as
authors of the code. All rights reserved.
grouping interface-mpls {
description "MPLS interface properties grouping";
leaf enabled {
type boolean;
description
"'true' if mpls encapsulation is enabled on the
interface. 'false' if mpls encapsulation is enabled
on the interface.";
}
}
Redistribution and use in source and binary forms, with or
without modification, is permitted pursuant to, and subject to
the license terms contained in, the Simplified BSD License set
forth in Section 4.c of the IETF Trust's Legal Provisions
Relating to IETF Documents
(http://trustee.ietf.org/license-info).
augment "/rt:routing/rt:routing-instance" {
description "MPLS augmentation.";
container mpls {
description
"MPLS container, to be used as an augmentation target node
other MPLS sub-features config, e.g. MPLS static LSP, MPLS
LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc.";
This version of this YANG module is part of RFC 7952
(http://www.rfc-editor.org/info/rfc7952); see the RFC itself
for full legal notices.";
list interface {
key "name";
description "List of MPLS interfaces";
leaf name {
type if:interface-ref;
description
"The name of a configured MPLS interface";
}
container config {
description "Holds intended configuration";
uses interface-mpls;
}
container state {
config false;
description "Holds inuse configuration";
uses interface-mpls;
}
}
}
}
revision 2016-08-05 {
description
"Initial revision.";
reference
"RFC 7952: Defining and Using Metadata with YANG";
}
augment "/rt:routing-state/rt:routing-instance" {
description "MPLS augmentation.";
container mpls {
config false;
description
"MPLS container, to be used as an augmentation target node
other MPLS sub-features state";
}
}
extension annotation {
argument name;
description
"This extension allows for defining metadata annotations in
YANG modules. The 'md:annotation' statement can appear only
at the top level of a YANG module or submodule, i.e., it
becomes a new alternative in the ABNF production rule for
'body-stmts' (Section 14 in RFC 7950).
The argument of the 'md:annotation' statement defines the name
of the annotation. Syntactically, it is a YANG identifier as
defined in Section 6.2 of RFC 7950.
An annotation defined with this 'extension' statement inherits
the namespace and other context from the YANG module in which
it is defined.
The data type of the annotation value is specified in the same
way as for a leaf data node using the 'type' statement.
The semantics of the annotation and other documentation can be
specified using the following standard YANG substatements (all
are optional): 'description', 'if-feature', 'reference',
'status', and 'units'.
A server announces support for a particular annotation by
including the module in which the annotation is defined among
the advertised YANG modules, e.g., in a NETCONF <hello>
message or in the YANG library (RFC 7950). The annotation can
then be attached to any instance of a data node defined in any
YANG module that is advertised by the server.
XML encoding and JSON encoding of annotations are defined in
RFC 7952.";
}
}
<CODE ENDS>

25
ietf/submit/tests.py
View file

@ -68,17 +68,17 @@ class SubmitTests(TestCase):
self.archive_dir = self.tempdir('submit-archive')
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.archive_dir
self.saved_yang_rfc_model_dir = settings.YANG_RFC_MODEL_DIR
self.saved_yang_rfc_model_dir = settings.SUBMIT_YANG_RFC_MODEL_DIR
self.yang_rfc_model_dir = self.tempdir('yang-rfc-model')
settings.YANG_RFC_MODEL_DIR = self.yang_rfc_model_dir
settings.SUBMIT_YANG_RFC_MODEL_DIR = self.yang_rfc_model_dir
self.saved_yang_draft_model_dir = settings.YANG_DRAFT_MODEL_DIR
self.saved_yang_draft_model_dir = settings.SUBMIT_YANG_DRAFT_MODEL_DIR
self.yang_draft_model_dir = self.tempdir('yang-draft-model')
settings.YANG_DRAFT_MODEL_DIR = self.yang_draft_model_dir
settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.yang_draft_model_dir
self.saved_yang_inval_model_dir = settings.YANG_INVAL_MODEL_DIR
self.saved_yang_inval_model_dir = settings.SUBMIT_YANG_INVAL_MODEL_DIR
self.yang_inval_model_dir = self.tempdir('yang-inval-model')
settings.YANG_INVAL_MODEL_DIR = self.yang_inval_model_dir
settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.yang_inval_model_dir
def tearDown(self):
shutil.rmtree(self.staging_dir)
@ -91,9 +91,9 @@ class SubmitTests(TestCase):
settings.INTERNET_DRAFT_PATH = self.saved_internet_draft_path
settings.IDSUBMIT_REPOSITORY_PATH = self.saved_idsubmit_repository_path
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir
settings.YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir
settings.YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir
settings.YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir
settings.SUBMIT_YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir
settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir
settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir
def do_submission(self, name, rev, group=None, formats=["txt",]):
@ -133,6 +133,8 @@ class SubmitTests(TestCase):
self.assertEqual(author["affiliation"], "Test Centre Inc.")
self.assertEqual(author["country"], "UK")
return status_url
def supply_extra_metadata(self, name, status_url, submitter_name, submitter_email, replaces):
@ -220,6 +222,11 @@ class SubmitTests(TestCase):
r = self.client.get(status_url)
self.assertEqual(r.status_code, 200)
self.assertContains(r, 'xym')
self.assertContains(r, 'pyang')
self.assertContains(r, 'yanglint')
q = PyQuery(r.content)
approve_button = q('[type=submit]:contains("Approve")')
self.assertEqual(len(approve_button), 1)

4
ietf/submit/views.py
View file

@ -165,7 +165,9 @@ def upload_submission(request):
def apply_check(submission, checker, method, fn):
func = getattr(checker, method)
passed, message, errors, warnings, items = func(fn)
check = SubmissionCheck(submission=submission, checker=checker.name, passed=passed, message=message, errors=errors, warnings=warnings, items=items, symbol=checker.symbol)
check = SubmissionCheck(submission=submission, checker=checker.name, passed=passed,
message=message, errors=errors, warnings=warnings, items=items,
symbol=checker.symbol)
check.save()
for checker_path in settings.IDSUBMIT_CHECKER_CLASSES:

30
ietf/templates/doc/document_draft.html
View file

@ -76,15 +76,6 @@
{% if latest_revision and latest_revision.time.date != doc.time.date %}
(latest revision {{ latest_revision.time|date:"Y-m-d" }})
{% endif %}
{% for check in doc.submission.checks.all %}
{% if check.passed != None and check.symbol.strip %}
{% if check.errors or check.warnings %}
<span class="checker-warning" title="Submission {{check.checker|title}} returned warnings or errors.">{{ check.symbol|safe }}</span>
{% else %}
<span class="checker-success" title="Submission {{check.checker|title}} passed">{{ check.symbol|safe }}</span>
{% endif %}
{% endif %}
{% endfor %}
</td>
</tr>
@ -192,6 +183,27 @@
</td>
</tr>
{% for check in doc.submission.latest_checks %}
{% if check.passed != None and check.symbol.strip %}
<tr>
<th></th>
<th>{{ check.checker|title }}</th>
<td class="edit"></td>
<td>
{% if check.errors or check.warnings %}
<span class="checker-warning" data-toggle="modal" data-target="#check-{{check.pk}}" title="{{check.checker|title}} returned warnings or errors." >{{ check.symbol|safe }}</span>
{% else %}
<span class="checker-success" data-toggle="modal" data-target="#check-{{check.pk}}" title="{{check.checker|title}} passed">{{ check.symbol|safe }}</span>
{% endif %}
<a href="#" data-toggle="modal" data-target="#check-{{check.pk}}">
{{ check.errors }} errors, {{ check.warnings }} warnings.
</a>
{% include "doc/yang-check-modal-overlay.html" %}
</td>
</tr>
{% endif %}
{% endfor %}
{% if review_requests or can_request_review %}
<tr>
<th></th>

7
ietf/templates/doc/search/search_result_row.html
View file

@ -56,13 +56,14 @@
{% if doc.latest_revision_date|timesince_days|new_enough:request and doc.get_state_slug != "rfc" %}</a>{% endif %}
</span>
{% for check in doc.submission.checks.all %}
{% for check in doc.submission.latest_checks %}
{% if check.passed != None and check.symbol.strip %}
{% if check.errors or check.warnings %}
<span class="checker-warning pull-right" title="{{check.checker|title}} returned warnings or errors.">{{ check.symbol|safe }}</span>
<span class="checker-warning pull-right" data-toggle="modal" data-target="#check-{{check.pk}}" title="{{check.checker|title}} returned warnings or errors." >{{ check.symbol|safe }}</span>
{% else %}
<span class="checker-success pull-right" title="{{check.checker|title}} passed">{{ check.symbol|safe }}</span>
<span class="checker-success pull-right" data-toggle="modal" data-target="#check-{{check.pk}}" title="{{check.checker|title}} passed">{{ check.symbol|safe }}</span>
{% endif %}
{% include "doc/yang-check-modal-overlay.html" %}
{% endif %}
{% endfor %}

17
ietf/templates/doc/yang-check-modal-overlay.html Normal file
View file

@ -0,0 +1,17 @@
{% load ietf_filters %}{% load origin %}{% origin %}
<div class="modal fade" id="check-{{check.pk}}" tabindex="-1" role="dialog" aria-labelledby="check-{{check.pk}}" aria-hidden="true">
<div class="modal-dialog modal-lg">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
<h4 class="modal-title" id="nitslabel">{{ check.checker|title }} for {{ doc.name }}-{{ doc.rev }} on {{ check.time|date:"Y-m-d" }}</h4>
</div>
<div class="modal-body">
<pre class="pasted">{{ check.message|zaptmp }}</pre>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>

4
ietf/templates/submit/submission_status.html
View file

@ -44,7 +44,7 @@
Your draft has <b>NOT</b> been verified to pass the submission checks.
{% endif %}
</p>
{% for check in submission.checks.all %}
{% for check in submission.latest_checks %}
{% if check.errors %}
<p class="alert alert-warning">
The {{check.checker}} returned {{ check.errors }} error{{ check.errors|pluralize }}
@ -58,7 +58,7 @@
{% endif %}
{% endfor %}
{% for check in submission.checks.all %}
{% for check in submission.latest_checks %}
{% if check.passed != None %}
<button class="btn btn-{% if check.passed %}{% if check.warnings %}warning{% elif check.errors %}warning{% else %}success{% endif %}{% else %}danger{% endif %}" data-toggle="modal" data-target="#check-{{check.pk}}">View {{ check.checker }}</button>

5
ietf/utils/admin.py
View file

@ -1,3 +1,5 @@
from django.contrib import admin
from ietf.utils.models import VersionInfo
def name(obj):
if hasattr(obj, 'abbrev'):
@ -45,3 +47,6 @@ def admin_link(field, label=None, ordering="", display=name, suffix=""):
_link.admin_order_field = ordering
return _link
class VersionInfoAdmin(admin.ModelAdmin):
list_display = ['command', 'switch', 'version', 'time', ]
admin.site.register(VersionInfo, VersionInfoAdmin)

105
ietf/utils/management/commands/populate_yang_model_dirs.py
View file

@ -1,8 +1,10 @@
from __future__ import print_function, unicode_literals
import os
import sys
import time
from pathlib import Path
from pathlib2 import Path
from StringIO import StringIO
from textwrap import dedent
from xym import xym
@ -10,9 +12,11 @@ from xym import xym
from django.conf import settings
from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
class Command(BaseCommand):
"""
Populate the yang models repositories from drafts and RFCs.
Populate the yang module repositories from drafts and RFCs.
Extracts yang models from RFCs (found in settings.RFC_PATH and places
them in settings.YANG_RFC_MODEL_DIR, and from active drafts, placed in
@ -57,22 +61,34 @@ class Command(BaseCommand):
verbosity = int(options.get('verbosity'))
def extract_from(file, dir, strict=True):
saved_stdout = sys.stdout
saved_stderr = sys.stderr
sys.stderr = StringIO()
xymerr = StringIO()
xymout = StringIO()
sys.stderr = xymerr
sys.stdout = xymout
model_list = []
try:
model_list = xym.xym(str(item), str(item.parent), str(dir), strict=strict, debug_level=(verbosity>1))
model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=verbosity-2)
for name in model_list:
modfile = moddir / name
mtime = item.stat().st_mtime
mtime = file.stat().st_mtime
os.utime(str(modfile), (mtime, mtime))
if '"' in name:
name = name.replace('"', '')
modfile.rename(str(moddir/name))
model_list = [ n.replace('"','') for n in model_list ]
model_list = [ n.replace('"','') for n in model_list ]
except Exception as e:
print("** Error when extracting from %s: %s" % (item, str(e)))
print("** Error when extracting from %s: %s" % (file, str(e)))
sys.stdout = saved_stdout
sys.stderr = saved_stderr
#
if verbosity > 1:
outmsg = xymout.getvalue()
self.stdout.write(outmsg)
if verbosity>2:
errmsg = xymerr.getvalue()
self.stderr.write(errmsg)
return model_list
# Extract from new RFCs
@ -87,10 +103,10 @@ class Command(BaseCommand):
for item in moddir.iterdir():
if item.stat().st_mtime > latest:
latest = item.stat().st_mtime
print("Extracting to %s ..." % moddir)
for item in rfcdir.iterdir():
if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt'):
if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit():
if item.stat().st_mtime > latest:
model_list = extract_from(item, moddir)
for name in model_list:
@ -124,20 +140,25 @@ class Command(BaseCommand):
print("Extracting to %s ..." % moddir)
for item in draftdir.iterdir():
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir)
for name in model_list:
if not name.startswith('example'):
if verbosity > 1:
print(" Extracted valid module from %s: %s" % (item, name))
try:
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir)
for name in model_list:
if not name.startswith('example'):
if verbosity > 1:
print(" Extracted valid module from %s: %s" % (item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
modfile = moddir / name
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" % (item, name))
modfile = moddir / name
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" % (item, name))
except UnicodeDecodeError as e:
sys.stderr.write('\nError: %s\n' % (e, ))
sys.stderr.write(item.name)
sys.stderr.write('\n')
print("")
# Extract invalid modules from drafts
@ -151,22 +172,28 @@ class Command(BaseCommand):
print("Extracting to %s ..." % moddir)
for item in draftdir.iterdir():
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir, strict=False)
for name in model_list:
modfile = moddir / name
if (valdir/name).exists():
modfile.unlink()
if verbosity > 1:
print(" Skipped valid module from %s: %s" % (item, name))
elif not name.startswith('example'):
if verbosity > 1:
print(" Extracted invalid module from %s: %s" % (item, name))
try:
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir, strict=False)
for name in model_list:
modfile = moddir / name
if (valdir/name).exists():
modfile.unlink()
if verbosity > 1:
print(" Skipped valid module from %s: %s" % (item, name))
elif not name.startswith('example'):
if verbosity > 1:
print(" Extracted invalid module from %s: %s" % (item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" % (item, name))
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" % (item, name))
except UnicodeDecodeError as e:
sys.stderr.write('\nError: %s\n' % (e, ))
sys.stderr.write(item.name)
sys.stderr.write('\n')
print("")

82
ietf/utils/management/commands/run_yang_model_checks.py Normal file
View file

@ -0,0 +1,82 @@
from __future__ import print_function, unicode_literals
import sys
import json
from textwrap import dedent
from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
from ietf.doc.models import Document, State
from ietf.submit.models import Submission, SubmissionCheck
from ietf.submit.checkers import DraftYangChecker
class Command(BaseCommand):
"""
Run yang model checks on active drafts.
Repeats the yang checks in ietf/submit/checkers.py for active drafts, in
order to catch changes in status due to new modules becoming available in
the module directories.
"""
help = dedent(__doc__).strip()
def add_arguments(self, parser):
parser.add_argument('filenames', nargs="*")
parser.add_argument('--clean',
action='store_true', dest='clean', default=False,
help='Remove the current directory content before writing new models.')
def check_yang(self, checker, draft, force=False):
if self.verbosity > 1:
self.stdout.write("Checking %s-%s" % (draft.name, draft.rev))
else:
sys.stderr.write('.')
submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first()
if submission or force:
check = submission.checks.filter(checker=checker.name).order_by('-id').first()
if check or force:
result = checker.check_file_txt(draft.get_file_name())
passed, message, errors, warnings, items = result
if self.verbosity > 2:
self.stdout.write(" Errors: %s\n"
" Warnings: %s\n"
" Message:\n%s\n" % (errors, warnings, message))
items = json.loads(json.dumps(items))
new_res = (passed, errors, warnings, message)
old_res = (check.passed, check.errors, check.warnings, check.message) if check else ()
if new_res != old_res:
if self.verbosity > 1:
self.stdout.write(" Saving new yang checker results for %s-%s" % (draft.name, draft.rev))
SubmissionCheck.objects.create(submission=submission, checker=checker.name, passed=passed,
message=message, errors=errors, warnings=warnings, items=items,
symbol=checker.symbol)
else:
self.stderr.write("Error: did not find any submission object for %s-%s\n" % (draft.name, draft.rev))
def handle(self, *filenames, **options):
"""
"""
self.verbosity = int(options.get('verbosity'))
filenames = options.get('filenames')
active_state = State.objects.get(type="draft", slug="active")
checker = DraftYangChecker()
if filenames:
for name in filenames:
parts = name.rsplit('-',1)
if len(parts)==2 and len(parts[1])==2 and parts[1].isdigit():
name = parts[0]
draft = Document.objects.get(name=name)
self.check_yang(checker, draft, force=True)
else:
for draft in Document.objects.filter(states=active_state, type_id='draft'):
self.check_yang(checker, draft)

37
ietf/utils/management/commands/update_external_command_info.py Normal file
View file

@ -0,0 +1,37 @@
from __future__ import print_function, unicode_literals
import sys
from textwrap import dedent
from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
from ietf.utils.models import VersionInfo
from ietf.utils.pipe import pipe
class Command(BaseCommand):
"""
Update the version information for external commands used by the datatracker.
Iterates through the entries in the VersionInfo table, runs the relevant
command, and updates the version string with the result.
"""
help = dedent(__doc__).strip()
def handle(self, *filenames, **options):
for c in VersionInfo.objects.filter(used=True):
cmd = "%s %s" % (c.command, c.switch)
code, out, err = pipe(cmd)
if code != 0:
sys.stderr.write("Command '%s' retuned %s: \n%s\n%s\n" % (cmd, code, out, err))
else:
c.version = (out.strip()+'\n'+err.strip()).strip()
if options.get('verbosity', 1) > 1:
sys.stdout.write(
"Command: %s\n"
" Version: %s\n" % (cmd, c.version))
c.save()

26
ietf/utils/migrations/0003_versioninfo.py Normal file
View file

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.10.7 on 2017-06-15 03:31
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('utils', '0002_dumpinfo_tz'),
]
operations = [
migrations.CreateModel(
name='VersionInfo',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('time', models.DateTimeField(auto_now=True)),
('command', models.CharField(max_length=32)),
('switch', models.CharField(max_length=16)),
('version', models.CharField(max_length=64)),
('used', models.BooleanField(default=True)),
],
),
]

9
ietf/utils/models.py
View file

@ -7,3 +7,12 @@ class DumpInfo(models.Model):
host = models.CharField(max_length=128)
tz = models.CharField(max_length=32, default='UTC')
class VersionInfo(models.Model):
time = models.DateTimeField(auto_now=True)
command = models.CharField(max_length=32)
switch = models.CharField(max_length=16)
version = models.CharField(max_length=64)
used = models.BooleanField(default=True)
class Meta:
verbose_name_plural = 'VersionInfo'

21
ietf/utils/resources.py
View file

@ -8,7 +8,8 @@ from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from ietf import api
from ietf.utils.models import DumpInfo
from ietf.utils.models import DumpInfo, VersionInfo
class UserResource(ModelResource):
username = CharField()
@ -17,6 +18,7 @@ class UserResource(ModelResource):
queryset = User.objects.all()
serializer = api.Serializer()
class ContentTypeResource(ModelResource):
username = CharField()
class Meta:
@ -24,6 +26,7 @@ class ContentTypeResource(ModelResource):
queryset = ContentType.objects.all()
serializer = api.Serializer()
class DumpInfoResource(ModelResource):
class Meta:
cache = SimpleCache()
@ -36,3 +39,19 @@ class DumpInfoResource(ModelResource):
}
api.utils.register(DumpInfoResource())
class VersionInfoResource(ModelResource):
class Meta:
queryset = VersionInfo.objects.all()
serializer = api.Serializer()
cache = SimpleCache()
#resource_name = 'versioninfo'
filtering = {
"id": ALL,
"time": ALL,
"command": ALL,
"switch": ALL,
"version": ALL,
"used": ALL,
}
api.utils.register(VersionInfoResource())

1
requirements.txt
View file

@ -31,6 +31,7 @@ mimeparse>=0.1.3 # from TastyPie
mock>=2.0.0
MySQL-python>=1.2.5
pathlib>=1.0
pathlib2>=2.3.0
Pillow>=3.0
pyang>=1.6
pyflakes>=0.8.1