altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Added a new yang checker, 'yanglint', to the existing Yang checker class, in

Browse source 
addition to the existing 'pyang' checker.

Added modal overlay displays showing the yang check results every place the
yin/yang symbol is shown (red or green) to indicate the presencee and result
of yang checks.  Added a Yang Validation: line in the document
meta-information section on the document's page in the datatracker.

Added the result of the xym extaction to the yang check results, to make
extration failures visible.

Added the version of the used xym, pyang, and yanglint commands to the check
results.

Added an action to move successfully extracted and validated modules to the
module library directories immediately on submission.

Added the xym and pyang repositories as svn:external components, rather than
listing them in requirements.txt, as there has been delays of many months
between essential features in the repositories, and an actual release.  We may
get occasional buildbot failures if broken code is pulled in from the
repository, but better that than the functionality failure of severely
outdated componets.

Added a new management command to re-run yang validation for active drafts for
which yang modules were found at submission time, in order to pick up imported
models which may have arrived in the model libraries after the draft's
submission.  Run daily from bin/daily.

Added a table to hold version information for external commands.  The yang
checker output should include the version information of the used checkers,
but seems unnecessary to run each command with its --version switch every
time we check a module...

Added a new management command to collect version information for external
commands on demand.  To be run daily from bin/daily.

Added tests to verify that xym, pyang and yanglint information is available
on the submission confirmation page, and updated the yang module contained in
the test document to validate under both pyang and yanglint.

Updated admin.py and resource.py files as needed.
 - Legacy-Id: 13630
This commit is contained in:
Henrik Levkowetz 2017-06-15 16:09:28 +00:00
parent c53e3784c3
commit d98054c103
29 changed files with 676 additions and 330 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -39,3 +39,4 @@
/static /static
/testresult /testresult
/unix.tag /unix.tag
/tmp-nomcom-public-keys-dir

9
bin/daily
View file

@ -18,9 +18,16 @@ logger -p user.info -t cron "Running $DTDIR/bin/daily"
# Set up the virtual environment # Set up the virtual environment
source $DTDIR/env/bin/activate source $DTDIR/env/bin/activate
# Update our information about the current version of some commands we use
$DTDIR/ietf/manage.py update_external_command_info
# Populate the yang repositories # Populate the yang repositories
$DTDIR/ietf/manage.py populate_yang_model_dirs $DTDIR/ietf/manage.py populate_yang_model_dirs
# Re-run yang checks on active documents
$DTDIR/ietf/manage.py run_yang_model_checks
# Expire internet drafts # Expire internet drafts
# Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner: # Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner:
$DTDIR/ietf/bin/expire-ids $DTDIR/ietf/bin/expire-ids
@ -36,5 +43,3 @@ $DTDIR/ietf/bin/rfc-editor-index-updates -d 1969-01-01
# Fetch meeting attendance data from ietf.org/registration/attendees # Fetch meeting attendance data from ietf.org/registration/attendees
$DTDIR/ietf/manage.py fetch_meeting_attendance --latest $DTDIR/ietf/manage.py fetch_meeting_attendance --latest

3
ietf/doc/templatetags/ietf_filters.py
View file

@ -533,3 +533,6 @@ def comma_separated_list(seq, end_word="and"):
def role_names(roles): def role_names(roles):
return list(set([ "%s %s" % (r.group.name, r.name.name) for r in roles ])) return list(set([ "%s %s" % (r.group.name, r.name.name) for r in roles ]))
@register.filter()
def zaptmp(s):
return re.sub(r'/tmp/tmp[^/]+/', '', s)

8
ietf/doc/views_doc.py
View file

@ -1277,3 +1277,11 @@ def all_presentations(request, name):
'in_progress': in_progress, 'in_progress': in_progress,
'past' : past, 'past' : past,
}) })
def document_checks(request, name):
doc = get_object_or_404(Document, docalias__name=name, type_id='draft')
checks = doc.submission.latest_checks()
debug.show('checks')
return render(request, 'doc/document_checks.html', {'doc': doc, 'checks': checks})

33
ietf/name/fixtures/names.json
View file

@ -9708,5 +9708,38 @@
}, },
"model": "name.topicaudiencename", "model": "name.topicaudiencename",
"pk": "nominees" "pk": "nominees"
},
{
"fields": {
"command": "xym",
"switch": "--version",
"time": "2017-06-15T06:24:58.869",
"used": true,
"version": "xym 0.3.2"
},
"model": "utils.versioninfo",
"pk": 1
},
{
"fields": {
"command": "pyang",
"switch": "--version",
"time": "2017-06-15T06:24:59.516",
"used": true,
"version": "pyang 1.7.2"
},
"model": "utils.versioninfo",
"pk": 2
},
{
"fields": {
"command": "yanglint",
"switch": "--version",
"time": "2017-06-15T06:24:59.531",
"used": true,
"version": "yanglint 0.12.183"
},
"model": "utils.versioninfo",
"pk": 3
} }
] ]

3
ietf/name/generate_fixtures.py
View file

@ -47,5 +47,8 @@ import ietf.mailtrigger.models
objects += ietf.mailtrigger.models.Recipient.objects.all() objects += ietf.mailtrigger.models.Recipient.objects.all()
objects += ietf.mailtrigger.models.MailTrigger.objects.all() objects += ietf.mailtrigger.models.MailTrigger.objects.all()
import ietf.utils.models
objects += ietf.utils.models.VersionInfo.objects.all()
output("names", objects) output("names", objects)

14
ietf/settings.py
View file

@ -660,11 +660,19 @@ IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_01 = 13
IDSUBMIT_DEFAULT_CUTOFF_TIME_UTC = datetime.timedelta(hours=23, minutes=59, seconds=59) IDSUBMIT_DEFAULT_CUTOFF_TIME_UTC = datetime.timedelta(hours=23, minutes=59, seconds=59)
IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21) IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21)
# 14 Jun 2017: New convention: prefix settings with the app name to which
# they (mainly) belong. So here, SUBMIT_, rather than IDSUBMIT_
SUBMIT_YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
SUBMIT_YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
SUBMIT_YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/'
IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/' IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/' IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits' IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
IDSUBMIT_PYANG_COMMAND = 'pyang -p %(modpath)s --verbose --ietf %(model)s' SUBMIT_PYANG_COMMAND = 'pyang --verbose --ietf -p {libs} {model}'
SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {rfclib} -p {draftlib} {model}'
SUBMIT_YANGLINT_COMMAND = None # use the value above if you have yanglint installed
IDSUBMIT_CHECKER_CLASSES = ( IDSUBMIT_CHECKER_CLASSES = (
"ietf.submit.checkers.DraftIdnitsChecker", "ietf.submit.checkers.DraftIdnitsChecker",
@ -696,10 +704,6 @@ IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE = 450 # in MB
IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000 IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000
IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB
YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/'
XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/" XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/"
# === Meeting Related Settings ================================================= # === Meeting Related Settings =================================================

6
ietf/static/ietf/css/ietf.css
View file

@ -865,3 +865,9 @@ blockquote {
#debug-query-table .code .current { #debug-query-table .code .current {
background-color: #ddd; background-color: #ddd;
} }
.checker-warning,
.checker-success {
line-height: 1.0;
cursor: pointer;
}

2
ietf/submit/admin.py
View file

@ -33,7 +33,7 @@ class SubmissionEventAdmin(admin.ModelAdmin):
admin.site.register(SubmissionEvent, SubmissionEventAdmin) admin.site.register(SubmissionEvent, SubmissionEventAdmin)
class SubmissionCheckAdmin(admin.ModelAdmin): class SubmissionCheckAdmin(admin.ModelAdmin):
list_display = ['submission', 'time', 'checker', 'passed', 'errors', 'warnings', 'items'] list_display = ['submission', 'time', 'checker', 'passed', 'errors', 'warnings', 'message']
raw_id_fields = ['submission'] raw_id_fields = ['submission']
search_fields = ['submission__name'] search_fields = ['submission__name']
admin.site.register(SubmissionCheck, SubmissionCheckAdmin) admin.site.register(SubmissionCheck, SubmissionCheckAdmin)

121
ietf/submit/checkers.py
View file

@ -2,16 +2,19 @@
import os import os
import re import re
import sys
from xym import xym from xym import xym
import shutil import shutil
import tempfile import tempfile
import StringIO
from django.conf import settings from django.conf import settings
import debug # pyflakes:ignore import debug # pyflakes:ignore
from ietf.utils.pipe import pipe
from ietf.utils.log import log from ietf.utils.log import log
from ietf.utils.models import VersionInfo
from ietf.utils.pipe import pipe
class DraftSubmissionChecker(): class DraftSubmissionChecker():
name = "" name = ""
@ -47,6 +50,9 @@ class DraftIdnitsChecker(object):
# start using this when we provide more in the way of warnings during # start using this when we provide more in the way of warnings during
# submission checking: # submission checking:
# symbol = '<span class="fa fa-check-square"></span>' # symbol = '<span class="fa fa-check-square"></span>'
# symbol = u'<span class="large">\ua17d</span>' # Yi syllable 'nit'
# symbol = u'<span class="large">\ub2e1</span>' # Hangul syllable 'nit'
symbol = "" symbol = ""
def __init__(self, options=["--submitcheck", "--nitcount", ]): def __init__(self, options=["--submitcheck", "--nitcount", ]):
@ -123,39 +129,66 @@ class DraftYangChecker(object):
def check_file_txt(self, path): def check_file_txt(self, path):
name = os.path.basename(path) name = os.path.basename(path)
workdir = tempfile.mkdtemp() workdir = tempfile.mkdtemp()
errors = [] errors = 0
warnings = [] warnings = 0
results = {} message = ""
results = []
passed = True # Used by the submission tool. Yang checks always pass.
extractor = xym.YangModuleExtractor(path, workdir, strict=True, debug_level = 0) extractor = xym.YangModuleExtractor(path, workdir, strict=True, strict_examples=False, debug_level=0)
if not os.path.exists(path): if not os.path.exists(path):
return None, "%s: No such file or directory: '%s'"%(name.capitalize(), path), errors, warnings, results return None, "%s: No such file or directory: '%s'"%(name.capitalize(), path), errors, warnings, results
with open(path) as file: with open(path) as file:
out = ""
err = ""
code = 0
try: try:
# This places the yang models as files in workdir # This places the yang models as files in workdir
saved_stdout = sys.stdout
saved_stderr = sys.stderr
sys.stdout = StringIO.StringIO()
sys.stderr = StringIO.StringIO()
extractor.extract_yang_model(file.readlines()) extractor.extract_yang_model(file.readlines())
out = sys.stdout.getvalue()
err = sys.stderr.getvalue()
sys.stdout = saved_stdout
sys.stderr = saved_stderr
model_list = extractor.get_extracted_models() model_list = extractor.get_extracted_models()
except Exception as exc: except Exception as exc:
passed = False code = 1
message = exc err = '\n'.join( [ m for m in [out, err, exc] if m ] )
errors = [ (name, None, None, exc) ] if err:
warnings = [] code += 1
return passed, message, errors, warnings command = "xym"
cmd_version = VersionInfo.objects.get(command=command).version
message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err)
results.append({
"name": name,
"passed": passed,
"message": message,
"warnings": 0,
"errors": code,
"items": [],
})
for model in model_list: for model in model_list:
path = os.path.join(workdir, model) path = os.path.join(workdir, model)
message = ""
modpath = ':'.join([ modpath = ':'.join([
workdir, workdir,
settings.YANG_RFC_MODEL_DIR, settings.SUBMIT_YANG_RFC_MODEL_DIR,
settings.YANG_DRAFT_MODEL_DIR, settings.SUBMIT_YANG_DRAFT_MODEL_DIR,
settings.YANG_INVAL_MODEL_DIR, settings.SUBMIT_YANG_INVAL_MODEL_DIR,
]) ])
with open(path) as file: with open(path) as file:
text = file.readlines() text = file.readlines()
cmd = settings.IDSUBMIT_PYANG_COMMAND % {"modpath": modpath, "model": path, } # pyang
cmd_template = settings.SUBMIT_PYANG_COMMAND
command = cmd_template.split()[0]
cmd_version = VersionInfo.objects.get(command=command).version
cmd = cmd_template.format(libs=modpath, model=path)
code, out, err = pipe(cmd) code, out, err = pipe(cmd)
errors = 0
warnings = 0
items = [] items = []
if code > 0: if code > 0:
error_lines = err.splitlines() error_lines = err.splitlines()
@ -175,26 +208,54 @@ class DraftYangChecker(object):
warnings += 1 warnings += 1
except ValueError: except ValueError:
pass pass
results[model] = { #passed = passed and code == 0 # For the submission tool. Yang checks always pass
"passed": code == 0, message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if code == 0 else err)
"message": out+"No validation errors\n" if code == 0 else err,
# yanglint
cmd_template = settings.SUBMIT_YANGLINT_COMMAND
command = cmd_template.split()[0]
cmd_version = VersionInfo.objects.get(command=command).version
cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
code, out, err = pipe(cmd)
if code > 0:
error_lines = err.splitlines()
for line in error_lines:
if line.strip():
try:
if 'err : ' in line:
errors += 1
if 'warn: ' in line:
warnings += 1
except ValueError:
pass
#passed = passed and code == 0 # For the submission tool. Yang checks always pass
message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if code == 0 else err)
if errors==0 and warnings==0:
dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model)
shutil.move(path, dest)
else:
dest = os.path.join(settings.SUBMIT_YANG_INVAL_MODEL_DIR, model)
shutil.move(path, dest)
# summary result
results.append({
"name": model,
"passed": passed,
"message": message,
"warnings": warnings, "warnings": warnings,
"errors": errors, "errors": errors,
"items": items, "items": items,
} })
shutil.rmtree(workdir) shutil.rmtree(workdir)
## For now, never fail because of failed yang validation. passed = all( res["passed"] for res in results )
if len(model_list): message = "\n".join([ "\n".join([res['name']+':', res["message"]]) for res in results ])
passed = True errors = sum(res["errors"] for res in results )
else: warnings = sum(res["warnings"] for res in results )
passed = None items = [ e for res in results for e in res["items"] ]
#passed = all( res["passed"] for res in results.values() )
message = "\n\n".join([ "\n".join([model+':', res["message"]]) for model, res in results.items() ])
errors = sum(res["errors"] for res in results.values() )
warnings = sum(res["warnings"] for res in results.values() )
items = [ e for res in results.values() for e in res["items"] ]
return passed, message, errors, warnings, items return passed, message, errors, warnings, items

2
ietf/submit/migrations/0010_data_set_submission_check_symbol.py
View file

@ -1,4 +1,4 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import unicode_literals from __future__ import unicode_literals
from tqdm import tqdm from tqdm import tqdm

4
ietf/submit/models.py
View file

@ -65,6 +65,10 @@ class Submission(models.Model):
def existing_document(self): def existing_document(self):
return Document.objects.filter(name=self.name).first() return Document.objects.filter(name=self.name).first()
def latest_checks(self):
checks = [ self.checks.filter(checker=c, passed__in=[True,False]).latest('time') for c in self.checks.values_list('checker', flat=True).distinct() ]
return checks
class SubmissionCheck(models.Model): class SubmissionCheck(models.Model):
time = models.DateTimeField(auto_now=True) time = models.DateTimeField(auto_now=True)
submission = models.ForeignKey(Submission, related_name='checks') submission = models.ForeignKey(Submission, related_name='checks')

139
ietf/submit/test_submission.nonascii
View file

@ -71,99 +71,90 @@ Table of Contents
2. Yang 2. Yang
<CODE BEGINS> file "ietf-mpls@2015-10-16.yang" <CODE BEGINS> file "ietf-yang-metadata@2016-08-05.yang"
module ietf-mpls { module ietf-yang-metadata {
namespace "urn:ietf:params:xml:ns:yang:ietf-mpls"; namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata";
prefix "mpls"; prefix "md";
import ietf-routing { organization
prefix "rt"; "IETF NETMOD (NETCONF Data Modeling Language) Working Group";
}
import ietf-interfaces { contact
prefix "if"; "WG Web: <https://datatracker.ietf.org/wg/netmod/>
}
organization "TBD"; WG List: <mailto:netmod@ietf.org>
contact "TBD"; WG Chair: Lou Berger
<mailto:lberger@labn.net>
WG Chair: Kent Watsen
<mailto:kwatsen@juniper.net>
Editor: Ladislav Lhotka
<mailto:lhotka@nic.cz>";
description description
"This YANG module defines the essential components for the "This YANG module defines an 'extension' statement that allows
management of the MPLS subsystem."; for defining metadata annotations.
revision "2015-10-16" { Copyright (c) 2016 IETF Trust and the persons identified as
authors of the code. All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, is permitted pursuant to, and subject to
the license terms contained in, the Simplified BSD License set
forth in Section 4.c of the IETF Trust's Legal Provisions
Relating to IETF Documents
(http://trustee.ietf.org/license-info).
This version of this YANG module is part of RFC 7952
(http://www.rfc-editor.org/info/rfc7952); see the RFC itself
for full legal notices.";
revision 2016-08-05 {
description description
"Initial revision"; "Initial revision.";
reference "RFC 3031: A YANG Data Model for base MPLS"; reference
"RFC 7952: Defining and Using Metadata with YANG";
} }
typedef mpls-label { extension annotation {
type uint32 { argument name;
range "0..1048575";
}
description description
"The MPLS label range"; "This extension allows for defining metadata annotations in
} YANG modules. The 'md:annotation' statement can appear only
at the top level of a YANG module or submodule, i.e., it
becomes a new alternative in the ABNF production rule for
'body-stmts' (Section 14 in RFC 7950).
typedef percent { The argument of the 'md:annotation' statement defines the name
type uint16 { of the annotation. Syntactically, it is a YANG identifier as
range "0 .. 100"; defined in Section 6.2 of RFC 7950.
}
description "Percentage";
}
grouping interface-mpls { An annotation defined with this 'extension' statement inherits
description "MPLS interface properties grouping"; the namespace and other context from the YANG module in which
leaf enabled { it is defined.
type boolean;
description
"'true' if mpls encapsulation is enabled on the
interface. 'false' if mpls encapsulation is enabled
on the interface.";
}
}
augment "/rt:routing/rt:routing-instance" { The data type of the annotation value is specified in the same
description "MPLS augmentation."; way as for a leaf data node using the 'type' statement.
container mpls {
description
"MPLS container, to be used as an augmentation target node
other MPLS sub-features config, e.g. MPLS static LSP, MPLS
LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc.";
list interface { The semantics of the annotation and other documentation can be
key "name"; specified using the following standard YANG substatements (all
description "List of MPLS interfaces"; are optional): 'description', 'if-feature', 'reference',
leaf name { 'status', and 'units'.
type if:interface-ref;
description
"The name of a configured MPLS interface";
}
container config {
description "Holds intended configuration";
uses interface-mpls;
}
container state {
config false;
description "Holds inuse configuration";
uses interface-mpls;
}
}
}
}
augment "/rt:routing-state/rt:routing-instance" { A server announces support for a particular annotation by
description "MPLS augmentation."; including the module in which the annotation is defined among
container mpls { the advertised YANG modules, e.g., in a NETCONF <hello>
config false; message or in the YANG library (RFC 7950). The annotation can
description then be attached to any instance of a data node defined in any
"MPLS container, to be used as an augmentation target node YANG module that is advertised by the server.
other MPLS sub-features state";
} XML encoding and JSON encoding of annotations are defined in
RFC 7952.";
} }
} }

139
ietf/submit/test_submission.txt
View file

@ -73,99 +73,90 @@ Table of Contents
2. Yang 2. Yang
<CODE BEGINS> file "ietf-mpls@2015-10-16.yang" <CODE BEGINS> file "ietf-yang-metadata@2016-08-05.yang"
module ietf-mpls { module ietf-yang-metadata {
namespace "urn:ietf:params:xml:ns:yang:ietf-mpls"; namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata";
prefix "mpls"; prefix "md";
import ietf-routing { organization
prefix "rt"; "IETF NETMOD (NETCONF Data Modeling Language) Working Group";
}
import ietf-interfaces { contact
prefix "if"; "WG Web: <https://datatracker.ietf.org/wg/netmod/>
}
organization "TBD"; WG List: <mailto:netmod@ietf.org>
contact "TBD"; WG Chair: Lou Berger
<mailto:lberger@labn.net>
WG Chair: Kent Watsen
<mailto:kwatsen@juniper.net>
Editor: Ladislav Lhotka
<mailto:lhotka@nic.cz>";
description description
"This YANG module defines the essential components for the "This YANG module defines an 'extension' statement that allows
management of the MPLS subsystem."; for defining metadata annotations.
revision "2015-10-16" { Copyright (c) 2016 IETF Trust and the persons identified as
authors of the code. All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, is permitted pursuant to, and subject to
the license terms contained in, the Simplified BSD License set
forth in Section 4.c of the IETF Trust's Legal Provisions
Relating to IETF Documents
(http://trustee.ietf.org/license-info).
This version of this YANG module is part of RFC 7952
(http://www.rfc-editor.org/info/rfc7952); see the RFC itself
for full legal notices.";
revision 2016-08-05 {
description description
"Initial revision"; "Initial revision.";
reference "RFC 3031: A YANG Data Model for base MPLS"; reference
"RFC 7952: Defining and Using Metadata with YANG";
} }
typedef mpls-label { extension annotation {
type uint32 { argument name;
range "0..1048575";
}
description description
"The MPLS label range"; "This extension allows for defining metadata annotations in
} YANG modules. The 'md:annotation' statement can appear only
at the top level of a YANG module or submodule, i.e., it
becomes a new alternative in the ABNF production rule for
'body-stmts' (Section 14 in RFC 7950).
typedef percent { The argument of the 'md:annotation' statement defines the name
type uint16 { of the annotation. Syntactically, it is a YANG identifier as
range "0 .. 100"; defined in Section 6.2 of RFC 7950.
}
description "Percentage";
}
grouping interface-mpls { An annotation defined with this 'extension' statement inherits
description "MPLS interface properties grouping"; the namespace and other context from the YANG module in which
leaf enabled { it is defined.
type boolean;
description
"'true' if mpls encapsulation is enabled on the
interface. 'false' if mpls encapsulation is enabled
on the interface.";
}
}
augment "/rt:routing/rt:routing-instance" { The data type of the annotation value is specified in the same
description "MPLS augmentation."; way as for a leaf data node using the 'type' statement.
container mpls {
description
"MPLS container, to be used as an augmentation target node
other MPLS sub-features config, e.g. MPLS static LSP, MPLS
LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc.";
list interface { The semantics of the annotation and other documentation can be
key "name"; specified using the following standard YANG substatements (all
description "List of MPLS interfaces"; are optional): 'description', 'if-feature', 'reference',
leaf name { 'status', and 'units'.
type if:interface-ref;
description
"The name of a configured MPLS interface";
}
container config {
description "Holds intended configuration";
uses interface-mpls;
}
container state {
config false;
description "Holds inuse configuration";
uses interface-mpls;
}
}
}
}
augment "/rt:routing-state/rt:routing-instance" { A server announces support for a particular annotation by
description "MPLS augmentation."; including the module in which the annotation is defined among
container mpls { the advertised YANG modules, e.g., in a NETCONF <hello>
config false; message or in the YANG library (RFC 7950). The annotation can
description then be attached to any instance of a data node defined in any
"MPLS container, to be used as an augmentation target node YANG module that is advertised by the server.
other MPLS sub-features state";
} XML encoding and JSON encoding of annotations are defined in
RFC 7952.";
} }
} }

149
ietf/submit/test_submission.xml
View file

@ -35,100 +35,91 @@
<figure> <figure>
<artwork> <artwork>
<![CDATA[ <![CDATA[
<CODE BEGINS> file "ietf-mpls@2015-10-16.yang" <CODE BEGINS> file "ietf-yang-metadata@2016-08-05.yang"
module ietf-mpls { module ietf-yang-metadata {
namespace "urn:ietf:params:xml:ns:yang:ietf-mpls"; namespace "urn:ietf:params:xml:ns:yang:ietf-yang-metadata";
prefix "mpls"; prefix "md";
import ietf-routing { organization
prefix "rt"; "IETF NETMOD (NETCONF Data Modeling Language) Working Group";
}
import ietf-interfaces { contact
prefix "if"; "WG Web: <https://datatracker.ietf.org/wg/netmod/>
}
organization "TBD"; WG List: <mailto:netmod@ietf.org>
contact "TBD"; WG Chair: Lou Berger
<mailto:lberger@labn.net>
description WG Chair: Kent Watsen
"This YANG module defines the essential components for the <mailto:kwatsen@juniper.net>
management of the MPLS subsystem.";
revision "2015-10-16" { Editor: Ladislav Lhotka
description <mailto:lhotka@nic.cz>";
"Initial revision";
reference "RFC 3031: A YANG Data Model for base MPLS";
}
typedef mpls-label { description
type uint32 { "This YANG module defines an 'extension' statement that allows
range "0..1048575"; for defining metadata annotations.
}
description
"The MPLS label range";
}
typedef percent { Copyright (c) 2016 IETF Trust and the persons identified as
type uint16 { authors of the code. All rights reserved.
range "0 .. 100";
}
description "Percentage";
}
grouping interface-mpls { Redistribution and use in source and binary forms, with or
description "MPLS interface properties grouping"; without modification, is permitted pursuant to, and subject to
leaf enabled { the license terms contained in, the Simplified BSD License set
type boolean; forth in Section 4.c of the IETF Trust's Legal Provisions
description Relating to IETF Documents
"'true' if mpls encapsulation is enabled on the (http://trustee.ietf.org/license-info).
interface. 'false' if mpls encapsulation is enabled
on the interface.";
}
}
augment "/rt:routing/rt:routing-instance" { This version of this YANG module is part of RFC 7952
description "MPLS augmentation."; (http://www.rfc-editor.org/info/rfc7952); see the RFC itself
container mpls { for full legal notices.";
description
"MPLS container, to be used as an augmentation target node
other MPLS sub-features config, e.g. MPLS static LSP, MPLS
LDP LSPs, and Trafic Engineering MPLS LSP Tunnels, etc.";
list interface { revision 2016-08-05 {
key "name"; description
description "List of MPLS interfaces"; "Initial revision.";
leaf name { reference
type if:interface-ref; "RFC 7952: Defining and Using Metadata with YANG";
description }
"The name of a configured MPLS interface";
}
container config {
description "Holds intended configuration";
uses interface-mpls;
}
container state {
config false;
description "Holds inuse configuration";
uses interface-mpls;
}
}
}
}
augment "/rt:routing-state/rt:routing-instance" { extension annotation {
description "MPLS augmentation."; argument name;
container mpls { description
config false; "This extension allows for defining metadata annotations in
description YANG modules. The 'md:annotation' statement can appear only
"MPLS container, to be used as an augmentation target node at the top level of a YANG module or submodule, i.e., it
other MPLS sub-features state"; becomes a new alternative in the ABNF production rule for
} 'body-stmts' (Section 14 in RFC 7950).
}
The argument of the 'md:annotation' statement defines the name
of the annotation. Syntactically, it is a YANG identifier as
defined in Section 6.2 of RFC 7950.
An annotation defined with this 'extension' statement inherits
the namespace and other context from the YANG module in which
it is defined.
The data type of the annotation value is specified in the same
way as for a leaf data node using the 'type' statement.
The semantics of the annotation and other documentation can be
specified using the following standard YANG substatements (all
are optional): 'description', 'if-feature', 'reference',
'status', and 'units'.
A server announces support for a particular annotation by
including the module in which the annotation is defined among
the advertised YANG modules, e.g., in a NETCONF <hello>
message or in the YANG library (RFC 7950). The annotation can
then be attached to any instance of a data node defined in any
YANG module that is advertised by the server.
XML encoding and JSON encoding of annotations are defined in
RFC 7952.";
}
} }
<CODE ENDS> <CODE ENDS>

25
ietf/submit/tests.py
View file

@ -68,17 +68,17 @@ class SubmitTests(TestCase):
self.archive_dir = self.tempdir('submit-archive') self.archive_dir = self.tempdir('submit-archive')
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.archive_dir settings.INTERNET_DRAFT_ARCHIVE_DIR = self.archive_dir
self.saved_yang_rfc_model_dir = settings.YANG_RFC_MODEL_DIR self.saved_yang_rfc_model_dir = settings.SUBMIT_YANG_RFC_MODEL_DIR
self.yang_rfc_model_dir = self.tempdir('yang-rfc-model') self.yang_rfc_model_dir = self.tempdir('yang-rfc-model')
settings.YANG_RFC_MODEL_DIR = self.yang_rfc_model_dir settings.SUBMIT_YANG_RFC_MODEL_DIR = self.yang_rfc_model_dir
self.saved_yang_draft_model_dir = settings.YANG_DRAFT_MODEL_DIR self.saved_yang_draft_model_dir = settings.SUBMIT_YANG_DRAFT_MODEL_DIR
self.yang_draft_model_dir = self.tempdir('yang-draft-model') self.yang_draft_model_dir = self.tempdir('yang-draft-model')
settings.YANG_DRAFT_MODEL_DIR = self.yang_draft_model_dir settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.yang_draft_model_dir
self.saved_yang_inval_model_dir = settings.YANG_INVAL_MODEL_DIR self.saved_yang_inval_model_dir = settings.SUBMIT_YANG_INVAL_MODEL_DIR
self.yang_inval_model_dir = self.tempdir('yang-inval-model') self.yang_inval_model_dir = self.tempdir('yang-inval-model')
settings.YANG_INVAL_MODEL_DIR = self.yang_inval_model_dir settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.yang_inval_model_dir
def tearDown(self): def tearDown(self):
shutil.rmtree(self.staging_dir) shutil.rmtree(self.staging_dir)
@ -91,9 +91,9 @@ class SubmitTests(TestCase):
settings.INTERNET_DRAFT_PATH = self.saved_internet_draft_path settings.INTERNET_DRAFT_PATH = self.saved_internet_draft_path
settings.IDSUBMIT_REPOSITORY_PATH = self.saved_idsubmit_repository_path settings.IDSUBMIT_REPOSITORY_PATH = self.saved_idsubmit_repository_path
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir
settings.YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir settings.SUBMIT_YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir
settings.YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir
settings.YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir
def do_submission(self, name, rev, group=None, formats=["txt",]): def do_submission(self, name, rev, group=None, formats=["txt",]):
@ -133,6 +133,8 @@ class SubmitTests(TestCase):
self.assertEqual(author["affiliation"], "Test Centre Inc.") self.assertEqual(author["affiliation"], "Test Centre Inc.")
self.assertEqual(author["country"], "UK") self.assertEqual(author["country"], "UK")
return status_url return status_url
def supply_extra_metadata(self, name, status_url, submitter_name, submitter_email, replaces): def supply_extra_metadata(self, name, status_url, submitter_name, submitter_email, replaces):
@ -220,6 +222,11 @@ class SubmitTests(TestCase):
r = self.client.get(status_url) r = self.client.get(status_url)
self.assertEqual(r.status_code, 200) self.assertEqual(r.status_code, 200)
self.assertContains(r, 'xym')
self.assertContains(r, 'pyang')
self.assertContains(r, 'yanglint')
q = PyQuery(r.content) q = PyQuery(r.content)
approve_button = q('[type=submit]:contains("Approve")') approve_button = q('[type=submit]:contains("Approve")')
self.assertEqual(len(approve_button), 1) self.assertEqual(len(approve_button), 1)

4
ietf/submit/views.py
View file

@ -165,7 +165,9 @@ def upload_submission(request):
def apply_check(submission, checker, method, fn): def apply_check(submission, checker, method, fn):
func = getattr(checker, method) func = getattr(checker, method)
passed, message, errors, warnings, items = func(fn) passed, message, errors, warnings, items = func(fn)
check = SubmissionCheck(submission=submission, checker=checker.name, passed=passed, message=message, errors=errors, warnings=warnings, items=items, symbol=checker.symbol) check = SubmissionCheck(submission=submission, checker=checker.name, passed=passed,
message=message, errors=errors, warnings=warnings, items=items,
symbol=checker.symbol)
check.save() check.save()
for checker_path in settings.IDSUBMIT_CHECKER_CLASSES: for checker_path in settings.IDSUBMIT_CHECKER_CLASSES:

30
ietf/templates/doc/document_draft.html
View file

@ -76,15 +76,6 @@
{% if latest_revision and latest_revision.time.date != doc.time.date %} {% if latest_revision and latest_revision.time.date != doc.time.date %}
(latest revision {{ latest_revision.time|date:"Y-m-d" }}) (latest revision {{ latest_revision.time|date:"Y-m-d" }})
{% endif %} {% endif %}
{% for check in doc.submission.checks.all %}
{% if check.passed != None and check.symbol.strip %}
{% if check.errors or check.warnings %}
<span class="checker-warning" title="Submission {{check.checker|title}} returned warnings or errors.">{{ check.symbol|safe }}</span>
{% else %}
<span class="checker-success" title="Submission {{check.checker|title}} passed">{{ check.symbol|safe }}</span>
{% endif %}
{% endif %}
{% endfor %}
</td> </td>
</tr> </tr>
@ -192,6 +183,27 @@
</td> </td>
</tr> </tr>
{% for check in doc.submission.latest_checks %}
{% if check.passed != None and check.symbol.strip %}
<tr>
<th></th>
<th>{{ check.checker|title }}</th>
<td class="edit"></td>
<td>
{% if check.errors or check.warnings %}
<span class="checker-warning" data-toggle="modal" data-target="#check-{{check.pk}}" title="{{check.checker|title}} returned warnings or errors." >{{ check.symbol|safe }}</span>
{% else %}
<span class="checker-success" data-toggle="modal" data-target="#check-{{check.pk}}" title="{{check.checker|title}} passed">{{ check.symbol|safe }}</span>
{% endif %}
<a href="#" data-toggle="modal" data-target="#check-{{check.pk}}">
{{ check.errors }} errors, {{ check.warnings }} warnings.
</a>
{% include "doc/yang-check-modal-overlay.html" %}
</td>
</tr>
{% endif %}
{% endfor %}
{% if review_requests or can_request_review %} {% if review_requests or can_request_review %}
<tr> <tr>
<th></th> <th></th>

7
ietf/templates/doc/search/search_result_row.html
View file

@ -56,13 +56,14 @@
{% if doc.latest_revision_date|timesince_days|new_enough:request and doc.get_state_slug != "rfc" %}</a>{% endif %} {% if doc.latest_revision_date|timesince_days|new_enough:request and doc.get_state_slug != "rfc" %}</a>{% endif %}
</span> </span>
{% for check in doc.submission.checks.all %} {% for check in doc.submission.latest_checks %}
{% if check.passed != None and check.symbol.strip %} {% if check.passed != None and check.symbol.strip %}
{% if check.errors or check.warnings %} {% if check.errors or check.warnings %}
<span class="checker-warning pull-right" title="{{check.checker|title}} returned warnings or errors.">{{ check.symbol|safe }}</span> <span class="checker-warning pull-right" data-toggle="modal" data-target="#check-{{check.pk}}" title="{{check.checker|title}} returned warnings or errors." >{{ check.symbol|safe }}</span>
{% else %} {% else %}
<span class="checker-success pull-right" title="{{check.checker|title}} passed">{{ check.symbol|safe }}</span> <span class="checker-success pull-right" data-toggle="modal" data-target="#check-{{check.pk}}" title="{{check.checker|title}} passed">{{ check.symbol|safe }}</span>
{% endif %} {% endif %}
{% include "doc/yang-check-modal-overlay.html" %}
{% endif %} {% endif %}
{% endfor %} {% endfor %}

17
ietf/templates/doc/yang-check-modal-overlay.html Normal file
View file

@ -0,0 +1,17 @@
{% load ietf_filters %}{% load origin %}{% origin %}
<div class="modal fade" id="check-{{check.pk}}" tabindex="-1" role="dialog" aria-labelledby="check-{{check.pk}}" aria-hidden="true">
<div class="modal-dialog modal-lg">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
<h4 class="modal-title" id="nitslabel">{{ check.checker|title }} for {{ doc.name }}-{{ doc.rev }} on {{ check.time|date:"Y-m-d" }}</h4>
</div>
<div class="modal-body">
<pre class="pasted">{{ check.message|zaptmp }}</pre>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>

4
ietf/templates/submit/submission_status.html
View file

@ -44,7 +44,7 @@
Your draft has <b>NOT</b> been verified to pass the submission checks. Your draft has <b>NOT</b> been verified to pass the submission checks.
{% endif %} {% endif %}
</p> </p>
{% for check in submission.checks.all %} {% for check in submission.latest_checks %}
{% if check.errors %} {% if check.errors %}
<p class="alert alert-warning"> <p class="alert alert-warning">
The {{check.checker}} returned {{ check.errors }} error{{ check.errors|pluralize }} The {{check.checker}} returned {{ check.errors }} error{{ check.errors|pluralize }}
@ -58,7 +58,7 @@
{% endif %} {% endif %}
{% endfor %} {% endfor %}
{% for check in submission.checks.all %} {% for check in submission.latest_checks %}
{% if check.passed != None %} {% if check.passed != None %}
<button class="btn btn-{% if check.passed %}{% if check.warnings %}warning{% elif check.errors %}warning{% else %}success{% endif %}{% else %}danger{% endif %}" data-toggle="modal" data-target="#check-{{check.pk}}">View {{ check.checker }}</button> <button class="btn btn-{% if check.passed %}{% if check.warnings %}warning{% elif check.errors %}warning{% else %}success{% endif %}{% else %}danger{% endif %}" data-toggle="modal" data-target="#check-{{check.pk}}">View {{ check.checker }}</button>

5
ietf/utils/admin.py
View file

@ -1,3 +1,5 @@
from django.contrib import admin
from ietf.utils.models import VersionInfo
def name(obj): def name(obj):
if hasattr(obj, 'abbrev'): if hasattr(obj, 'abbrev'):
@ -45,3 +47,6 @@ def admin_link(field, label=None, ordering="", display=name, suffix=""):
_link.admin_order_field = ordering _link.admin_order_field = ordering
return _link return _link
class VersionInfoAdmin(admin.ModelAdmin):
list_display = ['command', 'switch', 'version', 'time', ]
admin.site.register(VersionInfo, VersionInfoAdmin)

105
ietf/utils/management/commands/populate_yang_model_dirs.py
View file

@ -1,8 +1,10 @@
from __future__ import print_function, unicode_literals
import os import os
import sys import sys
import time import time
from pathlib import Path from pathlib2 import Path
from StringIO import StringIO from StringIO import StringIO
from textwrap import dedent from textwrap import dedent
from xym import xym from xym import xym
@ -10,9 +12,11 @@ from xym import xym
from django.conf import settings from django.conf import settings
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
class Command(BaseCommand): class Command(BaseCommand):
""" """
Populate the yang models repositories from drafts and RFCs. Populate the yang module repositories from drafts and RFCs.
Extracts yang models from RFCs (found in settings.RFC_PATH and places Extracts yang models from RFCs (found in settings.RFC_PATH and places
them in settings.YANG_RFC_MODEL_DIR, and from active drafts, placed in them in settings.YANG_RFC_MODEL_DIR, and from active drafts, placed in
@ -57,22 +61,34 @@ class Command(BaseCommand):
verbosity = int(options.get('verbosity')) verbosity = int(options.get('verbosity'))
def extract_from(file, dir, strict=True): def extract_from(file, dir, strict=True):
saved_stdout = sys.stdout
saved_stderr = sys.stderr saved_stderr = sys.stderr
sys.stderr = StringIO() xymerr = StringIO()
xymout = StringIO()
sys.stderr = xymerr
sys.stdout = xymout
model_list = [] model_list = []
try: try:
model_list = xym.xym(str(item), str(item.parent), str(dir), strict=strict, debug_level=(verbosity>1)) model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=verbosity-2)
for name in model_list: for name in model_list:
modfile = moddir / name modfile = moddir / name
mtime = item.stat().st_mtime mtime = file.stat().st_mtime
os.utime(str(modfile), (mtime, mtime)) os.utime(str(modfile), (mtime, mtime))
if '"' in name: if '"' in name:
name = name.replace('"', '') name = name.replace('"', '')
modfile.rename(str(moddir/name)) modfile.rename(str(moddir/name))
model_list = [ n.replace('"','') for n in model_list ] model_list = [ n.replace('"','') for n in model_list ]
except Exception as e: except Exception as e:
print("** Error when extracting from %s: %s" % (item, str(e))) print("** Error when extracting from %s: %s" % (file, str(e)))
sys.stdout = saved_stdout
sys.stderr = saved_stderr sys.stderr = saved_stderr
#
if verbosity > 1:
outmsg = xymout.getvalue()
self.stdout.write(outmsg)
if verbosity>2:
errmsg = xymerr.getvalue()
self.stderr.write(errmsg)
return model_list return model_list
# Extract from new RFCs # Extract from new RFCs
@ -87,10 +103,10 @@ class Command(BaseCommand):
for item in moddir.iterdir(): for item in moddir.iterdir():
if item.stat().st_mtime > latest: if item.stat().st_mtime > latest:
latest = item.stat().st_mtime latest = item.stat().st_mtime
print("Extracting to %s ..." % moddir) print("Extracting to %s ..." % moddir)
for item in rfcdir.iterdir(): for item in rfcdir.iterdir():
if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt'): if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit():
if item.stat().st_mtime > latest: if item.stat().st_mtime > latest:
model_list = extract_from(item, moddir) model_list = extract_from(item, moddir)
for name in model_list: for name in model_list:
@ -124,20 +140,25 @@ class Command(BaseCommand):
print("Extracting to %s ..." % moddir) print("Extracting to %s ..." % moddir)
for item in draftdir.iterdir(): for item in draftdir.iterdir():
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): try:
model_list = extract_from(item, moddir) if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
for name in model_list: model_list = extract_from(item, moddir)
if not name.startswith('example'): for name in model_list:
if verbosity > 1: if not name.startswith('example'):
print(" Extracted valid module from %s: %s" % (item, name)) if verbosity > 1:
print(" Extracted valid module from %s: %s" % (item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else: else:
sys.stdout.write('.') modfile = moddir / name
sys.stdout.flush() modfile.unlink()
else: if verbosity > 1:
modfile = moddir / name print(" Skipped module from %s: %s" % (item, name))
modfile.unlink() except UnicodeDecodeError as e:
if verbosity > 1: sys.stderr.write('\nError: %s\n' % (e, ))
print(" Skipped module from %s: %s" % (item, name)) sys.stderr.write(item.name)
sys.stderr.write('\n')
print("") print("")
# Extract invalid modules from drafts # Extract invalid modules from drafts
@ -151,22 +172,28 @@ class Command(BaseCommand):
print("Extracting to %s ..." % moddir) print("Extracting to %s ..." % moddir)
for item in draftdir.iterdir(): for item in draftdir.iterdir():
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item): try:
model_list = extract_from(item, moddir, strict=False) if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
for name in model_list: model_list = extract_from(item, moddir, strict=False)
modfile = moddir / name for name in model_list:
if (valdir/name).exists(): modfile = moddir / name
modfile.unlink() if (valdir/name).exists():
if verbosity > 1: modfile.unlink()
print(" Skipped valid module from %s: %s" % (item, name)) if verbosity > 1:
elif not name.startswith('example'): print(" Skipped valid module from %s: %s" % (item, name))
if verbosity > 1: elif not name.startswith('example'):
print(" Extracted invalid module from %s: %s" % (item, name)) if verbosity > 1:
print(" Extracted invalid module from %s: %s" % (item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else: else:
sys.stdout.write('.') modfile.unlink()
sys.stdout.flush() if verbosity > 1:
else: print(" Skipped module from %s: %s" % (item, name))
modfile.unlink() except UnicodeDecodeError as e:
if verbosity > 1: sys.stderr.write('\nError: %s\n' % (e, ))
print(" Skipped module from %s: %s" % (item, name)) sys.stderr.write(item.name)
sys.stderr.write('\n')
print("") print("")

82
ietf/utils/management/commands/run_yang_model_checks.py Normal file
View file

@ -0,0 +1,82 @@
from __future__ import print_function, unicode_literals
import sys
import json
from textwrap import dedent
from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
from ietf.doc.models import Document, State
from ietf.submit.models import Submission, SubmissionCheck
from ietf.submit.checkers import DraftYangChecker
class Command(BaseCommand):
"""
Run yang model checks on active drafts.
Repeats the yang checks in ietf/submit/checkers.py for active drafts, in
order to catch changes in status due to new modules becoming available in
the module directories.
"""
help = dedent(__doc__).strip()
def add_arguments(self, parser):
parser.add_argument('filenames', nargs="*")
parser.add_argument('--clean',
action='store_true', dest='clean', default=False,
help='Remove the current directory content before writing new models.')
def check_yang(self, checker, draft, force=False):
if self.verbosity > 1:
self.stdout.write("Checking %s-%s" % (draft.name, draft.rev))
else:
sys.stderr.write('.')
submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first()
if submission or force:
check = submission.checks.filter(checker=checker.name).order_by('-id').first()
if check or force:
result = checker.check_file_txt(draft.get_file_name())
passed, message, errors, warnings, items = result
if self.verbosity > 2:
self.stdout.write(" Errors: %s\n"
" Warnings: %s\n"
" Message:\n%s\n" % (errors, warnings, message))
items = json.loads(json.dumps(items))
new_res = (passed, errors, warnings, message)
old_res = (check.passed, check.errors, check.warnings, check.message) if check else ()
if new_res != old_res:
if self.verbosity > 1:
self.stdout.write(" Saving new yang checker results for %s-%s" % (draft.name, draft.rev))
SubmissionCheck.objects.create(submission=submission, checker=checker.name, passed=passed,
message=message, errors=errors, warnings=warnings, items=items,
symbol=checker.symbol)
else:
self.stderr.write("Error: did not find any submission object for %s-%s\n" % (draft.name, draft.rev))
def handle(self, *filenames, **options):
"""
"""
self.verbosity = int(options.get('verbosity'))
filenames = options.get('filenames')
active_state = State.objects.get(type="draft", slug="active")
checker = DraftYangChecker()
if filenames:
for name in filenames:
parts = name.rsplit('-',1)
if len(parts)==2 and len(parts[1])==2 and parts[1].isdigit():
name = parts[0]
draft = Document.objects.get(name=name)
self.check_yang(checker, draft, force=True)
else:
for draft in Document.objects.filter(states=active_state, type_id='draft'):
self.check_yang(checker, draft)

37
ietf/utils/management/commands/update_external_command_info.py Normal file
View file

@ -0,0 +1,37 @@
from __future__ import print_function, unicode_literals
import sys
from textwrap import dedent
from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
from ietf.utils.models import VersionInfo
from ietf.utils.pipe import pipe
class Command(BaseCommand):
"""
Update the version information for external commands used by the datatracker.
Iterates through the entries in the VersionInfo table, runs the relevant
command, and updates the version string with the result.
"""
help = dedent(__doc__).strip()
def handle(self, *filenames, **options):
for c in VersionInfo.objects.filter(used=True):
cmd = "%s %s" % (c.command, c.switch)
code, out, err = pipe(cmd)
if code != 0:
sys.stderr.write("Command '%s' retuned %s: \n%s\n%s\n" % (cmd, code, out, err))
else:
c.version = (out.strip()+'\n'+err.strip()).strip()
if options.get('verbosity', 1) > 1:
sys.stdout.write(
"Command: %s\n"
" Version: %s\n" % (cmd, c.version))
c.save()

26
ietf/utils/migrations/0003_versioninfo.py Normal file
View file

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.10.7 on 2017-06-15 03:31
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('utils', '0002_dumpinfo_tz'),
]
operations = [
migrations.CreateModel(
name='VersionInfo',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('time', models.DateTimeField(auto_now=True)),
('command', models.CharField(max_length=32)),
('switch', models.CharField(max_length=16)),
('version', models.CharField(max_length=64)),
('used', models.BooleanField(default=True)),
],
),
]

9
ietf/utils/models.py
View file

@ -7,3 +7,12 @@ class DumpInfo(models.Model):
host = models.CharField(max_length=128) host = models.CharField(max_length=128)
tz = models.CharField(max_length=32, default='UTC') tz = models.CharField(max_length=32, default='UTC')
class VersionInfo(models.Model):
time = models.DateTimeField(auto_now=True)
command = models.CharField(max_length=32)
switch = models.CharField(max_length=16)
version = models.CharField(max_length=64)
used = models.BooleanField(default=True)
class Meta:
verbose_name_plural = 'VersionInfo'

21
ietf/utils/resources.py
View file

@ -8,7 +8,8 @@ from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from ietf import api from ietf import api
from ietf.utils.models import DumpInfo from ietf.utils.models import DumpInfo, VersionInfo
class UserResource(ModelResource): class UserResource(ModelResource):
username = CharField() username = CharField()
@ -17,6 +18,7 @@ class UserResource(ModelResource):
queryset = User.objects.all() queryset = User.objects.all()
serializer = api.Serializer() serializer = api.Serializer()
class ContentTypeResource(ModelResource): class ContentTypeResource(ModelResource):
username = CharField() username = CharField()
class Meta: class Meta:
@ -24,6 +26,7 @@ class ContentTypeResource(ModelResource):
queryset = ContentType.objects.all() queryset = ContentType.objects.all()
serializer = api.Serializer() serializer = api.Serializer()
class DumpInfoResource(ModelResource): class DumpInfoResource(ModelResource):
class Meta: class Meta:
cache = SimpleCache() cache = SimpleCache()
@ -36,3 +39,19 @@ class DumpInfoResource(ModelResource):
} }
api.utils.register(DumpInfoResource()) api.utils.register(DumpInfoResource())
class VersionInfoResource(ModelResource):
class Meta:
queryset = VersionInfo.objects.all()
serializer = api.Serializer()
cache = SimpleCache()
#resource_name = 'versioninfo'
filtering = {
"id": ALL,
"time": ALL,
"command": ALL,
"switch": ALL,
"version": ALL,
"used": ALL,
}
api.utils.register(VersionInfoResource())

1
requirements.txt
View file

@ -31,6 +31,7 @@ mimeparse>=0.1.3 # from TastyPie
mock>=2.0.0 mock>=2.0.0
MySQL-python>=1.2.5 MySQL-python>=1.2.5
pathlib>=1.0 pathlib>=1.0
pathlib2>=2.3.0
Pillow>=3.0 Pillow>=3.0
pyang>=1.6 pyang>=1.6
pyflakes>=0.8.1 pyflakes>=0.8.1