altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Added a django management command to populate yang model libraries from RFCs and drafts.

Browse source 
- Legacy-Id: 11110
This commit is contained in:
Henrik Levkowetz 2016-04-08 17:18:54 +00:00
parent 658a5cecbb
commit d96f0fe964
4 changed files with 210 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
ietf/settings.py
View file

@ -45,6 +45,10 @@ ADMINS = (
ALLOWED_HOSTS = [".ietf.org", ".ietf.org.", "209.208.19.216", "4.31.198.44", ]
# This is used to construct the path to manage.py, in order to run management
# commands, for instance in etc/cron.d/datatracker
DEPLOYMENT_DIR = '/a/www/ietf-datatracker'
# Server name of the tools server
TOOLS_SERVER = 'tools.' + IETF_DOMAIN
TOOLS_SERVER_URL = 'https://' + TOOLS_SERVER
@ -472,7 +476,7 @@ IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
IDSUBMIT_PYANG_COMMAND = 'pyang -p %(workdir)s --verbose --ietf %(model)s'
IDSUBMIT_PYANG_COMMAND = 'pyang -p %(modpath)s --verbose --ietf %(model)s'
IDSUBMIT_CHECKER_CLASSES = (
"ietf.submit.checkers.DraftIdnitsChecker",
@ -504,6 +508,10 @@ IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE = 450 # in MB
IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000
IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB
YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/'
XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/"
MEETING_MATERIALS_SUBMISSION_START_DAYS = -90

8
ietf/submit/checkers.py
View file

@ -138,9 +138,15 @@ class DraftYangChecker(object):
for model in model_list:
path = os.path.join(workdir, model)
modpath = ':'.join([
workdir,
settings.YANG_RFC_MODEL_DIR,
settings.YANG_DRAFT_MODEL_DIR,
settings.YANG_INVAL_MODEL_DIR,
])
with open(path) as file:
text = file.readlines()
cmd = settings.IDSUBMIT_PYANG_COMMAND % {"workdir": workdir, "model": path, }
cmd = settings.IDSUBMIT_PYANG_COMMAND % {"modpath": modpath, "model": path, }
code, out, err = pipe(cmd)
errors = 0
warnings = 0

25
ietf/submit/tests.py
View file

@ -1,13 +1,14 @@
import datetime
import os
import shutil
from django.conf import settings
from django.core.urlresolvers import reverse as urlreverse
import sys
from StringIO import StringIO
from pyquery import PyQuery
from django.conf import settings
from django.core.urlresolvers import reverse as urlreverse
from django.core.management import call_command
import debug # pyflakes:ignore
from ietf.utils.test_utils import login_testing_unauthorized, unicontent
@ -39,14 +40,21 @@ class SubmitTests(TestCase):
os.mkdir(self.archive_dir)
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.archive_dir
self.saved_yang_model_dir = settings.YANG_DRAFT_MODEL_DIR
self.yang_model_dir = os.path.abspath("tmp-yang-model-dir")
os.mkdir(self.yang_model_dir)
settings.YANG_DRAFT_MODEL_DIR = self.yang_model_dir
def tearDown(self):
shutil.rmtree(self.staging_dir)
shutil.rmtree(self.repository_dir)
shutil.rmtree(self.archive_dir)
shutil.rmtree(self.yang_model_dir)
settings.IDSUBMIT_STAGING_PATH = self.saved_idsubmit_staging_path
settings.INTERNET_DRAFT_PATH = self.saved_internet_draft_path
settings.IDSUBMIT_REPOSITORY_PATH = self.saved_idsubmit_repository_path
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir
settings.YANG_DRAFT_MODEL_DIR = self.saved_yang_model_dir
def submission_file(self, name, rev, group, format, templatename):
@ -235,6 +243,15 @@ class SubmitTests(TestCase):
def test_submit_new_wg_txt(self):
self.submit_new_wg(["txt"])
# Test yang module extraction management command
saved_sys_stdout = sys.stdout
sys.stdout = StringIO()
call_command('populate_yang_model_dirs')
self.assertIn('Extracting to', sys.stdout.getvalue())
sys.stdout.close()
sys.stdout = saved_sys_stdout
self.assertTrue(os.path.exists(os.path.join(settings.YANG_DRAFT_MODEL_DIR, "ietf-mpls@2015-10-16.yang")))
def text_submit_new_wg_xml(self):
self.submit_new_wg(["xml"])

173
ietf/utils/management/commands/populate_yang_model_dirs.py Normal file
View file

@ -0,0 +1,173 @@
import os
import sys
import time
from optparse import make_option
from pathlib import Path
from StringIO import StringIO
from textwrap import dedent
from xym import xym
from django.conf import settings
from django.core.management.base import BaseCommand
class Command(BaseCommand):
"""
Populate the yang models repositories from drafts and RFCs.
Extracts yang models from RFCs (found in settings.RFC_PATH and places
them in settings.YANG_RFC_MODEL_DIR, and from active drafts, placed in
settings.YANG_DRAFT_MODEL_DIR if valid and settings.YANG_INVAL_MODEL_DIR
if not.
"""
help = dedent(__doc__).strip()
option_list = BaseCommand.option_list + (
make_option('--clean',
action='store_true', dest='clean', default=False,
help='Remove the current directory content before writing new models.'),
)
def handle(self, *filenames, **options):
"""
* All yang modules from published RFCs should be extracted and be
available in an rfc-yang repository.
* All valid yang modules from active, not replaced, internet drafts
should be extracted and be available in a draft-valid-yang repository.
* All, valid and invalid, yang modules from active, not replaced,
internet drafts should be available in a draft-all-yang repository.
(Actually, given precedence ordering, it would be enough to place
non-validating modules in a draft-invalid-yang repository instead).
* In all cases, example modules should be excluded.
* Precedence is established by the search order of the repository as
provided to pyang.
* As drafts expire, models should be removed in order to catch cases
where a module being worked on depends on one which has slipped out
of the work queue.
"""
verbosity = int(options.get('verbosity'))
def extract_from(file, dir, strict=True):
saved_stderr = sys.stderr
sys.stderr = StringIO()
model_list = []
try:
model_list = xym.xym(str(item), str(item.parent), str(dir), strict=strict, debug_level=(verbosity>1))
for name in model_list:
modfile = moddir / name
mtime = item.stat().st_mtime
os.utime(str(modfile), (mtime, mtime))
if '"' in name:
name = name.replace('"', '')
modfile.rename(str(moddir/name))
model_list = [ n.replace('"','') for n in model_list ]
except Exception as e:
print("** Error when extracting from %s: %s" % (item, str(e)))
sys.stderr = saved_stderr
return model_list
# Extract from new RFCs
rfcdir = Path(settings.RFC_PATH)
moddir = Path(settings.YANG_RFC_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
latest = 0
for item in moddir.iterdir():
if item.stat().st_mtime > latest:
latest = item.stat().st_mtime
print("Extracting to %s ..." % moddir)
for item in rfcdir.iterdir():
if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt'):
if item.stat().st_mtime > latest:
model_list = extract_from(item, moddir)
for name in model_list:
if name.startswith('ietf') or name.startswith('iana'):
if verbosity > 1:
print(" Extracted from %s: %s" % (item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
modfile = moddir / name
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" % (item, name))
print("")
# Extract valid modules from drafts
six_months_ago = time.time() - 6*31*24*60*60
def active(item):
return item.stat().st_mtime > six_months_ago
draftdir = Path(settings.INTERNET_DRAFT_PATH)
moddir = Path(settings.YANG_DRAFT_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
print("Emptying %s ..." % moddir)
for item in moddir.iterdir():
item.unlink()
print("Extracting to %s ..." % moddir)
for item in draftdir.iterdir():
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir)
for name in model_list:
if not name.startswith('example'):
if verbosity > 1:
print(" Extracted valid module from %s: %s" % (item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
modfile = moddir / name
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" % (item, name))
print("")
# Extract invalid modules from drafts
valdir = moddir
moddir = Path(settings.YANG_INVAL_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
print("Emptying %s ..." % moddir)
for item in moddir.iterdir():
item.unlink()
print("Extracting to %s ..." % moddir)
for item in draftdir.iterdir():
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir, strict=False)
for name in model_list:
modfile = moddir / name
if (valdir/name).exists():
modfile.unlink()
if verbosity > 1:
print(" Skipped valid module from %s: %s" % (item, name))
elif not name.startswith('example'):
if verbosity > 1:
print(" Extracted invalid module from %s: %s" % (item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" % (item, name))
print("")