altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Changed handling of yang modeules from drafts with errors, to not place them in separate directories, due to possible follow-on effects. Added use of IANA-held modules when validating, and regular rsyncing from IANA.

Browse source 
- Legacy-Id: 14389
This commit is contained in:
Henrik Levkowetz 2017-12-04 14:37:15 +00:00
parent 8a43f7aedf
commit 47a0b9d3ce
6 changed files with 21 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bin/daily
View file

@ -22,6 +22,10 @@ source $DTDIR/env/bin/activate
# Update our information about the current version of some commands we use
$DTDIR/ietf/manage.py update_external_command_info
# Get IANA-registered yang models
YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
rsync -avz --delete rsync.ietf.org::iana/yang-parameters $YANG_IANA_DIR/
# Populate the yang repositories
$DTDIR/ietf/manage.py populate_yang_model_dirs

2
ietf/checks.py
View file

@ -145,7 +145,7 @@ def check_yang_model_directories(app_configs, **kwargs):
return []
#
errors = []
for s in ("SUBMIT_YANG_RFC_MODEL_DIR", "SUBMIT_YANG_DRAFT_MODEL_DIR", "SUBMIT_YANG_INVAL_MODEL_DIR", ):
for s in ("SUBMIT_YANG_RFC_MODEL_DIR", "SUBMIT_YANG_DRAFT_MODEL_DIR", "SUBMIT_YANG_IANA_MODEL_DIR", ):
p = getattr(settings, s)
if not os.path.exists(p):
errors.append(checks.Critical(

4
ietf/settings.py
View file

@ -677,14 +677,14 @@ IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21)
# they (mainly) belong. So here, SUBMIT_, rather than IDSUBMIT_
SUBMIT_YANG_RFC_MODEL_DIR = '/a/www/ietf-ftp/yang/rfcmod/'
SUBMIT_YANG_DRAFT_MODEL_DIR = '/a/www/ietf-ftp/yang/draftmod/'
SUBMIT_YANG_INVAL_MODEL_DIR = '/a/www/ietf-ftp/yang/invalmod/'
SUBMIT_YANG_IANA_MODEL_DIR = '/a/www/ietf-ftp/yang/ianamod/'
IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
SUBMIT_PYANG_COMMAND = 'pyang --verbose --ietf -p {libs} {model}'
SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {tmplib} -p {rfclib} -p {draftlib} -p {invallib} {model} -i'
SUBMIT_YANGLINT_COMMAND = 'yanglint --verbose -p {tmplib} -p {rfclib} -p {draftlib} -p {ianalib} {model} -i'
SUBMIT_YANGLINT_COMMAND = None # use the value above if you have yanglint installed
SUBMIT_YANG_CATALOG_MODULEARG = "modules[]={module}"

13
ietf/submit/checkers.py
View file

@ -192,7 +192,7 @@ class DraftYangChecker(object):
workdir,
settings.SUBMIT_YANG_RFC_MODEL_DIR,
settings.SUBMIT_YANG_DRAFT_MODEL_DIR,
settings.SUBMIT_YANG_INVAL_MODEL_DIR,
settings.SUBMIT_YANG_IANA_MODEL_DIR,
])
if os.path.exists(path):
with open(path) as file:
@ -230,7 +230,7 @@ class DraftYangChecker(object):
command = [ w for w in cmd_template.split() if not '=' in w ][0]
cmd_version = VersionInfo.objects.get(command=command).version
cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir,
draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, invallib=settings.SUBMIT_YANG_INVAL_MODEL_DIR, )
draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, )
code, out, err = pipe(cmd)
if code > 0 or len(err.strip()) > 0:
error_lines = err.splitlines()
@ -248,14 +248,9 @@ class DraftYangChecker(object):
else:
errors += 1
message += "No such file: %s\nPossible mismatch between extracted xym file name and returned module name?\n" % (path)
if errors==0 and warnings==0:
dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model)
shutil.move(path, dest)
else:
dest = os.path.join(settings.SUBMIT_YANG_INVAL_MODEL_DIR, model)
shutil.move(path, dest)
dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model)
shutil.move(path, dest)
# summary result
results.append({

10
ietf/submit/tests.py
View file

@ -92,9 +92,9 @@ class SubmitTests(TestCase):
self.yang_draft_model_dir = self.tempdir('yang-draft-model')
settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.yang_draft_model_dir
self.saved_yang_inval_model_dir = settings.SUBMIT_YANG_INVAL_MODEL_DIR
self.yang_inval_model_dir = self.tempdir('yang-inval-model')
settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.yang_inval_model_dir
self.saved_yang_iana_model_dir = settings.SUBMIT_YANG_IANA_MODEL_DIR
self.yang_iana_model_dir = self.tempdir('yang-iana-model')
settings.SUBMIT_YANG_IANA_MODEL_DIR = self.yang_iana_model_dir
def tearDown(self):
shutil.rmtree(self.staging_dir)
@ -102,14 +102,14 @@ class SubmitTests(TestCase):
shutil.rmtree(self.archive_dir)
shutil.rmtree(self.yang_rfc_model_dir)
shutil.rmtree(self.yang_draft_model_dir)
shutil.rmtree(self.yang_inval_model_dir)
shutil.rmtree(self.yang_iana_model_dir)
settings.IDSUBMIT_STAGING_PATH = self.saved_idsubmit_staging_path
settings.INTERNET_DRAFT_PATH = self.saved_internet_draft_path
settings.IDSUBMIT_REPOSITORY_PATH = self.saved_idsubmit_repository_path
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir
settings.SUBMIT_YANG_RFC_MODEL_DIR = self.saved_yang_rfc_model_dir
settings.SUBMIT_YANG_DRAFT_MODEL_DIR = self.saved_yang_draft_model_dir
settings.SUBMIT_YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir
settings.SUBMIT_YANG_IANA_MODEL_DIR = self.saved_yang_iana_model_dir
def do_submission(self, name, rev, group=None, formats=["txt",], author=None):

47
ietf/utils/management/commands/populate_yang_model_dirs.py
View file

@ -20,8 +20,7 @@ class Command(BaseCommand):
Extracts yang models from RFCs (found in settings.RFC_PATH and places
them in settings.SUBMIT_YANG_RFC_MODEL_DIR, and from active drafts, placed in
settings.SUBMIT_YANG_DRAFT_MODEL_DIR if valid and settings.SUBMIT_YANG_INVAL_MODEL_DIR
if not.
settings.SUBMIT_YANG_DRAFT_MODEL_DIR.
"""
@ -142,14 +141,14 @@ class Command(BaseCommand):
for item in draftdir.iterdir():
try:
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir)
model_list = extract_from(item, moddir, strict=False)
for name in model_list:
if not name.startswith('example'):
if verbosity > 1:
print(" Extracted valid module from %s: %s" % (item, name))
print(" Extracted module from %s: %s" % (item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
sys.stdout.write('.')
sys.stdout.flush()
else:
modfile = moddir / name
modfile.unlink()
@ -161,39 +160,3 @@ class Command(BaseCommand):
sys.stderr.write('\n')
print("")
# Extract invalid modules from drafts
valdir = moddir
moddir = Path(settings.SUBMIT_YANG_INVAL_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
print("Emptying %s ..." % moddir)
for item in moddir.iterdir():
item.unlink()
print("Extracting to %s ..." % moddir)
for item in draftdir.iterdir():
try:
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir, strict=False)
for name in model_list:
modfile = moddir / name
if (valdir/name).exists():
modfile.unlink()
if verbosity > 1:
print(" Skipped valid module from %s: %s" % (item, name))
elif not name.startswith('example'):
if verbosity > 1:
print(" Extracted invalid module from %s: %s" % (item, name))
else:
sys.stdout.write('.')
sys.stdout.flush()
else:
modfile.unlink()
if verbosity > 1:
print(" Skipped module from %s: %s" % (item, name))
except UnicodeDecodeError as e:
sys.stderr.write('\nError: %s\n' % (e, ))
sys.stderr.write(item.name)
sys.stderr.write('\n')
print("")