altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

ci: merge main to release (#7560)

Browse source 
ci: merge main to release
This commit is contained in:
Robert Sparks 2024-06-18 15:08:15 -05:00 committed by GitHub
parent 99b3f628bf b78ec6dff8
commit d04e542d18
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 445 additions and 530 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
bin/daily
View file

@ -5,7 +5,6 @@
# This script is expected to be triggered by cron from
# /etc/cron.d/datatracker
export LANG=en_US.UTF-8
export PYTHONIOENCODING=utf-8
# Make sure we stop if something goes wrong:
program=${0##*/}
@ -17,10 +16,6 @@ cd $DTDIR/
logger -p user.info -t cron "Running $DTDIR/bin/daily"
# Set up the virtual environment
source $DTDIR/env/bin/activate
# Get IANA-registered yang models
#YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
# Hardcode the rsync target to avoid any unwanted deletes:
@ -30,9 +25,3 @@ rsync -avzq --delete /a/www/ietf-ftp/iana/yang-parameters/ /a/www/ietf-ftp/yang/
# Get Yang models from Yangcatalog.
#rsync -avzq rsync://rsync.yangcatalog.org:10873/yangdeps /a/www/ietf-ftp/yang/catalogmod/
/a/www/ietf-datatracker/scripts/sync_to_yangcatalog
# Populate the yang repositories
$DTDIR/ietf/manage.py populate_yang_model_dirs -v0
# Re-run yang checks on active documents
$DTDIR/ietf/manage.py run_yang_model_checks -v0

55
ietf/checks.py
View file

@ -28,61 +28,6 @@ def already_ran():
checks_run.append(name)
return False
@checks.register('files')
def check_group_email_aliases_exists(app_configs, **kwargs):
from ietf.group.views import check_group_email_aliases
#
if already_ran():
return []
#
errors = []
try:
ok = check_group_email_aliases()
if not ok:
errors.append(checks.Error(
"Found no aliases in the group email aliases file\n'%s'."%settings.GROUP_ALIASES_PATH,
hint="These should be created by the infrastructure using ietf/bin/aliases-from-json.py.",
obj=None,
id="datatracker.E0002",
))
except IOError as e:
errors.append(checks.Error(
"Could not read group email aliases:\n %s" % e,
hint="These should be created by the infrastructure using ietf/bin/aliases-from-json.py.",
obj=None,
id="datatracker.E0003",
))
return errors
@checks.register('files')
def check_doc_email_aliases_exists(app_configs, **kwargs):
from ietf.doc.views_doc import check_doc_email_aliases
#
if already_ran():
return []
#
errors = []
try:
ok = check_doc_email_aliases()
if not ok:
errors.append(checks.Error(
"Found no aliases in the document email aliases file\n'%s'."%settings.DRAFT_VIRTUAL_PATH,
hint="These should be created by the infrastructure using ietf/bin/aliases-from-json.py.",
obj=None,
id="datatracker.E0004",
))
except IOError as e:
errors.append(checks.Error(
"Could not read document email aliases:\n %s" % e,
hint="These should be created by the infrastructure using ietf/bin/aliases-from-json.py.",
obj=None,
id="datatracker.E0005",
))
return errors
@checks.register('directories')
def check_id_submission_directories(app_configs, **kwargs):
#

139
ietf/doc/tests.py
View file

@ -16,7 +16,6 @@ from http.cookies import SimpleCookie
from pathlib import Path
from pyquery import PyQuery
from urllib.parse import urlparse, parse_qs
from tempfile import NamedTemporaryFile
from collections import defaultdict
from zoneinfo import ZoneInfo
@ -51,6 +50,7 @@ from ietf.doc.utils import (
DraftAliasGenerator,
generate_idnits2_rfc_status,
generate_idnits2_rfcs_obsoleted,
get_doc_email_aliases,
)
from ietf.group.models import Group, Role
from ietf.group.factories import GroupFactory, RoleFactory
@ -2169,24 +2169,6 @@ class ReferencesTest(TestCase):
self.assertContains(r, doc1.name)
class GenerateDraftAliasesTests(TestCase):
def setUp(self):
super().setUp()
self.doc_aliases_file = NamedTemporaryFile(delete=False, mode="w+")
self.doc_aliases_file.close()
self.doc_virtual_file = NamedTemporaryFile(delete=False, mode="w+")
self.doc_virtual_file.close()
self.saved_draft_aliases_path = settings.DRAFT_ALIASES_PATH
self.saved_draft_virtual_path = settings.DRAFT_VIRTUAL_PATH
settings.DRAFT_ALIASES_PATH = self.doc_aliases_file.name
settings.DRAFT_VIRTUAL_PATH = self.doc_virtual_file.name
def tearDown(self):
settings.DRAFT_ALIASES_PATH = self.saved_draft_aliases_path
settings.DRAFT_VIRTUAL_PATH = self.saved_draft_virtual_path
os.unlink(self.doc_aliases_file.name)
os.unlink(self.doc_virtual_file.name)
super().tearDown()
@override_settings(TOOLS_SERVER="tools.example.org", DRAFT_ALIAS_DOMAIN="draft.example.org")
def test_generator_class(self):
"""The DraftAliasGenerator should generate the same lists as the old mgmt cmd"""
@ -2286,6 +2268,28 @@ class GenerateDraftAliasesTests(TestCase):
{k: sorted(v) for k, v in expected_dict.items()},
)
# check single name
output = [(alias, alist) for alias, alist in DraftAliasGenerator(Document.objects.filter(name=doc1.name))]
alias_dict = dict(output)
self.assertEqual(len(alias_dict), len(output)) # no duplicate aliases
expected_dict = {
doc1.name: [author1.email_address()],
doc1.name + ".ad": [ad.email_address()],
doc1.name + ".authors": [author1.email_address()],
doc1.name + ".shepherd": [shepherd.email_address()],
doc1.name
+ ".all": [
author1.email_address(),
ad.email_address(),
shepherd.email_address(),
],
}
# Sort lists for comparison
self.assertEqual(
{k: sorted(v) for k, v in alias_dict.items()},
{k: sorted(v) for k, v in expected_dict.items()},
)
@override_settings(TOOLS_SERVER="tools.example.org", DRAFT_ALIAS_DOMAIN="draft.example.org")
def test_get_draft_notify_emails(self):
ad = PersonFactory()
@ -2336,37 +2340,20 @@ class EmailAliasesTests(TestCase):
WgDraftFactory(name='draft-ietf-mars-test',group__acronym='mars')
WgDraftFactory(name='draft-ietf-ames-test',group__acronym='ames')
RoleFactory(group__type_id='review', group__acronym='yangdoctors', name_id='secr')
self.doc_alias_file = NamedTemporaryFile(delete=False, mode='w+')
self.doc_alias_file.write("""# Generated by hand at 2015-02-12_16:26:45
virtual.ietf.org anything
draft-ietf-mars-test@ietf.org xfilter-draft-ietf-mars-test
expand-draft-ietf-mars-test@virtual.ietf.org mars-author@example.com, mars-collaborator@example.com
draft-ietf-mars-test.authors@ietf.org xfilter-draft-ietf-mars-test.authors
expand-draft-ietf-mars-test.authors@virtual.ietf.org mars-author@example.mars, mars-collaborator@example.mars
draft-ietf-mars-test.chairs@ietf.org xfilter-draft-ietf-mars-test.chairs
expand-draft-ietf-mars-test.chairs@virtual.ietf.org mars-chair@example.mars
draft-ietf-mars-test.all@ietf.org xfilter-draft-ietf-mars-test.all
expand-draft-ietf-mars-test.all@virtual.ietf.org mars-author@example.mars, mars-collaborator@example.mars, mars-chair@example.mars
draft-ietf-ames-test@ietf.org xfilter-draft-ietf-ames-test
expand-draft-ietf-ames-test@virtual.ietf.org ames-author@example.com, ames-collaborator@example.com
draft-ietf-ames-test.authors@ietf.org xfilter-draft-ietf-ames-test.authors
expand-draft-ietf-ames-test.authors@virtual.ietf.org ames-author@example.ames, ames-collaborator@example.ames
draft-ietf-ames-test.chairs@ietf.org xfilter-draft-ietf-ames-test.chairs
expand-draft-ietf-ames-test.chairs@virtual.ietf.org ames-chair@example.ames
draft-ietf-ames-test.all@ietf.org xfilter-draft-ietf-ames-test.all
expand-draft-ietf-ames-test.all@virtual.ietf.org ames-author@example.ames, ames-collaborator@example.ames, ames-chair@example.ames
""")
self.doc_alias_file.close()
self.saved_draft_virtual_path = settings.DRAFT_VIRTUAL_PATH
settings.DRAFT_VIRTUAL_PATH = self.doc_alias_file.name
def tearDown(self):
settings.DRAFT_VIRTUAL_PATH = self.saved_draft_virtual_path
os.unlink(self.doc_alias_file.name)
super().tearDown()
def testAliases(self):
@mock.patch("ietf.doc.views_doc.get_doc_email_aliases")
def testAliases(self, mock_get_aliases):
mock_get_aliases.return_value = [
{"doc_name": "draft-ietf-mars-test", "alias_type": "", "expansion": "mars-author@example.mars, mars-collaborator@example.mars"},
{"doc_name": "draft-ietf-mars-test", "alias_type": ".authors", "expansion": "mars-author@example.mars, mars-collaborator@example.mars"},
{"doc_name": "draft-ietf-mars-test", "alias_type": ".chairs", "expansion": "mars-chair@example.mars"},
{"doc_name": "draft-ietf-mars-test", "alias_type": ".all", "expansion": "mars-author@example.mars, mars-collaborator@example.mars, mars-chair@example.mars"},
{"doc_name": "draft-ietf-ames-test", "alias_type": "", "expansion": "ames-author@example.ames, ames-collaborator@example.ames"},
{"doc_name": "draft-ietf-ames-test", "alias_type": ".authors", "expansion": "ames-author@example.ames, ames-collaborator@example.ames"},
{"doc_name": "draft-ietf-ames-test", "alias_type": ".chairs", "expansion": "ames-chair@example.ames"},
{"doc_name": "draft-ietf-ames-test", "alias_type": ".all", "expansion": "ames-author@example.ames, ames-collaborator@example.ames, ames-chair@example.ames"},
]
PersonFactory(user__username='plain')
url = urlreverse('ietf.doc.urls.redirect.document_email', kwargs=dict(name="draft-ietf-mars-test"))
r = self.client.get(url)
@ -2376,16 +2363,70 @@ expand-draft-ietf-ames-test.all@virtual.ietf.org ames-author@example.ames, ames
login_testing_unauthorized(self, "plain", url)
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
self.assertEqual(mock_get_aliases.call_args, mock.call())
self.assertTrue(all([x in unicontent(r) for x in ['mars-test@','mars-test.authors@','mars-test.chairs@']]))
self.assertTrue(all([x in unicontent(r) for x in ['ames-test@','ames-test.authors@','ames-test.chairs@']]))
def testExpansions(self):
@mock.patch("ietf.doc.views_doc.get_doc_email_aliases")
def testExpansions(self, mock_get_aliases):
mock_get_aliases.return_value = [
{"doc_name": "draft-ietf-mars-test", "alias_type": "", "expansion": "mars-author@example.mars, mars-collaborator@example.mars"},
{"doc_name": "draft-ietf-mars-test", "alias_type": ".authors", "expansion": "mars-author@example.mars, mars-collaborator@example.mars"},
{"doc_name": "draft-ietf-mars-test", "alias_type": ".chairs", "expansion": "mars-chair@example.mars"},
{"doc_name": "draft-ietf-mars-test", "alias_type": ".all", "expansion": "mars-author@example.mars, mars-collaborator@example.mars, mars-chair@example.mars"},
]
url = urlreverse('ietf.doc.views_doc.document_email', kwargs=dict(name="draft-ietf-mars-test"))
r = self.client.get(url)
self.assertEqual(mock_get_aliases.call_args, mock.call("draft-ietf-mars-test"))
self.assertEqual(r.status_code, 200)
self.assertContains(r, 'draft-ietf-mars-test.all@ietf.org')
self.assertContains(r, 'iesg_ballot_saved')
@mock.patch("ietf.doc.utils.DraftAliasGenerator")
def test_get_doc_email_aliases(self, mock_alias_gen_cls):
mock_alias_gen_cls.return_value = [
("draft-something-or-other.some-type", ["somebody@example.com"]),
("draft-something-or-other", ["somebody@example.com"]),
("draft-nothing-at-all", ["nobody@example.com"]),
("draft-nothing-at-all.some-type", ["nobody@example.com"]),
]
# order is important in the response - should be sorted by doc name and otherwise left
# in order
self.assertEqual(
get_doc_email_aliases(),
[
{
"doc_name": "draft-nothing-at-all",
"alias_type": "",
"expansion": "nobody@example.com",
},
{
"doc_name": "draft-nothing-at-all",
"alias_type": ".some-type",
"expansion": "nobody@example.com",
},
{
"doc_name": "draft-something-or-other",
"alias_type": ".some-type",
"expansion": "somebody@example.com",
},
{
"doc_name": "draft-something-or-other",
"alias_type": "",
"expansion": "somebody@example.com",
},
],
)
self.assertEqual(mock_alias_gen_cls.call_args, mock.call(None))
# Repeat with a name, no need to re-test that the alias list is actually passed through, just
# check that the DraftAliasGenerator is called correctly
draft = WgDraftFactory()
get_doc_email_aliases(draft.name)
self.assertQuerySetEqual(mock_alias_gen_cls.call_args[0][0], Document.objects.filter(pk=draft.pk))
class DocumentMeetingTests(TestCase):
def setUp(self):

26
ietf/doc/utils.py
View file

@ -14,7 +14,7 @@ import textwrap
from collections import defaultdict, namedtuple, Counter
from dataclasses import dataclass
from pathlib import Path
from typing import Iterator, Union
from typing import Iterator, Optional, Union
from zoneinfo import ZoneInfo
from django.conf import settings
@ -1265,6 +1265,12 @@ def bibxml_for_draft(doc, rev=None):
class DraftAliasGenerator:
days = 2 * 365
def __init__(self, draft_queryset=None):
if draft_queryset is not None:
self.draft_queryset = draft_queryset.filter(type_id="draft") # only drafts allowed
else:
self.draft_queryset = Document.objects.filter(type_id="draft")
def get_draft_ad_emails(self, doc):
"""Get AD email addresses for the given draft, if any."""
from ietf.group.utils import get_group_ad_emails # avoid circular import
@ -1333,7 +1339,7 @@ class DraftAliasGenerator:
def __iter__(self) -> Iterator[tuple[str, list[str]]]:
# Internet-Drafts with active status or expired within self.days
show_since = timezone.now() - datetime.timedelta(days=self.days)
drafts = Document.objects.filter(type_id="draft")
drafts = self.draft_queryset
active_drafts = drafts.filter(states__slug='active')
inactive_recent_drafts = drafts.exclude(states__slug='active').filter(expires__gte=show_since)
interesting_drafts = active_drafts | inactive_recent_drafts
@ -1384,6 +1390,22 @@ class DraftAliasGenerator:
if all:
yield alias + ".all", list(all)
def get_doc_email_aliases(name: Optional[str] = None):
aliases = []
for (alias, alist) in DraftAliasGenerator(
Document.objects.filter(type_id="draft", name=name) if name else None
):
# alias is draft-name.alias_type
doc_name, _dot, alias_type = alias.partition(".")
aliases.append({
"doc_name": doc_name,
"alias_type": f".{alias_type}" if alias_type else "",
"expansion": ", ".join(sorted(alist)),
})
return sorted(aliases, key=lambda a: (a["doc_name"]))
def investigate_fragment(name_fragment):
can_verify = set()
for root in [settings.INTERNET_DRAFT_PATH, settings.INTERNET_DRAFT_ARCHIVE_DIR]:

61
ietf/doc/views_doc.py
View file

@ -35,13 +35,13 @@
import glob
import io
import json
import os
import re
from pathlib import Path
from django.core.cache import caches
from django.db.models import Max
from django.http import HttpResponse, Http404
from django.shortcuts import render, get_object_or_404, redirect
@ -49,6 +49,7 @@ from django.template.loader import render_to_string
from django.urls import reverse as urlreverse
from django.conf import settings
from django import forms
from django.contrib.auth.decorators import login_required
from django.contrib.staticfiles import finders
import debug # pyflakes:ignore
@ -64,7 +65,7 @@ from ietf.doc.utils import (augment_events_with_revision,
add_events_message_info, get_unicode_document_content,
augment_docs_and_person_with_person_info, irsg_needed_ballot_positions, add_action_holder_change_event,
build_file_urls, update_documentauthors, fuzzy_find_documents,
bibxml_for_draft)
bibxml_for_draft, get_doc_email_aliases)
from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible
from ietf.group.models import Role, Group
from ietf.group.utils import can_manage_all_groups_of_type, can_manage_materials, group_features_role_filter
@ -1071,32 +1072,6 @@ def document_pdfized(request, name, rev=None, ext=None):
else:
raise Http404
def check_doc_email_aliases():
pattern = re.compile(r'^expand-(.*?)(\..*?)?@.*? +(.*)$')
good_count = 0
tot_count = 0
with io.open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file:
for line in virtual_file.readlines():
m = pattern.match(line)
tot_count += 1
if m:
good_count += 1
if good_count > 50 and tot_count < 3*good_count:
return True
return False
def get_doc_email_aliases(name):
if name:
pattern = re.compile(r'^expand-(%s)(\..*?)?@.*? +(.*)$'%name)
else:
pattern = re.compile(r'^expand-(.*?)(\..*?)?@.*? +(.*)$')
aliases = []
with io.open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file:
for line in virtual_file.readlines():
m = pattern.match(line)
if m:
aliases.append({'doc_name':m.group(1),'alias_type':m.group(2),'expansion':m.group(3)})
return aliases
def document_email(request,name):
doc = get_object_or_404(Document, name=name)
@ -2021,16 +1996,26 @@ def remind_action_holders(request, name):
)
def email_aliases(request,name=''):
doc = get_object_or_404(Document, name=name) if name else None
if not name:
# require login for the overview page, but not for the
# document-specific pages
if not request.user.is_authenticated:
return redirect('%s?next=%s' % (settings.LOGIN_URL, request.path))
aliases = get_doc_email_aliases(name)
return render(request,'doc/email_aliases.html',{'aliases':aliases,'ietf_domain':settings.IETF_DOMAIN,'doc':doc})
@login_required
def email_aliases(request):
"""List of all email aliases
This is currently slow except when cached
"""
slowcache = caches["slowpages"]
cache_key = "emailaliasesview"
aliases = slowcache.get(cache_key)
if not aliases:
aliases = get_doc_email_aliases() # gets all aliases
slowcache.set(cache_key, aliases, 3600)
return render(
request,
"doc/email_aliases.html",
{
"aliases": aliases,
"ietf_domain": settings.IETF_DOMAIN,
},
)
class VersionForm(forms.Form):

84
ietf/group/tests.py
View file

@ -1,13 +1,10 @@
# Copyright The IETF Trust 2013-2020, All Rights Reserved
# -*- coding: utf-8 -*-
import os
import datetime
import json
import mock
from tempfile import NamedTemporaryFile
from django.conf import settings
from django.urls import reverse as urlreverse
from django.db.models import Q
from django.test import Client
@ -22,6 +19,7 @@ from ietf.group.utils import (
get_group_role_emails,
get_child_group_role_emails,
get_group_ad_emails,
get_group_email_aliases,
GroupAliasGenerator,
role_holder_emails,
)
@ -132,24 +130,6 @@ class GroupDocDependencyTests(TestCase):
class GenerateGroupAliasesTests(TestCase):
def setUp(self):
super().setUp()
self.doc_aliases_file = NamedTemporaryFile(delete=False, mode='w+')
self.doc_aliases_file.close()
self.doc_virtual_file = NamedTemporaryFile(delete=False, mode='w+')
self.doc_virtual_file.close()
self.saved_draft_aliases_path = settings.GROUP_ALIASES_PATH
self.saved_draft_virtual_path = settings.GROUP_VIRTUAL_PATH
settings.GROUP_ALIASES_PATH = self.doc_aliases_file.name
settings.GROUP_VIRTUAL_PATH = self.doc_virtual_file.name
def tearDown(self):
settings.GROUP_ALIASES_PATH = self.saved_draft_aliases_path
settings.GROUP_VIRTUAL_PATH = self.saved_draft_virtual_path
os.unlink(self.doc_aliases_file.name)
os.unlink(self.doc_virtual_file.name)
super().tearDown()
def test_generator_class(self):
"""The GroupAliasGenerator should generate the same lists as the old mgmt cmd"""
# clean out test fixture group roles we don't need for this test
@ -208,6 +188,66 @@ class GenerateGroupAliasesTests(TestCase):
{k: (sorted(doms), sorted(addrs)) for k, (doms, addrs) in expected_dict.items()},
)
@mock.patch("ietf.group.utils.GroupAliasGenerator")
def test_get_group_email_aliases(self, mock_alias_gen_cls):
GroupFactory(name="agroup", type_id="rg")
GroupFactory(name="bgroup")
GroupFactory(name="cgroup", type_id="rg")
GroupFactory(name="dgroup")
mock_alias_gen_cls.return_value = [
("bgroup-chairs", ["ietf"], ["c1@example.com", "c2@example.com"]),
("agroup-ads", ["ietf", "irtf"], ["ad@example.com"]),
("bgroup-ads", ["ietf"], ["ad@example.com"]),
]
# order is important - should be by acronym, otherwise left in order returned by generator
self.assertEqual(
get_group_email_aliases(None, None),
[
{
"acronym": "agroup",
"alias_type": "-ads",
"expansion": "ad@example.com",
},
{
"acronym": "bgroup",
"alias_type": "-chairs",
"expansion": "c1@example.com, c2@example.com",
},
{
"acronym": "bgroup",
"alias_type": "-ads",
"expansion": "ad@example.com",
},
],
)
self.assertQuerySetEqual(
mock_alias_gen_cls.call_args[0][0],
Group.objects.all(),
ordered=False,
)
# test other parameter combinations but we already checked that the alias generator's
# output will be passed through, so don't re-test the processing
get_group_email_aliases("agroup", None)
self.assertQuerySetEqual(
mock_alias_gen_cls.call_args[0][0],
Group.objects.filter(acronym="agroup"),
ordered=False,
)
get_group_email_aliases(None, "wg")
self.assertQuerySetEqual(
mock_alias_gen_cls.call_args[0][0],
Group.objects.filter(type_id="wg"),
ordered=False,
)
get_group_email_aliases("agroup", "wg")
self.assertQuerySetEqual(
mock_alias_gen_cls.call_args[0][0],
Group.objects.none(),
ordered=False,
)
class GroupRoleEmailTests(TestCase):

61
ietf/group/tests_info.py
View file

@ -2,16 +2,15 @@
# -*- coding: utf-8 -*-
import os
import calendar
import datetime
import io
import bleach
import mock
from unittest.mock import call, patch
from pathlib import Path
from pyquery import PyQuery
from tempfile import NamedTemporaryFile
import debug # pyflakes:ignore
@ -1925,58 +1924,72 @@ class EmailAliasesTests(TestCase):
PersonFactory(user__username='plain')
GroupFactory(acronym='mars',parent=GroupFactory(type_id='area'))
GroupFactory(acronym='ames',parent=GroupFactory(type_id='area'))
self.group_alias_file = NamedTemporaryFile(delete=False)
self.group_alias_file.write(b"""# Generated by hand at 2015-02-12_16:30:52
virtual.ietf.org anything
mars-ads@ietf.org xfilter-mars-ads
expand-mars-ads@virtual.ietf.org aread@example.org
mars-chairs@ietf.org xfilter-mars-chairs
expand-mars-chairs@virtual.ietf.org mars_chair@ietf.org
ames-ads@ietf.org xfilter-mars-ads
expand-ames-ads@virtual.ietf.org aread@example.org
ames-chairs@ietf.org xfilter-mars-chairs
expand-ames-chairs@virtual.ietf.org mars_chair@ietf.org
""")
self.group_alias_file.close()
self.saved_group_virtual_path = settings.GROUP_VIRTUAL_PATH
settings.GROUP_VIRTUAL_PATH = self.group_alias_file.name
def tearDown(self):
settings.GROUP_VIRTUAL_PATH = self.saved_group_virtual_path
os.unlink(self.group_alias_file.name)
super().tearDown()
def testAliases(self):
@mock.patch("ietf.group.views.get_group_email_aliases")
def testAliases(self, mock_get_aliases):
url = urlreverse('ietf.group.urls_info_details.redirect.email', kwargs=dict(acronym="mars"))
r = self.client.get(url)
self.assertEqual(r.status_code, 302)
mock_get_aliases.return_value = [
{"acronym": "mars", "alias_type": "-ads", "expansion": "aread@example.org"},
{"acronym": "mars", "alias_type": "-chairs", "expansion": "mars_chair@ietf.org"},
]
for testdict in [dict(acronym="mars"),dict(acronym="mars",group_type="wg")]:
url = urlreverse('ietf.group.urls_info_details.redirect.email', kwargs=testdict)
r = self.client.get(url,follow=True)
self.assertEqual(
mock_get_aliases.call_args,
mock.call(testdict.get("acronym", None), testdict.get("group_type", None)),
)
self.assertTrue(all([x in unicontent(r) for x in ['mars-ads@','mars-chairs@']]))
self.assertFalse(any([x in unicontent(r) for x in ['ames-ads@','ames-chairs@']]))
url = urlreverse('ietf.group.views.email_aliases', kwargs=dict())
login_testing_unauthorized(self, "plain", url)
mock_get_aliases.return_value = [
{"acronym": "mars", "alias_type": "-ads", "expansion": "aread@example.org"},
{"acronym": "mars", "alias_type": "-chairs", "expansion": "mars_chair@ietf.org"},
{"acronym": "ames", "alias_type": "-ads", "expansion": "aread@example.org"},
{"acronym": "ames", "alias_type": "-chairs", "expansion": "mars_chair@ietf.org"},
]
r = self.client.get(url)
self.assertTrue(r.status_code,200)
self.assertEqual(mock_get_aliases.call_args, mock.call(None, None))
self.assertTrue(all([x in unicontent(r) for x in ['mars-ads@','mars-chairs@','ames-ads@','ames-chairs@']]))
url = urlreverse('ietf.group.views.email_aliases', kwargs=dict(group_type="wg"))
mock_get_aliases.return_value = [
{"acronym": "mars", "alias_type": "-ads", "expansion": "aread@example.org"},
{"acronym": "mars", "alias_type": "-chairs", "expansion": "mars_chair@ietf.org"},
{"acronym": "ames", "alias_type": "-ads", "expansion": "aread@example.org"},
{"acronym": "ames", "alias_type": "-chairs", "expansion": "mars_chair@ietf.org"},
]
r = self.client.get(url)
self.assertEqual(r.status_code,200)
self.assertEqual(mock_get_aliases.call_args, mock.call(None, "wg"))
self.assertContains(r, 'mars-ads@')
url = urlreverse('ietf.group.views.email_aliases', kwargs=dict(group_type="rg"))
mock_get_aliases.return_value = []
r = self.client.get(url)
self.assertEqual(r.status_code,200)
self.assertEqual(mock_get_aliases.call_args, mock.call(None, "rg"))
self.assertNotContains(r, 'mars-ads@')
def testExpansions(self):
@mock.patch("ietf.group.views.get_group_email_aliases")
def testExpansions(self, mock_get_aliases):
mock_get_aliases.return_value = [
{"acronym": "mars", "alias_type": "-ads", "expansion": "aread@example.org"},
{"acronym": "mars", "alias_type": "-chairs", "expansion": "mars_chair@ietf.org"},
{"acronym": "ames", "alias_type": "-ads", "expansion": "aread@example.org"},
{"acronym": "ames", "alias_type": "-chairs", "expansion": "mars_chair@ietf.org"},
]
url = urlreverse('ietf.group.views.email', kwargs=dict(acronym="mars"))
r = self.client.get(url)
self.assertEqual(r.status_code,200)
self.assertEqual(mock_get_aliases.call_args, mock.call("mars", None))
self.assertContains(r, 'Email aliases')
self.assertContains(r, 'mars-ads@ietf.org')
self.assertContains(r, 'group_personnel_change')

30
ietf/group/utils.py
View file

@ -373,6 +373,12 @@ class GroupAliasGenerator:
] # This should become groupfeature driven...
no_ad_group_types = ["rg", "rag", "team", "program", "rfcedtyp", "edappr", "edwg"]
def __init__(self, group_queryset=None):
if group_queryset is None:
self.group_queryset = Group.objects.all()
else:
self.group_queryset = group_queryset
def __iter__(self):
show_since = timezone.now() - datetime.timedelta(days=self.days)
@ -384,7 +390,7 @@ class GroupAliasGenerator:
if g == "program":
domains.append("iab")
entries = Group.objects.filter(type=g).all()
entries = self.group_queryset.filter(type=g).all()
active_entries = entries.filter(state__in=self.active_states)
inactive_recent_entries = entries.exclude(
state__in=self.active_states
@ -405,7 +411,7 @@ class GroupAliasGenerator:
yield name + "-chairs", domains, list(chair_emails)
# The area lists include every chair in active working groups in the area
areas = Group.objects.filter(type="area").all()
areas = self.group_queryset.filter(type="area").all()
active_areas = areas.filter(state__in=self.active_states)
for area in active_areas:
name = area.acronym
@ -418,7 +424,7 @@ class GroupAliasGenerator:
# Other groups with chairs that require Internet-Draft submission approval
gtypes = GroupTypeName.objects.values_list("slug", flat=True)
special_groups = Group.objects.filter(
special_groups = self.group_queryset.filter(
type__features__req_subm_approval=True, acronym__in=gtypes, state="active"
)
for group in special_groups:
@ -427,6 +433,24 @@ class GroupAliasGenerator:
yield group.acronym + "-chairs", ["ietf"], list(chair_emails)
def get_group_email_aliases(acronym, group_type):
aliases = []
group_queryset = Group.objects.all()
if acronym:
group_queryset = group_queryset.filter(acronym=acronym)
if group_type:
group_queryset = group_queryset.filter(type__slug=group_type)
for (alias, _, alist) in GroupAliasGenerator(group_queryset):
acro, _hyphen, alias_type = alias.partition("-")
expansion = ", ".join(sorted(alist))
aliases.append({
"acronym": acro,
"alias_type": f"-{alias_type}" if alias_type else "",
"expansion": expansion,
})
return sorted(aliases, key=lambda a: a["acronym"])
def role_holder_emails():
"""Get queryset of active Emails for group role holders"""
group_types_of_interest = [

35
ietf/group/views.py
View file

@ -37,9 +37,7 @@
import copy
import datetime
import itertools
import io
import math
import re
import json
import types
@ -81,7 +79,8 @@ from ietf.group.utils import (can_manage_all_groups_of_type,
can_manage_materials, group_attribute_change_desc,
construct_group_menu_context, get_group_materials,
save_group_in_history, can_manage_group, update_role_set,
get_group_or_404, setup_default_community_list_for_group, fill_in_charter_info)
get_group_or_404, setup_default_community_list_for_group, fill_in_charter_info,
get_group_email_aliases)
#
from ietf.ietfauth.utils import has_role, is_authorized_in_group
from ietf.mailtrigger.utils import gather_relevant_expansions
@ -137,21 +136,6 @@ def extract_last_name(role):
return role.person.name_parts()[3]
def check_group_email_aliases():
pattern = re.compile(r'expand-(.*?)(-\w+)@.*? +(.*)$')
tot_count = 0
good_count = 0
with io.open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file:
for line in virtual_file.readlines():
m = pattern.match(line)
tot_count += 1
if m:
good_count += 1
if good_count > 50 and tot_count < 3*good_count:
return True
return False
def response_from_file(fpath: Path) -> HttpResponse:
"""Helper to shovel a file back in an HttpResponse"""
try:
@ -580,21 +564,6 @@ def group_about_status_edit(request, acronym, group_type=None):
}
)
def get_group_email_aliases(acronym, group_type):
if acronym:
pattern = re.compile(r'expand-(%s)(-\w+)@.*? +(.*)$'%acronym)
else:
pattern = re.compile(r'expand-(.*?)(-\w+)@.*? +(.*)$')
aliases = []
with io.open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file:
for line in virtual_file.readlines():
m = pattern.match(line)
if m:
if acronym or not group_type or Group.objects.filter(acronym=m.group(1),type__slug=group_type):
aliases.append({'acronym':m.group(1),'alias_type':m.group(2),'expansion':m.group(3)})
return aliases
def email(request, acronym, group_type=None):
group = get_group_or_404(acronym, group_type)

7
ietf/ipr/tests.py
View file

@ -789,12 +789,7 @@ Subject: test
mock_process_response_email.side_effect = None
mock_process_response_email.return_value = None # rejected message
with self.assertRaises(EmailIngestionError) as context:
ingest_response_email(message)
self.assertIsNone(context.exception.email_recipients) # default recipients
self.assertIsNotNone(context.exception.email_body) # body set
self.assertIsNotNone(context.exception.email_original_message) # original message attached
self.assertEqual(context.exception.email_attach_traceback, True)
ingest_response_email(message) # should _not_ send an exception email on a clean rejection
self.assertTrue(mock_process_response_email.called)
self.assertEqual(mock_process_response_email.call_args, mock.call(message))
mock_process_response_email.reset_mock()

14
ietf/ipr/utils.py
View file

@ -92,7 +92,7 @@ def generate_draft_recursive_txt():
def ingest_response_email(message: bytes):
from ietf.api.views import EmailIngestionError # avoid circular import
try:
result = process_response_email(message)
process_response_email(message)
except Exception as err:
raise EmailIngestionError(
"Datatracker IPR email ingestion error",
@ -104,15 +104,3 @@ def ingest_response_email(message: bytes):
email_original_message=message,
email_attach_traceback=True,
) from err
if result is None:
raise EmailIngestionError(
"Datatracker IPR email ingestion rejected",
email_body=dedent("""\
A message was rejected while ingesting IPR email into the Datatracker. The original message is attached.
{error_summary}
"""),
email_original_message=message,
email_attach_traceback=True,
)

8
ietf/settings.py
View file

@ -1065,14 +1065,6 @@ GROUP_ALIAS_DOMAIN = IETF_DOMAIN
TEST_DATA_DIR = os.path.abspath(BASE_DIR + "/../test/data")
# Path to the email alias lists. Used by ietf.utils.aliases
DRAFT_ALIASES_PATH = os.path.join(TEST_DATA_DIR, "draft-aliases")
DRAFT_VIRTUAL_PATH = os.path.join(TEST_DATA_DIR, "draft-virtual")
DRAFT_VIRTUAL_DOMAIN = "virtual.ietf.org"
GROUP_ALIASES_PATH = os.path.join(TEST_DATA_DIR, "group-aliases")
GROUP_VIRTUAL_PATH = os.path.join(TEST_DATA_DIR, "group-virtual")
GROUP_VIRTUAL_DOMAIN = "virtual.ietf.org"
POSTCONFIRM_PATH = "/a/postconfirm/wrapper"

9
ietf/submit/tasks.py
View file

@ -10,7 +10,8 @@ from django.utils import timezone
from ietf.submit.models import Submission
from ietf.submit.utils import (cancel_submission, create_submission_event, process_uploaded_submission,
process_and_accept_uploaded_submission)
process_and_accept_uploaded_submission, run_all_yang_model_checks,
populate_yang_model_dirs)
from ietf.utils import log
@ -66,6 +67,12 @@ def cancel_stale_submissions():
create_submission_event(None, subm, 'Submission canceled: expired without being posted')
@shared_task
def run_yang_model_checks_task():
populate_yang_model_dirs()
run_all_yang_model_checks()
@shared_task(bind=True)
def poke(self):
log.log(f'Poked {self.name}, request id {self.request.id}')

26
ietf/submit/tests.py
View file

@ -49,6 +49,7 @@ from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactor
from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm
from ietf.submit.models import Submission, Preapproval, SubmissionExtResource
from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task
from ietf.submit.utils import apply_yang_checker_to_draft, run_all_yang_model_checks
from ietf.utils import tool_version
from ietf.utils.accesstoken import generate_access_token
from ietf.utils.mail import outbox, get_payload_text
@ -3487,3 +3488,28 @@ class SubmissionStatusTests(BaseSubmitTestCase):
"Your Internet-Draft failed at least one submission check.",
status_code=200,
)
class YangCheckerTests(TestCase):
@mock.patch("ietf.submit.utils.apply_yang_checker_to_draft")
def test_run_all_yang_model_checks(self, mock_apply):
active_drafts = WgDraftFactory.create_batch(3)
WgDraftFactory(states=[("draft", "expired")])
run_all_yang_model_checks()
self.assertEqual(mock_apply.call_count, 3)
self.assertCountEqual(
[args[0][1] for args in mock_apply.call_args_list],
active_drafts,
)
def test_apply_yang_checker_to_draft(self):
draft = WgDraftFactory()
submission = SubmissionFactory(name=draft.name, rev=draft.rev)
submission.checks.create(checker="my-checker")
checker = mock.Mock()
checker.name = "my-checker"
checker.symbol = "X"
checker.check_file_txt.return_value = (True, "whee", None, None, {})
apply_yang_checker_to_draft(checker, draft)
self.assertEqual(checker.check_file_txt.call_args, mock.call(draft.get_file_name()))

134
ietf/submit/utils.py
View file

@ -4,9 +4,11 @@
import datetime
import io
import json
import os
import pathlib
import re
import sys
import time
import traceback
import xml2rfc
@ -15,6 +17,7 @@ from pathlib import Path
from shutil import move
from typing import Optional, Union # pyflakes:ignore
from unidecode import unidecode
from xym import xym
from django.conf import settings
from django.core.exceptions import ValidationError
@ -43,6 +46,7 @@ from ietf.person.models import Person, Email
from ietf.community.utils import update_name_contains_indexes_with_new_doc
from ietf.submit.mail import ( announce_to_lists, announce_new_version, announce_to_authors,
send_approval_request, send_submission_confirmation, announce_new_wg_00, send_manual_post_request )
from ietf.submit.checkers import DraftYangChecker
from ietf.submit.models import ( Submission, SubmissionEvent, Preapproval, DraftSubmissionStateName,
SubmissionCheck, SubmissionExtResource )
from ietf.utils import log
@ -1431,3 +1435,133 @@ def process_uploaded_submission(submission):
submission.state_id = "uploaded"
submission.save()
create_submission_event(None, submission, desc="Completed submission validation checks")
def apply_yang_checker_to_draft(checker, draft):
submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first()
if submission:
check = submission.checks.filter(checker=checker.name).order_by('-id').first()
if check:
result = checker.check_file_txt(draft.get_file_name())
passed, message, errors, warnings, items = result
items = json.loads(json.dumps(items))
new_res = (passed, errors, warnings, message)
old_res = (check.passed, check.errors, check.warnings, check.message) if check else ()
if new_res != old_res:
log.log(f"Saving new yang checker results for {draft.name}-{draft.rev}")
qs = submission.checks.filter(checker=checker.name).order_by('time')
submission.checks.filter(checker=checker.name).exclude(pk=qs.first().pk).delete()
submission.checks.create(submission=submission, checker=checker.name, passed=passed,
message=message, errors=errors, warnings=warnings, items=items,
symbol=checker.symbol)
else:
log.log(f"Could not run yang checker for {draft.name}-{draft.rev}: missing submission object")
def run_all_yang_model_checks():
checker = DraftYangChecker()
for draft in Document.objects.filter(
type_id="draft",
states=State.objects.get(type="draft", slug="active"),
):
apply_yang_checker_to_draft(checker, draft)
def populate_yang_model_dirs():
"""Update the yang model dirs
* All yang modules from published RFCs should be extracted and be
available in an rfc-yang repository.
* All valid yang modules from active, not replaced, Internet-Drafts
should be extracted and be available in a draft-valid-yang repository.
* All, valid and invalid, yang modules from active, not replaced,
Internet-Drafts should be available in a draft-all-yang repository.
(Actually, given precedence ordering, it would be enough to place
non-validating modules in a draft-invalid-yang repository instead).
* In all cases, example modules should be excluded.
* Precedence is established by the search order of the repository as
provided to pyang.
* As drafts expire, models should be removed in order to catch cases
where a module being worked on depends on one which has slipped out
of the work queue.
"""
def extract_from(file, dir, strict=True):
saved_stdout = sys.stdout
saved_stderr = sys.stderr
xymerr = io.StringIO()
xymout = io.StringIO()
sys.stderr = xymerr
sys.stdout = xymout
model_list = []
try:
model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=-2)
for name in model_list:
modfile = moddir / name
mtime = file.stat().st_mtime
os.utime(str(modfile), (mtime, mtime))
if '"' in name:
name = name.replace('"', '')
modfile.rename(str(moddir / name))
model_list = [n.replace('"', '') for n in model_list]
except Exception as e:
log.log("Error when extracting from %s: %s" % (file, str(e)))
finally:
sys.stdout = saved_stdout
sys.stderr = saved_stderr
return model_list
# Extract from new RFCs
rfcdir = Path(settings.RFC_PATH)
moddir = Path(settings.SUBMIT_YANG_RFC_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
latest = 0
for item in moddir.iterdir():
if item.stat().st_mtime > latest:
latest = item.stat().st_mtime
log.log(f"Extracting RFC Yang models to {moddir} ...")
for item in rfcdir.iterdir():
if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit():
if item.stat().st_mtime > latest:
model_list = extract_from(item, moddir)
for name in model_list:
if not (name.startswith('ietf') or name.startswith('iana')):
modfile = moddir / name
modfile.unlink()
# Extract valid modules from drafts
six_months_ago = time.time() - 6 * 31 * 24 * 60 * 60
def active(dirent):
return dirent.stat().st_mtime > six_months_ago
draftdir = Path(settings.INTERNET_DRAFT_PATH)
moddir = Path(settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
log.log(f"Emptying {moddir} ...")
for item in moddir.iterdir():
item.unlink()
log.log(f"Extracting draft Yang models to {moddir} ...")
for item in draftdir.iterdir():
try:
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir, strict=False)
for name in model_list:
if name.startswith('example'):
modfile = moddir / name
modfile.unlink()
except UnicodeDecodeError as e:
log.log(f"Error processing {item.name}: {e}")

2
ietf/templates/doc/email_aliases.html
View file

@ -3,13 +3,11 @@
{% load origin %}
{% block title %}
Document email aliases
{% if doc %}for {{ doc.name }}{% endif %}
{% endblock %}
{% block content %}
{% origin %}
<h1>
Document email aliases
{% if doc %}for {{ doc.name }}{% endif %}
</h1>
{% regroup aliases|dictsort:"doc_name" by doc_name as alias_list %}
<table class="table table-borderless table-sm mt-3">

10
ietf/utils/management/commands/periodic_tasks.py
View file

@ -273,6 +273,16 @@ class Command(BaseCommand):
),
)
PeriodicTask.objects.get_or_create(
name="Run Yang model checks",
task="ietf.submit.tasks.run_yang_model_checks_task",
defaults=dict(
enabled=False,
crontab=self.crontabs["daily"],
description="Re-run Yang model checks on all active drafts",
),
)
def show_tasks(self):
for label, crontab in self.crontabs.items():
tasks = PeriodicTask.objects.filter(crontab=crontab).order_by(

172
ietf/utils/management/commands/populate_yang_model_dirs.py
View file

@ -1,172 +0,0 @@
# Copyright The IETF Trust 2016-2020, All Rights Reserved
# -*- coding: utf-8 -*-
import io
import os
import sys
import time
from pathlib import Path
from textwrap import dedent
from xym import xym
from django.conf import settings
from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
class Command(BaseCommand):
"""
Populate the yang module repositories from drafts and RFCs.
Extracts yang models from RFCs (found in settings.RFC_PATH and places
them in settings.SUBMIT_YANG_RFC_MODEL_DIR, and from active drafts, placed in
settings.SUBMIT_YANG_DRAFT_MODEL_DIR.
"""
help = dedent(__doc__).strip()
def add_arguments(self, parser):
parser.add_argument('--clean',
action='store_true', dest='clean', default=False,
help='Remove the current directory content before writing new models.')
def handle(self, *filenames, **options):
"""
* All yang modules from published RFCs should be extracted and be
available in an rfc-yang repository.
* All valid yang modules from active, not replaced, Internet-Drafts
should be extracted and be available in a draft-valid-yang repository.
* All, valid and invalid, yang modules from active, not replaced,
Internet-Drafts should be available in a draft-all-yang repository.
(Actually, given precedence ordering, it would be enough to place
non-validating modules in a draft-invalid-yang repository instead).
* In all cases, example modules should be excluded.
* Precedence is established by the search order of the repository as
provided to pyang.
* As drafts expire, models should be removed in order to catch cases
where a module being worked on depends on one which has slipped out
of the work queue.
"""
verbosity = int(options.get('verbosity'))
def extract_from(file, dir, strict=True):
saved_stdout = sys.stdout
saved_stderr = sys.stderr
xymerr = io.StringIO()
xymout = io.StringIO()
sys.stderr = xymerr
sys.stdout = xymout
model_list = []
try:
model_list = xym.xym(str(file), str(file.parent), str(dir), strict=strict, debug_level=verbosity-2)
for name in model_list:
modfile = moddir / name
mtime = file.stat().st_mtime
os.utime(str(modfile), (mtime, mtime))
if '"' in name:
name = name.replace('"', '')
modfile.rename(str(moddir/name))
model_list = [ n.replace('"','') for n in model_list ]
except Exception as e:
self.stderr.write("** Error when extracting from %s: %s" % (file, str(e)))
finally:
sys.stdout = saved_stdout
sys.stderr = saved_stderr
#
if verbosity > 1:
outmsg = xymout.getvalue()
if outmsg.strip():
self.stdout.write(outmsg)
if verbosity>2:
errmsg = xymerr.getvalue()
if errmsg.strip():
self.stderr.write(errmsg)
return model_list
# Extract from new RFCs
rfcdir = Path(settings.RFC_PATH)
moddir = Path(settings.SUBMIT_YANG_RFC_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
latest = 0
for item in moddir.iterdir():
if item.stat().st_mtime > latest:
latest = item.stat().st_mtime
if verbosity > 0:
self.stdout.write("Extracting to %s ..." % moddir)
for item in rfcdir.iterdir():
if item.is_file() and item.name.startswith('rfc') and item.name.endswith('.txt') and item.name[3:-4].isdigit():
if item.stat().st_mtime > latest:
model_list = extract_from(item, moddir)
for name in model_list:
if name.startswith('ietf') or name.startswith('iana'):
if verbosity > 1:
self.stdout.write(" Extracted from %s: %s" % (item, name))
elif verbosity > 0:
self.stdout.write('.', ending='')
self.stdout.flush()
else:
modfile = moddir / name
modfile.unlink()
if verbosity > 1:
self.stdout.write(" Skipped module from %s: %s" % (item, name))
if verbosity > 0:
self.stdout.write("")
# Extract valid modules from drafts
six_months_ago = time.time() - 6*31*24*60*60
def active(item):
return item.stat().st_mtime > six_months_ago
draftdir = Path(settings.INTERNET_DRAFT_PATH)
moddir = Path(settings.SUBMIT_YANG_DRAFT_MODEL_DIR)
if not moddir.exists():
moddir.mkdir(parents=True)
if verbosity > 0:
self.stdout.write("Emptying %s ..." % moddir)
for item in moddir.iterdir():
item.unlink()
if verbosity > 0:
self.stdout.write("Extracting to %s ..." % moddir)
for item in draftdir.iterdir():
try:
if item.is_file() and item.name.startswith('draft') and item.name.endswith('.txt') and active(item):
model_list = extract_from(item, moddir, strict=False)
for name in model_list:
if not name.startswith('example'):
if verbosity > 1:
self.stdout.write(" Extracted module from %s: %s" % (item, name))
elif verbosity > 0:
self.stdout.write('.', ending='')
self.stdout.flush()
else:
modfile = moddir / name
modfile.unlink()
if verbosity > 1:
self.stdout.write(" Skipped module from %s: %s" % (item, name))
except UnicodeDecodeError as e:
self.stderr.write('\nError: %s' % (e, ))
self.stderr.write(item.name)
self.stderr.write('')
if verbosity > 0:
self.stdout.write('')

85
ietf/utils/management/commands/run_yang_model_checks.py
View file

@ -1,85 +0,0 @@
# Copyright The IETF Trust 2017-2020, All Rights Reserved
# -*- coding: utf-8 -*-
import json
from textwrap import dedent
from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
from ietf.doc.models import Document, State
from ietf.submit.models import Submission
from ietf.submit.checkers import DraftYangChecker
class Command(BaseCommand):
"""
Run yang model checks on active drafts.
Repeats the yang checks in ietf/submit/checkers.py for active drafts, in
order to catch changes in status due to new modules becoming available in
the module directories.
"""
help = dedent(__doc__).strip()
def add_arguments(self, parser):
parser.add_argument('draftnames', nargs="*", help="drafts to check, or none to check all active yang drafts")
parser.add_argument('--clean',
action='store_true', dest='clean', default=False,
help='Remove the current directory content before writing new models.')
def check_yang(self, checker, draft, force=False):
if self.verbosity > 1:
self.stdout.write("Checking %s-%s" % (draft.name, draft.rev))
elif self.verbosity > 0:
self.stderr.write('.', ending='')
submission = Submission.objects.filter(name=draft.name, rev=draft.rev).order_by('-id').first()
if submission or force:
check = submission.checks.filter(checker=checker.name).order_by('-id').first()
if check or force:
result = checker.check_file_txt(draft.get_file_name())
passed, message, errors, warnings, items = result
if self.verbosity > 2:
self.stdout.write(" Errors: %s\n"
" Warnings: %s\n"
" Message:\n%s\n" % (errors, warnings, message))
items = json.loads(json.dumps(items))
new_res = (passed, errors, warnings, message)
old_res = (check.passed, check.errors, check.warnings, check.message) if check else ()
if new_res != old_res:
if self.verbosity > 1:
self.stdout.write(" Saving new yang checker results for %s-%s" % (draft.name, draft.rev))
qs = submission.checks.filter(checker=checker.name).order_by('time')
submission.checks.filter(checker=checker.name).exclude(pk=qs.first().pk).delete()
submission.checks.create(submission=submission, checker=checker.name, passed=passed,
message=message, errors=errors, warnings=warnings, items=items,
symbol=checker.symbol)
else:
self.stderr.write("Error: did not find any submission object for %s-%s" % (draft.name, draft.rev))
def handle(self, *filenames, **options):
"""
"""
self.verbosity = int(options.get('verbosity'))
drafts = options.get('draftnames')
active_state = State.objects.get(type="draft", slug="active")
checker = DraftYangChecker()
if drafts:
for name in drafts:
parts = name.rsplit('-',1)
if len(parts)==2 and len(parts[1])==2 and parts[1].isdigit():
name = parts[0]
draft = Document.objects.get(name=name)
self.check_yang(checker, draft, force=True)
else:
for draft in Document.objects.filter(states=active_state, type_id='draft'):
self.check_yang(checker, draft)

6
k8s/settings_local.py
View file

@ -192,12 +192,6 @@ if _SCOUT_KEY is not None:
SCOUT_CORE_AGENT_LAUNCH = False
SCOUT_REVISION_SHA = __release_hash__[:7]
# Path to the email alias lists. Used by ietf.utils.aliases
DRAFT_ALIASES_PATH = "/a/postfix/draft-aliases"
DRAFT_VIRTUAL_PATH = "/a/postfix/draft-virtual"
GROUP_ALIASES_PATH = "/a/postfix/group-aliases"
GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual"
STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None)
if STATIC_URL is None:
from ietf import __version__