altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Cleaned up created names. Improved validation during migration. Cleaned up migration output. Cleaned the last of the awp includes from secr views. Removed now unused secr templates. Build extresource urls correctly during submission and when creating group wikis.

Browse source 
- Legacy-Id: 18158
This commit is contained in:
Robert Sparks 2020-07-13 16:03:39 +00:00
parent 53f7bc3ce6
commit 99df95d542
17 changed files with 14883 additions and 14896 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
ietf/doc/migrations/0034_populate_docextresources.py
View file

@ -5,9 +5,10 @@ from __future__ import unicode_literals
import re
import debug
import debug # pyflakes:ignore
from collections import OrderedDict, Counter
from io import StringIO
from django.db import migrations
@ -30,14 +31,12 @@ name_map = {
"Github page": "github_repo",
"GitHub repo.*": "github_repo",
"Github repository.*": "github_repo",
"GitHub notifications": "github_notify",
"GitHub org.*": "github_org",
"GitHub User.*": "github_username",
"GitLab User": "gitlab_username",
"GitLab User Name": "gitlab_username",
}
# TODO: Review all the None values below and make sure ignoring the URLs they match is really the right thing to do.
url_map = OrderedDict({
"https?://github\\.com": "github_repo",
"https://git.sr.ht/": "repo",
@ -62,19 +61,25 @@ def forward(apps, schema_editor):
DocumentUrl = apps.get_model('doc', 'DocumentUrl')
stats = Counter()
stats_file = StringIO()
for doc_url in DocumentUrl.objects.all():
doc_url.url = doc_url.url.strip()
match_found = False
for regext,slug in name_map.items():
if re.match(regext, doc_url.desc):
if re.fullmatch(regext, doc_url.desc):
match_found = True
stats['mapped'] += 1
name = ExtResourceName.objects.get(slug=slug)
DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url, display_name=doc_url.desc) # TODO: validate this value against name.type
try:
validate_external_resource_value(name, doc_url.url)
DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url, display_name=doc_url.desc)
except ValidationError as e: # pyflakes:ignore
print("Failed validation:", doc_url.url, e, file=stats_file)
stats['failed_validation'] +=1
break
if not match_found:
for regext, slug in url_map.items():
doc_url.url = doc_url.url.strip()
if re.search(regext, doc_url.url):
match_found = True
if slug:
@ -90,16 +95,18 @@ def forward(apps, schema_editor):
doc_url.url = re.sub('/blob/master.*$', '', doc_url.url)
try:
validate_external_resource_value(name, doc_url.url)
DocExtResource.objects.create(doc=doc_url.doc, name=name, value=doc_url.url, display_name=doc_url.desc) # TODO: validate this value against name.type
DocExtResource.objects.create(doc=doc_url.doc, name=name, value=doc_url.url, display_name=doc_url.desc)
except ValidationError as e: # pyflakes:ignore
debug.show('("Failed validation:", doc_url.url, e)')
print("Failed validation:", doc_url.url, e, file=stats_file)
stats['failed_validation'] +=1
else:
stats['ignored'] +=1
break
if not match_found:
debug.show('("Not Mapped:",doc_url.desc, doc_url.tag.slug, doc_url.doc.name, doc_url.url)')
print("Not Mapped:",doc_url.desc, doc_url.tag.slug, doc_url.doc.name, doc_url.url, file=stats_file)
stats['not_mapped'] += 1
print('')
print(stats_file.getvalue())
print (stats)
def reverse(apps, schema_editor):

4
ietf/doc/tests.py
View file

@ -1056,9 +1056,9 @@ class DocTestCase(TestCase):
self.assertNotIn('day', entry)
april1 = IndividualRfcFactory.create(
stream_id = 'rse',
stream_id = 'ise',
states = [('draft','rfc'),('draft-iesg','pub')],
std_level_id = 'ind',
std_level_id = 'inf',
time = datetime.datetime(1990,0o4,0o1),
)
num = april1.rfc_number()

5
ietf/doc/tests_draft.py
View file

@ -1129,9 +1129,8 @@ class IndividualInfoFormsTests(TestCase):
goodlines = """
github_repo https://github.com/some/repo Some display text
github_notify notify@example.com
github_username githubuser
website http://example.com/http/is/fine
webpage http://example.com/http/is/fine
"""
r = self.client.post(url, dict(resources=goodlines, submit="1"))
@ -1139,7 +1138,7 @@ class IndividualInfoFormsTests(TestCase):
doc = Document.objects.get(name=self.docname)
self.assertEqual(doc.latest_event(DocEvent,type="changed_document").desc[:35], 'Changed document external resources')
self.assertIn('github_username githubuser', doc.latest_event(DocEvent,type="changed_document").desc)
self.assertEqual(doc.docextresource_set.count(), 4)
self.assertEqual(doc.docextresource_set.count(), 3)
self.assertEqual(doc.docextresource_set.get(name__slug='github_repo').display_name, 'Some display text')

36
ietf/group/migrations/0034_populate_groupextresources.py
View file

@ -5,9 +5,10 @@ from __future__ import unicode_literals
import re
import debug
import debug # pyflakes:ignore
from collections import OrderedDict, Counter
from io import StringIO
from django.db import migrations
from django.core.exceptions import ValidationError
@ -30,25 +31,12 @@ name_map = {
"Github page": "github_repo",
"GitHub repo.*": "github_repo",
"Github repository.*": "github_repo",
"GitHub notifications": "github_notify",
"GitHub org.*": "github_org",
"GitHub User.*": "github_username",
"GitLab User": "gitlab_username",
"GitLab User Name": "gitlab_username",
}
# TODO: Consider dropping known bad links at this point
# " *https?://www.ietf.org/html.charters/*": None, # all these links are dead
# " *http://www.bell-labs.com/mailing-lists/pint": None, # dead link
# "http://www.ietf.org/wg/videos/mile-overview.html": None, # dead link
# " http://domen.uninett.no/~hta/ietf/notary-status.h": None, # dead link
# " http://www.ERC.MsState.Edu/packetway": None, # dead link
# "mailarchive\\.ietf\\.org" : None,
# "bell-labs\\.com": None,
# "html\\.charters": None,
# "datatracker\\.ietf\\.org": None,
# etc.
url_map = OrderedDict({
"https?://github\\.com": "github_repo",
"https?://trac\\.ietf\\.org/.*/wiki": "wiki",
@ -68,19 +56,25 @@ def forward(apps, schema_editor):
GroupUrl = apps.get_model('group', 'GroupUrl')
stats = Counter()
stats_file = StringIO()
for group_url in GroupUrl.objects.all():
group_url.url = group_url.url.strip()
match_found = False
for regext,slug in name_map.items():
if re.match(regext, group_url.name):
if re.fullmatch(regext, group_url.name):
match_found = True
stats['mapped'] += 1
name = ExtResourceName.objects.get(slug=slug)
GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url, display_name=group_url.name) # TODO: validate this value against name.type
try:
validate_external_resource_value(name, group_url.url)
GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url, display_name=group_url.name) # TODO: validate this value against name.type
except ValidationError as e: # pyflakes:ignore
print("Failed validation:", group_url.url, e, file=stats_file)
stats['failed_validation'] +=1
break
if not match_found:
for regext, slug in url_map.items():
group_url.url = group_url.url.strip()
if re.search(regext, group_url.url):
match_found = True
if slug:
@ -96,16 +90,18 @@ def forward(apps, schema_editor):
group_url.url = re.sub('/blob/master.*$', '', group_url.url)
try:
validate_external_resource_value(name, group_url.url)
GroupExtResource.objects.create(group=group_url.group, name=name, value=group_url.url, display_name=group_url.name) # TODO: validate this value against name.type
GroupExtResource.objects.create(group=group_url.group, name=name, value=group_url.url, display_name=group_url.name)
except ValidationError as e: # pyflakes:ignore
debug.show('("Failed validation:", group_url.url, e)')
print("Failed validation:", group_url.url, e, file=stats_file)
stats['failed_validation'] +=1
else:
stats['ignored'] +=1
break
if not match_found:
debug.show('("Not Mapped:",group_url.group.acronym, group_url.name, group_url.url)')
print("Not Mapped:",group_url.group.acronym, group_url.name, group_url.url, file=stats_file)
stats['not_mapped'] += 1
print('')
print(stats_file.getvalue())
print(stats)
def reverse(apps, schema_editor):

2
ietf/group/tests.py
View file

@ -74,7 +74,7 @@ class GroupDocDependencyGraphTests(TestCase):
set_coverage_checking(False)
a = WgDraftFactory()
b = WgDraftFactory()
RelatedDocument.objects.create(source=a,target=b.docalias.first(),relationship_id='normref')
RelatedDocument.objects.create(source=a,target=b.docalias.first(),relationship_id='refnorm')
def tearDown(self):
set_coverage_checking(True)

5
ietf/group/tests_info.py
View file

@ -658,9 +658,8 @@ class GroupEditTests(TestCase):
goodlines = """
github_repo https://github.com/some/repo Some display text
github_notify notify@example.com
github_username githubuser
website http://example.com/http/is/fine
webpage http://example.com/http/is/fine
"""
r = self.client.post(url, dict(resources=goodlines, submit="1"))
@ -668,7 +667,7 @@ class GroupEditTests(TestCase):
group = Group.objects.get(acronym=group.acronym)
self.assertEqual(group.latest_event(GroupEvent,type="info_changed").desc[:20], 'Resources changed to')
self.assertIn('github_username githubuser', group.latest_event(GroupEvent,type="info_changed").desc)
self.assertEqual(group.groupextresource_set.count(), 4)
self.assertEqual(group.groupextresource_set.count(), 3)
self.assertEqual(group.groupextresource_set.get(name__slug='github_repo').display_name, 'Some display text')

5
ietf/ietfauth/tests.py
View file

@ -693,14 +693,13 @@ class IetfAuthTests(TestCase):
goodlines = """
github_repo https://github.com/some/repo Some display text
github_notify notify@example.com
github_username githubuser
website http://example.com/http/is/fine
webpage http://example.com/http/is/fine
"""
r = self.client.post(url, dict(resources=goodlines, submit="1"))
self.assertEqual(r.status_code,302)
self.assertEqual(person.personextresource_set.count(), 4)
self.assertEqual(person.personextresource_set.count(), 3)
self.assertEqual(person.personextresource_set.get(name__slug='github_repo').display_name, 'Some display text')

29612
ietf/name/fixtures/names.json
View file

File diff suppressed because it is too large Load diff

2
ietf/name/migrations/0014_populate_extres.py
View file

@ -25,9 +25,7 @@ def forward(apps, schema_editor):
resourcename("github_repo","GitHub Repository", "url"),
resourcename("gitlab_username","GitLab Username", "string"),
resourcename("tracker","Issuer Tracker", "url"),
resourcename("github_notify","GitHub Notifications Email", "email"),
resourcename("slack","Slack Channel", "url"),
resourcename("website","Website", "url"),
resourcename("wiki","Wiki", "url"),
resourcename("yc_entry","Yang Catalog Entry", "url"),
resourcename("yc_impact","Yang Impact Analysis", "url"),

4
ietf/secr/templates/areas/view.html
View file

@ -35,10 +35,6 @@
</table>
</div> <!-- inline-group -->
{% with area.groupurl_set.all as awps %}
{% include "includes/awp_view.html" %}
{% endwith %}
<div class="button-group">
<ul>
<!-- <li><button onclick="window.location='../../'">Back</button></li> -->

3
ietf/secr/templates/groups/view.html
View file

@ -58,9 +58,6 @@
<tr><td>Last Modified Date:</td><td>{{ group.time }}</td></tr>
</table>
{% with group.groupurl_set.all as awps %}
{% include "includes/awp_view.html" %}
{% endwith %}
</div> <!-- groups-view-col1 -->
<div id="groups-view-col2">

15
ietf/secr/templates/includes/awp_add_form.html
View file

@ -1,15 +0,0 @@
<div class="inline-group">
<h2>Additional Web Pages</h2>
{{ awp_formset.management_form }}
{{ awp_formset.non_form_errors }}
{% for form in awp_formset.forms %}
<div class="inline-related{% if forloop.last %} last-related{% endif %}">
<h3><b>Web Page:</b>&nbsp; #{{ forloop.counter }}</h3>
<table class="awp-form full-width amstable">
<col width="150">
{{ form.as_table }}
</table>
</div> <!-- iniline-related -->
{% endfor %}
</div> <!-- inline-group -->

24
ietf/secr/templates/includes/awp_edit_form.html
View file

@ -1,24 +0,0 @@
<h2>Additional Web Pages</h2>
{{ awp_formset.management_form }}
{{ awp_formset.non_form_errors }}
{% for form in awp_formset.forms %}
<div class="inline-related{% if forloop.last %} last-related{% endif %}">
<h3><b>Web Page:</b>&nbsp; #{{ forloop.counter }}
{% if awp_formset.can_delete %}<span class="delete">{{ form.DELETE }} Delete</span>{% endif %}
</h3>
{% if form.non_field_errors %}{{ form.non_field_errors }}{% endif %}
<table id="area-awp-table" class="full-width amstable">
<col width="150">
<tr>
<th>URL:</th>
<td>{{ form.url.errors }}{{ form.url }}</td>
</tr>
<tr>
<th>Name:</th>
<td>{{ form.name.errors }}{{ form.name }}</td>
</tr>
{{ form.id }}
</table>
</div> <!-- inline-related -->
{% endfor %}

13
ietf/secr/templates/includes/awp_view.html
View file

@ -1,13 +0,0 @@
<div class="inline-group">
<h2>Additional Web Pages</h2>
{% for item in awps %}
<div class="inline-related{% if forloop.last %} last-related{% endif %}">
<h3><b>Web Page:</b>&nbsp; #{{ forloop.counter }}</h3>
<table class="full-width">
<col width="150">
<tr><td>URL:</td><td>{{ item.url }}</td><tr>
<tr><td>Name:</td><td>{{ item.name }}</td></tr>
</table>
</div> <!-- iniline-related -->
{% endfor %}
</div> <!-- inline-group -->

5
ietf/submit/utils.py
View file

@ -382,10 +382,9 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc):
moduleargs = '&'.join([ f.format(module=m) for m in modules])
url = settings.SUBMIT_YANG_CATALOG_IMPACT_URL.format(moduleargs=moduleargs, draft=draft.name)
desc = settings.SUBMIT_YANG_CATALOG_IMPACT_DESC.format(modules=','.join(modules), draft=draft.name)
draft.docextresource_set.create(value=url, name_id='yang-impact-analysis', display_name=desc)
draft.docextresource_set.create(value=url, name_id='yc_impact', display_name=desc)
# Yang module metadata URLs
old_urls = draft.documenturl_set.filter(tag_id='yc_entry')
old_urls.delete()
draft.docextresource_set.filter(name_id='yc_entry').delete()
for module in modules:
url = settings.SUBMIT_YANG_CATALOG_MODULE_URL.format(module=module)
desc = settings.SUBMIT_YANG_CATALOG_MODULE_DESC.format(module=module)

16
ietf/utils/management/commands/create_group_wikis.py
View file

@ -22,7 +22,7 @@ from django.template.loader import render_to_string
import debug # pyflakes:ignore
from ietf.group.models import Group, GroupURL, GroupFeatures
from ietf.group.models import Group, GroupFeatures
from ietf.utils.pipe import pipe
logtag = __name__.split('.')[-1]
@ -217,8 +217,8 @@ class Command(BaseCommand):
env = Environment(group.trac_dir, create=True, options=options)
self.remove_demo_components(env)
self.remove_demo_milestones(env)
self.maybe_add_group_url(group, 'Wiki', settings.TRAC_WIKI_URL_PATTERN % group.acronym)
self.maybe_add_group_url(group, 'Issue tracker', settings.TRAC_ISSUE_URL_PATTERN % group.acronym)
self.maybe_add_group_url(group, 'wiki', settings.TRAC_WIKI_URL_PATTERN % group.acronym)
self.maybe_add_group_url(group, 'tracker', settings.TRAC_ISSUE_URL_PATTERN % group.acronym)
# Use custom assets (if any) from the master setup
self.symlink_to_master_assets(group.trac_dir, env)
if group.features.acts_like_wg:
@ -301,12 +301,10 @@ class Command(BaseCommand):
comp.owner = "%s@ietf.org" % doc.name
comp.insert()
def maybe_add_group_url(self, group, name, url):
urls = [ u for u in group.groupurl_set.all() if name.lower() in u.name.lower() ]
if not urls:
self.note(" adding %s %s URL ..." % (group.acronym, name.lower()))
url = GroupURL.objects.create(group=group, name=name, url=url)
group.groupurl_set.add(url)
def maybe_add_group_url(self, group, slug, url):
if not group.groupextresource_set.filter(name__slug=slug).exists():
self.note(" adding %s %s URL ..." % (group.acronym, slug))
group.groupextresource_set.create(name_id=slug,value=url)
def add_custom_pages(self, group, env):
for template_name in settings.TRAC_WIKI_PAGES_TEMPLATES:

3
ietf/utils/validators.py
View file

@ -174,7 +174,8 @@ def validate_external_resource_value(name, value):
if name.slug in ( 'github_org', 'github_repo' ):
validate_http_url(value)
if urlparse(value).netloc.lower() != 'github.com':
hostname = urlparse(value).netloc.lower()
if not any([ hostname.endswith(x) for x in ('github.com','github.io' ) ]):
raise ValidationError('URL must be a github url')
elif name.slug == 'jabber_room':
validate_xmpp(value)