From 672f9bce2637727a70af076dbd3a47dad07b437d Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 15 Apr 2020 19:54:29 +0000 Subject: [PATCH] Added display-name. Refined migrations. - Legacy-Id: 17646 --- ietf/doc/migrations/0032_extres.py | 3 +- .../0033_populate_docextresources.py | 15 ++-------- ietf/doc/models.py | 1 + ietf/group/migrations/0024_extres.py | 3 +- .../0025_populate_groupextresources.py | 29 ++++++++++++------- ietf/group/models.py | 1 + ietf/name/migrations/0011_populate_extres.py | 2 ++ ietf/person/migrations/0010_extres.py | 3 +- ietf/person/models.py | 1 + 9 files changed, 33 insertions(+), 25 deletions(-) diff --git a/ietf/doc/migrations/0032_extres.py b/ietf/doc/migrations/0032_extres.py index fdf526ebc..912f08a6e 100644 --- a/ietf/doc/migrations/0032_extres.py +++ b/ietf/doc/migrations/0032_extres.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-03-22 10:49 +# Generated by Django 1.11.29 on 2020-04-15 10:20 from __future__ import unicode_literals from django.db import migrations, models @@ -19,6 +19,7 @@ class Migration(migrations.Migration): name='DocExtResource', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('display_name', models.CharField(blank=True, default='', max_length=255)), ('value', models.CharField(max_length=2083)), ('doc', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='doc.Document')), ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')), diff --git a/ietf/doc/migrations/0033_populate_docextresources.py b/ietf/doc/migrations/0033_populate_docextresources.py index d996eb62c..4755abb1e 100644 --- a/ietf/doc/migrations/0033_populate_docextresources.py +++ b/ietf/doc/migrations/0033_populate_docextresources.py @@ -11,16 +11,6 @@ from collections import OrderedDict from django.db import migrations -""" -This makes me very nervous: - ->>> DocumentURL.objects.filter(desc__icontains='notifications').values_list('tag',flat=True).distinct() - - -I suspect the wrong thing is happening with the map below wrt the GitHub notificaitons string. - -""" - name_map = { "Issue.*": "tracker", ".*FAQ.*": "faq", @@ -76,10 +66,11 @@ def forward(apps, schema_editor): match_found = True mapped += 1 name = ExtResourceName.objects.get(slug=slug) - DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url) # TODO: validate this value against name.type + DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url, display_name=doc_url.desc) # TODO: validate this value against name.type break if not match_found: for regext, slug in url_map.items(): + doc_url.url = doc_url.url.strip() if re.search(regext, doc_url.url): match_found = True if slug: @@ -93,7 +84,7 @@ def forward(apps, schema_editor): doc_url.url = doc_url.url.replace("/tree/master","") doc_url.url = re.sub('/issues$', '', doc_url.url) doc_url.url = re.sub('/blob/master.*$', '', doc_url.url) - DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url) # TODO: validate this value against name.type + DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url, display_name=doc_url.desc) # TODO: validate this value against name.type else: ignored +=1 break diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 68a9b2592..a2ae074cf 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -865,6 +865,7 @@ class DocumentURL(models.Model): class DocExtResource(models.Model): doc = ForeignKey(Document) # Should this really be to DocumentInfo rather than Document? name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE) + display_name = models.CharField(max_length=255, default='', blank=True) value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length @python_2_unicode_compatible diff --git a/ietf/group/migrations/0024_extres.py b/ietf/group/migrations/0024_extres.py index 684a468b3..4735f08e9 100644 --- a/ietf/group/migrations/0024_extres.py +++ b/ietf/group/migrations/0024_extres.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-03-22 10:49 +# Generated by Django 1.11.29 on 2020-04-15 10:20 from __future__ import unicode_literals from django.db import migrations, models @@ -19,6 +19,7 @@ class Migration(migrations.Migration): name='GroupExtResource', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('display_name', models.CharField(blank=True, default='', max_length=255)), ('value', models.CharField(max_length=2083)), ('group', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='group.Group')), ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')), diff --git a/ietf/group/migrations/0025_populate_groupextresources.py b/ietf/group/migrations/0025_populate_groupextresources.py index 198793818..465b89009 100644 --- a/ietf/group/migrations/0025_populate_groupextresources.py +++ b/ietf/group/migrations/0025_populate_groupextresources.py @@ -11,7 +11,6 @@ from collections import OrderedDict from django.db import migrations - name_map = { "Issue.*": "tracker", ".*FAQ.*": "faq", @@ -34,21 +33,29 @@ name_map = { "GitLab User Name": "gitlab_username", } -# TODO: Review all the None values below and make sure ignoring the URLs they match is really the right thing to do. +# TODO: Consider dropping known bad links at this point +# " *https?://www.ietf.org/html.charters/*": None, # all these links are dead +# " *http://www.bell-labs.com/mailing-lists/pint": None, # dead link +# "http://www.ietf.org/wg/videos/mile-overview.html": None, # dead link +# " http://domen.uninett.no/~hta/ietf/notary-status.h": None, # dead link +# " http://www.ERC.MsState.Edu/packetway": None, # dead link +# "mailarchive\\.ietf\\.org" : None, +# "bell-labs\\.com": None, +# "html\\.charters": None, +# "datatracker\\.ietf\\.org": None, +# etc. + url_map = OrderedDict({ "https?://github\\.com": "github_repo", "https?://trac\\.ietf\\.org/.*/wiki": "wiki", "ietf\\.org.*/trac/wiki": "wiki", "trac.*wiki": "wiki", - "www\\.ietf\\.org/mailman" : None, - "www\\.ietf\\.org/mail-archive" : None, - "mailarchive\\.ietf\\.org" : None, + "www\\.ietf\\.org/mailman" : "mailing_list", + "www\\.ietf\\.org/mail-archive" : "mailing_list_archive", "ietf\\.org/logs": "jabber_log", "ietf\\.org/jabber/logs": "jabber_log", "xmpp:.*?join": "jabber_room", - "bell-labs\\.com": None, - "html\\.charters": None, - "datatracker\\.ietf\\.org": None, + "https?://.*": "webpage" }) def forward(apps, schema_editor): @@ -60,6 +67,7 @@ def forward(apps, schema_editor): not_mapped = 0 ignored = 0 + debug.say("Matching...") for group_url in GroupUrl.objects.all(): match_found = False for regext,slug in name_map.items(): @@ -67,10 +75,11 @@ def forward(apps, schema_editor): match_found = True mapped += 1 name = ExtResourceName.objects.get(slug=slug) - GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url) # TODO: validate this value against name.type + GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url, display_name=group_url.name) # TODO: validate this value against name.type break if not match_found: for regext, slug in url_map.items(): + group_url.url = group_url.url.strip() if re.search(regext, group_url.url): match_found = True if slug: @@ -84,7 +93,7 @@ def forward(apps, schema_editor): group_url.url = group_url.url.replace("/tree/master","") group_url.url = re.sub('/issues$', '', group_url.url) group_url.url = re.sub('/blob/master.*$', '', group_url.url) - GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url) # TODO: validate this value against name.type + GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url, display_name=group_url.name) # TODO: validate this value against name.type else: ignored +=1 break diff --git a/ietf/group/models.py b/ietf/group/models.py index 895d7837b..e8b8840e6 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -266,6 +266,7 @@ class GroupURL(models.Model): class GroupExtResource(models.Model): group = ForeignKey(Group) # Should this really be to GroupInfo? name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE) + display_name = models.CharField(max_length=255, default='', blank=True) value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length @python_2_unicode_compatible diff --git a/ietf/name/migrations/0011_populate_extres.py b/ietf/name/migrations/0011_populate_extres.py index 9a251e03f..1be2a2635 100644 --- a/ietf/name/migrations/0011_populate_extres.py +++ b/ietf/name/migrations/0011_populate_extres.py @@ -34,6 +34,8 @@ def forward(apps, schema_editor): resourcename("yc_impact","Yang Impact Analysis", "url"), resourcename("jabber_room","Jabber Room", "url"), resourcename("jabber_log","Jabber Log", "url"), + resourcename("mailing_list","Mailing List", "url"), + resourcename("mailing_list_archive","Mailing List Archive","url"), ] for name in resourcenames: diff --git a/ietf/person/migrations/0010_extres.py b/ietf/person/migrations/0010_extres.py index 2aaca439f..fb5fa7d10 100644 --- a/ietf/person/migrations/0010_extres.py +++ b/ietf/person/migrations/0010_extres.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-03-22 10:49 +# Generated by Django 1.11.29 on 2020-04-15 10:20 from __future__ import unicode_literals from django.db import migrations, models @@ -19,6 +19,7 @@ class Migration(migrations.Migration): name='PersonExtResource', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('display_name', models.CharField(blank=True, default='', max_length=255)), ('value', models.CharField(max_length=2083)), ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')), ('person', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='person.Person')), diff --git a/ietf/person/models.py b/ietf/person/models.py index d05a8abb5..ade392dcd 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -242,6 +242,7 @@ class Person(models.Model): class PersonExtResource(models.Model): person = ForeignKey(Person) name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE) + display_name = models.CharField(max_length=255, default='', blank=True) value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length @python_2_unicode_compatible