From c055d37dcbd592b04d4a85c3f9913a6493d73fda Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Sun, 22 Mar 2020 19:27:28 +0000 Subject: [PATCH] Removed extref app. Refined population migrations. - Legacy-Id: 17509 --- .../commands/find_github_backup_info.py | 1 - ietf/doc/migrations/0032_extres.py | 8 +-- .../0033_populate_docextresources.py | 46 +++++++++++++++-- ietf/doc/models.py | 6 +-- ietf/extresource/__init__.py | 1 - ietf/extresource/admin.py | 4 -- ietf/extresource/apps.py | 6 --- ietf/extresource/management/__init__.py | 1 - .../management/commands/__init__.py | 1 - ietf/extresource/migrations/0001_extres.py | 27 ---------- ietf/extresource/migrations/__init__.py | 1 - ietf/extresource/models.py | 9 ---- ietf/extresource/tests.py | 4 -- ietf/extresource/views.py | 4 -- ietf/group/migrations/0024_extres.py | 8 +-- .../0025_populate_groupextresources.py | 49 ++++++++++++++++--- ietf/group/models.py | 6 +-- ietf/name/migrations/0011_populate_extres.py | 2 + ietf/person/migrations/0010_extres.py | 8 +-- ietf/person/models.py | 5 +- ietf/settings.py | 1 - 21 files changed, 106 insertions(+), 92 deletions(-) rename ietf/{extresource => doc}/management/commands/find_github_backup_info.py (97%) delete mode 100644 ietf/extresource/__init__.py delete mode 100644 ietf/extresource/admin.py delete mode 100644 ietf/extresource/apps.py delete mode 100644 ietf/extresource/management/__init__.py delete mode 100644 ietf/extresource/management/commands/__init__.py delete mode 100644 ietf/extresource/migrations/0001_extres.py delete mode 100644 ietf/extresource/migrations/__init__.py delete mode 100644 ietf/extresource/models.py delete mode 100644 ietf/extresource/tests.py delete mode 100644 ietf/extresource/views.py diff --git a/ietf/extresource/management/commands/find_github_backup_info.py b/ietf/doc/management/commands/find_github_backup_info.py similarity index 97% rename from ietf/extresource/management/commands/find_github_backup_info.py rename to ietf/doc/management/commands/find_github_backup_info.py index ad48024a8..77a7e86cb 100644 --- a/ietf/extresource/management/commands/find_github_backup_info.py +++ b/ietf/doc/management/commands/find_github_backup_info.py @@ -2,7 +2,6 @@ import json from django.core.management.base import BaseCommand -from django.db.models import Q from ietf.extresource.models import ExtResource from ietf.doc.models import DocExtResource diff --git a/ietf/doc/migrations/0032_extres.py b/ietf/doc/migrations/0032_extres.py index 4ee7957b6..fdf526ebc 100644 --- a/ietf/doc/migrations/0032_extres.py +++ b/ietf/doc/migrations/0032_extres.py @@ -1,6 +1,5 @@ -# Copyright The IETF Trust 2020, All Rights Reserved # -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-03-19 13:56 +# Generated by Django 1.11.29 on 2020-03-22 10:49 from __future__ import unicode_literals from django.db import migrations, models @@ -11,7 +10,7 @@ import ietf.utils.models class Migration(migrations.Migration): dependencies = [ - ('extresource', '0001_extres'), + ('name', '0011_populate_extres'), ('doc', '0031_set_state_for_charters_of_replaced_groups'), ] @@ -20,8 +19,9 @@ class Migration(migrations.Migration): name='DocExtResource', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('value', models.CharField(max_length=2083)), ('doc', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='doc.Document')), - ('extresource', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='extresource.ExtResource')), + ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')), ], ), ] diff --git a/ietf/doc/migrations/0033_populate_docextresources.py b/ietf/doc/migrations/0033_populate_docextresources.py index 0c5af62d0..d996eb62c 100644 --- a/ietf/doc/migrations/0033_populate_docextresources.py +++ b/ietf/doc/migrations/0033_populate_docextresources.py @@ -7,6 +7,8 @@ import re import debug +from collections import OrderedDict + from django.db import migrations """ @@ -41,14 +43,31 @@ name_map = { "GitLab User Name": "gitlab_username", } +# TODO: Review all the None values below and make sure ignoring the URLs they match is really the right thing to do. +url_map = OrderedDict({ + "https?://github\\.com": "github_repo", + "https?://trac\\.ietf\\.org/.*/wiki": "wiki", + "ietf\\.org.*/trac/wiki": "wiki", + "trac.*wiki": "wiki", + "www\\.ietf\\.org/mailman" : None, + "www\\.ietf\\.org/mail-archive" : None, + "mailarchive\\.ietf\\.org" : None, + "ietf\\.org/logs": "jabber_log", + "ietf\\.org/jabber/logs": "jabber_log", + "xmpp:.*?join": "jabber_room", + "bell-labs\\.com": None, + "html\\.charters": None, + "datatracker\\.ietf\\.org": None, +}) + def forward(apps, schema_editor): DocExtResource = apps.get_model('doc', 'DocExtResource') - ExtResource = apps.get_model('extresource', 'ExtResource') ExtResourceName = apps.get_model('name', 'ExtResourceName') DocumentUrl = apps.get_model('doc', 'DocumentUrl') mapped = 0 not_mapped = 0 + ignored = 0 for doc_url in DocumentUrl.objects.all(): match_found = False @@ -57,13 +76,31 @@ def forward(apps, schema_editor): match_found = True mapped += 1 name = ExtResourceName.objects.get(slug=slug) - ext_res = ExtResource.objects.create(name_id=slug, value= doc_url.url) # TODO: validate this value against name.type - DocExtResource.objects.create(doc=doc_url.doc, extresource=ext_res) + DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url) # TODO: validate this value against name.type break + if not match_found: + for regext, slug in url_map.items(): + if re.search(regext, doc_url.url): + match_found = True + if slug: + mapped +=1 + name = ExtResourceName.objects.get(slug=slug) + # Munge the URL if it's the first github repo match + # Remove "/tree/master" substring if it exists + # Remove trailing "/issues" substring if it exists + # Remove "/blob/master/.*" pattern if present + if regext == "https?://github\\.com": + doc_url.url = doc_url.url.replace("/tree/master","") + doc_url.url = re.sub('/issues$', '', doc_url.url) + doc_url.url = re.sub('/blob/master.*$', '', doc_url.url) + DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url) # TODO: validate this value against name.type + else: + ignored +=1 + break if not match_found: debug.show('("Not Mapped:",doc_url.desc, doc_url.tag.slug, doc_url.doc.name, doc_url.url)') not_mapped += 1 - debug.show('(mapped, not_mapped)') + debug.show('(mapped, ignored, not_mapped)') def reverse(apps, schema_editor): DocExtResource = apps.get_model('doc', 'DocExtResource') @@ -73,7 +110,6 @@ class Migration(migrations.Migration): dependencies = [ ('doc', '0032_extres'), - ('extresource', '0001_extres'), ('name', '0011_populate_extres'), ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 3e3d27c34..68a9b2592 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -22,11 +22,10 @@ from django.utils.html import mark_safe # type:ignore import debug # pyflakes:ignore -from ietf.extresource.models import ExtResource from ietf.group.models import Group from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName, DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, ReviewAssignmentStateName, FormalLanguageName, - DocUrlTagName) + DocUrlTagName, ExtResourceName) from ietf.person.models import Email, Person from ietf.person.utils import get_active_balloters from ietf.utils import log @@ -865,7 +864,8 @@ class DocumentURL(models.Model): class DocExtResource(models.Model): doc = ForeignKey(Document) # Should this really be to DocumentInfo rather than Document? - extresource = ForeignKey(ExtResource) + name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE) + value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length @python_2_unicode_compatible class RelatedDocHistory(models.Model): diff --git a/ietf/extresource/__init__.py b/ietf/extresource/__init__.py deleted file mode 100644 index a40a66567..000000000 --- a/ietf/extresource/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright The IETF Trust 2020, All Rights Reserved diff --git a/ietf/extresource/admin.py b/ietf/extresource/admin.py deleted file mode 100644 index 2c06aef09..000000000 --- a/ietf/extresource/admin.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright The IETF Trust 2020, All Rights Reserved -from django.contrib import admin - -# Register your models here. diff --git a/ietf/extresource/apps.py b/ietf/extresource/apps.py deleted file mode 100644 index 2bc3bbfe6..000000000 --- a/ietf/extresource/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright The IETF Trust 2020, All Rights Reserved -from django.apps import AppConfig - - -class ExtresourceConfig(AppConfig): - name = 'extresource' diff --git a/ietf/extresource/management/__init__.py b/ietf/extresource/management/__init__.py deleted file mode 100644 index 2646ad577..000000000 --- a/ietf/extresource/management/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright The IETF Trust ... 2020, All Rights Reserved diff --git a/ietf/extresource/management/commands/__init__.py b/ietf/extresource/management/commands/__init__.py deleted file mode 100644 index 2646ad577..000000000 --- a/ietf/extresource/management/commands/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright The IETF Trust ... 2020, All Rights Reserved diff --git a/ietf/extresource/migrations/0001_extres.py b/ietf/extresource/migrations/0001_extres.py deleted file mode 100644 index 54d93131a..000000000 --- a/ietf/extresource/migrations/0001_extres.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright The IETF Trust 2020, All Rights Reserved -# -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-03-19 13:56 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('name', '0010_extres'), - ] - - operations = [ - migrations.CreateModel( - name='ExtResource', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('value', models.CharField(max_length=2083)), - ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')), - ], - ), - ] diff --git a/ietf/extresource/migrations/__init__.py b/ietf/extresource/migrations/__init__.py deleted file mode 100644 index 2646ad577..000000000 --- a/ietf/extresource/migrations/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright The IETF Trust ... 2020, All Rights Reserved diff --git a/ietf/extresource/models.py b/ietf/extresource/models.py deleted file mode 100644 index f39c13eee..000000000 --- a/ietf/extresource/models.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright The IETF Trust 2020, All Rights Reserved -from django.db import models - -from ietf.name.models import ExtResourceName, ExtResourceTypeName - -# Create your models here. -class ExtResource(models.Model): - name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE) - value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length \ No newline at end of file diff --git a/ietf/extresource/tests.py b/ietf/extresource/tests.py deleted file mode 100644 index 27ff3b268..000000000 --- a/ietf/extresource/tests.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright The IETF Trust 2020, All Rights Reserved -from django.test import TestCase - -# Create your tests here. diff --git a/ietf/extresource/views.py b/ietf/extresource/views.py deleted file mode 100644 index 85ce29d69..000000000 --- a/ietf/extresource/views.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright The IETF Trust 2020, All Rights Reserved -from django.shortcuts import render - -# Create your views here. diff --git a/ietf/group/migrations/0024_extres.py b/ietf/group/migrations/0024_extres.py index 02091ffc6..684a468b3 100644 --- a/ietf/group/migrations/0024_extres.py +++ b/ietf/group/migrations/0024_extres.py @@ -1,6 +1,5 @@ -# Copyright The IETF Trust 2020, All Rights Reserved # -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-03-19 13:56 +# Generated by Django 1.11.29 on 2020-03-22 10:49 from __future__ import unicode_literals from django.db import migrations, models @@ -11,7 +10,7 @@ import ietf.utils.models class Migration(migrations.Migration): dependencies = [ - ('extresource', '0001_extres'), + ('name', '0011_populate_extres'), ('group', '0023_use_milestone_dates_default_to_true'), ] @@ -20,8 +19,9 @@ class Migration(migrations.Migration): name='GroupExtResource', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('extresource', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='extresource.ExtResource')), + ('value', models.CharField(max_length=2083)), ('group', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='group.Group')), + ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')), ], ), ] diff --git a/ietf/group/migrations/0025_populate_groupextresources.py b/ietf/group/migrations/0025_populate_groupextresources.py index 1a676a53e..198793818 100644 --- a/ietf/group/migrations/0025_populate_groupextresources.py +++ b/ietf/group/migrations/0025_populate_groupextresources.py @@ -7,6 +7,8 @@ import re import debug +from collections import OrderedDict + from django.db import migrations @@ -32,14 +34,32 @@ name_map = { "GitLab User Name": "gitlab_username", } +# TODO: Review all the None values below and make sure ignoring the URLs they match is really the right thing to do. +url_map = OrderedDict({ + "https?://github\\.com": "github_repo", + "https?://trac\\.ietf\\.org/.*/wiki": "wiki", + "ietf\\.org.*/trac/wiki": "wiki", + "trac.*wiki": "wiki", + "www\\.ietf\\.org/mailman" : None, + "www\\.ietf\\.org/mail-archive" : None, + "mailarchive\\.ietf\\.org" : None, + "ietf\\.org/logs": "jabber_log", + "ietf\\.org/jabber/logs": "jabber_log", + "xmpp:.*?join": "jabber_room", + "bell-labs\\.com": None, + "html\\.charters": None, + "datatracker\\.ietf\\.org": None, +}) + def forward(apps, schema_editor): GroupExtResource = apps.get_model('group', 'GroupExtResource') - ExtResource = apps.get_model('extresource', 'ExtResource') ExtResourceName = apps.get_model('name', 'ExtResourceName') GroupUrl = apps.get_model('group', 'GroupUrl') mapped = 0 not_mapped = 0 + ignored = 0 + for group_url in GroupUrl.objects.all(): match_found = False for regext,slug in name_map.items(): @@ -47,14 +67,31 @@ def forward(apps, schema_editor): match_found = True mapped += 1 name = ExtResourceName.objects.get(slug=slug) - # This should probably be a get_or_create if there's any value to having a single ExtResource class - ext_res = ExtResource.objects.create(name_id=slug, value= group_url.url) # TODO: validate this value against name.type - GroupExtResource.objects.create(group=group_url.group, extresource=ext_res) + GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url) # TODO: validate this value against name.type break + if not match_found: + for regext, slug in url_map.items(): + if re.search(regext, group_url.url): + match_found = True + if slug: + mapped +=1 + name = ExtResourceName.objects.get(slug=slug) + # Munge the URL if it's the first github repo match + # Remove "/tree/master" substring if it exists + # Remove trailing "/issues" substring if it exists + # Remove "/blob/master/.*" pattern if present + if regext == "https?://github\\.com": + group_url.url = group_url.url.replace("/tree/master","") + group_url.url = re.sub('/issues$', '', group_url.url) + group_url.url = re.sub('/blob/master.*$', '', group_url.url) + GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url) # TODO: validate this value against name.type + else: + ignored +=1 + break if not match_found: debug.show('("Not Mapped:",group_url.group.acronym, group_url.name, group_url.url)') not_mapped += 1 - debug.show('(mapped, not_mapped)') + debug.show('(mapped, ignored, not_mapped)') def reverse(apps, schema_editor): GroupExtResource = apps.get_model('group', 'GroupExtResource') @@ -64,9 +101,7 @@ class Migration(migrations.Migration): dependencies = [ ('group', '0024_extres'), - ('extresource', '0001_extres'), ('name', '0011_populate_extres'), - ('doc', '0033_populate_docextresources'), ] operations = [ diff --git a/ietf/group/models.py b/ietf/group/models.py index b7d2d6a22..895d7837b 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -22,9 +22,8 @@ from simple_history.models import HistoricalRecords import debug # pyflakes:ignore -from ietf.extresource.models import ExtResource from ietf.group.colors import fg_group_colors, bg_group_colors -from ietf.name.models import GroupStateName, GroupTypeName, DocTagName, GroupMilestoneStateName, RoleName, AgendaTypeName +from ietf.name.models import GroupStateName, GroupTypeName, DocTagName, GroupMilestoneStateName, RoleName, AgendaTypeName, ExtResourceName from ietf.person.models import Email, Person from ietf.utils.mail import formataddr, send_mail_text from ietf.utils import log @@ -266,7 +265,8 @@ class GroupURL(models.Model): class GroupExtResource(models.Model): group = ForeignKey(Group) # Should this really be to GroupInfo? - extresource = ForeignKey(ExtResource) + name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE) + value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length @python_2_unicode_compatible class GroupMilestoneInfo(models.Model): diff --git a/ietf/name/migrations/0011_populate_extres.py b/ietf/name/migrations/0011_populate_extres.py index db1ca3645..9a251e03f 100644 --- a/ietf/name/migrations/0011_populate_extres.py +++ b/ietf/name/migrations/0011_populate_extres.py @@ -32,6 +32,8 @@ def forward(apps, schema_editor): resourcename("wiki","Wiki", "url"), resourcename("yc_entry","Yang Catalog Entry", "url"), resourcename("yc_impact","Yang Impact Analysis", "url"), + resourcename("jabber_room","Jabber Room", "url"), + resourcename("jabber_log","Jabber Log", "url"), ] for name in resourcenames: diff --git a/ietf/person/migrations/0010_extres.py b/ietf/person/migrations/0010_extres.py index 8468efe1f..2aaca439f 100644 --- a/ietf/person/migrations/0010_extres.py +++ b/ietf/person/migrations/0010_extres.py @@ -1,6 +1,5 @@ -# Copyright The IETF Trust 2020, All Rights Reserved # -*- coding: utf-8 -*- -# Generated by Django 1.11.29 on 2020-03-19 13:56 +# Generated by Django 1.11.29 on 2020-03-22 10:49 from __future__ import unicode_literals from django.db import migrations, models @@ -11,7 +10,7 @@ import ietf.utils.models class Migration(migrations.Migration): dependencies = [ - ('extresource', '0001_extres'), + ('name', '0011_populate_extres'), ('person', '0009_auto_20190118_0725'), ] @@ -20,7 +19,8 @@ class Migration(migrations.Migration): name='PersonExtResource', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('extresource', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='extresource.ExtResource')), + ('value', models.CharField(max_length=2083)), + ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')), ('person', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='person.Person')), ], ), diff --git a/ietf/person/models.py b/ietf/person/models.py index 94c7cf404..d05a8abb5 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -27,7 +27,7 @@ from simple_history.models import HistoricalRecords import debug # pyflakes:ignore -from ietf.extresource.models import ExtResource +from ietf.name.models import ExtResourceName from ietf.person.name import name_parts, initials, plain_name from ietf.utils.mail import send_mail_preformatted from ietf.utils.storage import NoLocationMigrationFileSystemStorage @@ -241,7 +241,8 @@ class Person(models.Model): class PersonExtResource(models.Model): person = ForeignKey(Person) - extresource = ForeignKey(ExtResource) + name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE) + value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length @python_2_unicode_compatible class Alias(models.Model): diff --git a/ietf/settings.py b/ietf/settings.py index 4850fa099..39cd60a53 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -417,7 +417,6 @@ INSTALLED_APPS = [ 'ietf.community', 'ietf.dbtemplate', 'ietf.doc', - 'ietf.extresource', 'ietf.group', 'ietf.idindex', 'ietf.iesg',