altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Removed extref app. Refined population migrations.

Browse source 
- Legacy-Id: 17509
This commit is contained in:
Robert Sparks 2020-03-22 19:27:28 +00:00
parent 316dd183d9
commit c055d37dcb
21 changed files with 106 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
ietf/extresource/management/commands/find_github_backup_info.py → ietf/doc/management/commands/find_github_backup_info.py
View file

@ -2,7 +2,6 @@
import json
from django.core.management.base import BaseCommand
from django.db.models import Q
from ietf.extresource.models import ExtResource
from ietf.doc.models import DocExtResource

8
ietf/doc/migrations/0032_extres.py
View file

@ -1,6 +1,5 @@
# Copyright The IETF Trust 2020, All Rights Reserved
# -*- coding: utf-8 -*-
# Generated by Django 1.11.29 on 2020-03-19 13:56
# Generated by Django 1.11.29 on 2020-03-22 10:49
from __future__ import unicode_literals
from django.db import migrations, models
@ -11,7 +10,7 @@ import ietf.utils.models
class Migration(migrations.Migration):
dependencies = [
('extresource', '0001_extres'),
('name', '0011_populate_extres'),
('doc', '0031_set_state_for_charters_of_replaced_groups'),
]
@ -20,8 +19,9 @@ class Migration(migrations.Migration):
name='DocExtResource',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('value', models.CharField(max_length=2083)),
('doc', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='doc.Document')),
('extresource', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='extresource.ExtResource')),
('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),
],
),
]

46
ietf/doc/migrations/0033_populate_docextresources.py
View file

@ -7,6 +7,8 @@ import re
import debug
from collections import OrderedDict
from django.db import migrations
"""
@ -41,14 +43,31 @@ name_map = {
"GitLab User Name": "gitlab_username",
}
# TODO: Review all the None values below and make sure ignoring the URLs they match is really the right thing to do.
url_map = OrderedDict({
"https?://github\\.com": "github_repo",
"https?://trac\\.ietf\\.org/.*/wiki": "wiki",
"ietf\\.org.*/trac/wiki": "wiki",
"trac.*wiki": "wiki",
"www\\.ietf\\.org/mailman" : None,
"www\\.ietf\\.org/mail-archive" : None,
"mailarchive\\.ietf\\.org" : None,
"ietf\\.org/logs": "jabber_log",
"ietf\\.org/jabber/logs": "jabber_log",
"xmpp:.*?join": "jabber_room",
"bell-labs\\.com": None,
"html\\.charters": None,
"datatracker\\.ietf\\.org": None,
})
def forward(apps, schema_editor):
DocExtResource = apps.get_model('doc', 'DocExtResource')
ExtResource = apps.get_model('extresource', 'ExtResource')
ExtResourceName = apps.get_model('name', 'ExtResourceName')
DocumentUrl = apps.get_model('doc', 'DocumentUrl')
mapped = 0
not_mapped = 0
ignored = 0
for doc_url in DocumentUrl.objects.all():
match_found = False
@ -57,13 +76,31 @@ def forward(apps, schema_editor):
match_found = True
mapped += 1
name = ExtResourceName.objects.get(slug=slug)
ext_res = ExtResource.objects.create(name_id=slug, value= doc_url.url) # TODO: validate this value against name.type
DocExtResource.objects.create(doc=doc_url.doc, extresource=ext_res)
DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url) # TODO: validate this value against name.type
break
if not match_found:
for regext, slug in url_map.items():
if re.search(regext, doc_url.url):
match_found = True
if slug:
mapped +=1
name = ExtResourceName.objects.get(slug=slug)
# Munge the URL if it's the first github repo match
# Remove "/tree/master" substring if it exists
# Remove trailing "/issues" substring if it exists
# Remove "/blob/master/.*" pattern if present
if regext == "https?://github\\.com":
doc_url.url = doc_url.url.replace("/tree/master","")
doc_url.url = re.sub('/issues$', '', doc_url.url)
doc_url.url = re.sub('/blob/master.*$', '', doc_url.url)
DocExtResource.objects.create(doc=doc_url.doc, name_id=slug, value=doc_url.url) # TODO: validate this value against name.type
else:
ignored +=1
break
if not match_found:
debug.show('("Not Mapped:",doc_url.desc, doc_url.tag.slug, doc_url.doc.name, doc_url.url)')
not_mapped += 1
debug.show('(mapped, not_mapped)')
debug.show('(mapped, ignored, not_mapped)')
def reverse(apps, schema_editor):
DocExtResource = apps.get_model('doc', 'DocExtResource')
@ -73,7 +110,6 @@ class Migration(migrations.Migration):
dependencies = [
('doc', '0032_extres'),
('extresource', '0001_extres'),
('name', '0011_populate_extres'),
]

6
ietf/doc/models.py
View file

@ -22,11 +22,10 @@ from django.utils.html import mark_safe # type:ignore
import debug # pyflakes:ignore
from ietf.extresource.models import ExtResource
from ietf.group.models import Group
from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName,
DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, ReviewAssignmentStateName, FormalLanguageName,
DocUrlTagName)
DocUrlTagName, ExtResourceName)
from ietf.person.models import Email, Person
from ietf.person.utils import get_active_balloters
from ietf.utils import log
@ -865,7 +864,8 @@ class DocumentURL(models.Model):
class DocExtResource(models.Model):
doc = ForeignKey(Document) # Should this really be to DocumentInfo rather than Document?
extresource = ForeignKey(ExtResource)
name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length
@python_2_unicode_compatible
class RelatedDocHistory(models.Model):

1
ietf/extresource/__init__.py
View file

@ -1 +0,0 @@
# Copyright The IETF Trust 2020, All Rights Reserved

4
ietf/extresource/admin.py
View file

@ -1,4 +0,0 @@
# Copyright The IETF Trust 2020, All Rights Reserved
from django.contrib import admin
# Register your models here.

6
ietf/extresource/apps.py
View file

@ -1,6 +0,0 @@
# Copyright The IETF Trust 2020, All Rights Reserved
from django.apps import AppConfig
class ExtresourceConfig(AppConfig):
name = 'extresource'

1
ietf/extresource/management/__init__.py
View file

@ -1 +0,0 @@
# Copyright The IETF Trust ... 2020, All Rights Reserved

1
ietf/extresource/management/commands/__init__.py
View file

@ -1 +0,0 @@
# Copyright The IETF Trust ... 2020, All Rights Reserved

27
ietf/extresource/migrations/0001_extres.py
View file

@ -1,27 +0,0 @@
# Copyright The IETF Trust 2020, All Rights Reserved
# -*- coding: utf-8 -*-
# Generated by Django 1.11.29 on 2020-03-19 13:56
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
('name', '0010_extres'),
]
operations = [
migrations.CreateModel(
name='ExtResource',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('value', models.CharField(max_length=2083)),
('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),
],
),
]

1
ietf/extresource/migrations/__init__.py
View file

@ -1 +0,0 @@
# Copyright The IETF Trust ... 2020, All Rights Reserved

9
ietf/extresource/models.py
View file

@ -1,9 +0,0 @@
# Copyright The IETF Trust 2020, All Rights Reserved
from django.db import models
from ietf.name.models import ExtResourceName, ExtResourceTypeName
# Create your models here.
class ExtResource(models.Model):
name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length

4
ietf/extresource/tests.py
View file

@ -1,4 +0,0 @@
# Copyright The IETF Trust 2020, All Rights Reserved
from django.test import TestCase
# Create your tests here.

4
ietf/extresource/views.py
View file

@ -1,4 +0,0 @@
# Copyright The IETF Trust 2020, All Rights Reserved
from django.shortcuts import render
# Create your views here.

8
ietf/group/migrations/0024_extres.py
View file

@ -1,6 +1,5 @@
# Copyright The IETF Trust 2020, All Rights Reserved
# -*- coding: utf-8 -*-
# Generated by Django 1.11.29 on 2020-03-19 13:56
# Generated by Django 1.11.29 on 2020-03-22 10:49
from __future__ import unicode_literals
from django.db import migrations, models
@ -11,7 +10,7 @@ import ietf.utils.models
class Migration(migrations.Migration):
dependencies = [
('extresource', '0001_extres'),
('name', '0011_populate_extres'),
('group', '0023_use_milestone_dates_default_to_true'),
]
@ -20,8 +19,9 @@ class Migration(migrations.Migration):
name='GroupExtResource',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('extresource', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='extresource.ExtResource')),
('value', models.CharField(max_length=2083)),
('group', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='group.Group')),
('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),
],
),
]

49
ietf/group/migrations/0025_populate_groupextresources.py
View file

@ -7,6 +7,8 @@ import re
import debug
from collections import OrderedDict
from django.db import migrations
@ -32,14 +34,32 @@ name_map = {
"GitLab User Name": "gitlab_username",
}
# TODO: Review all the None values below and make sure ignoring the URLs they match is really the right thing to do.
url_map = OrderedDict({
"https?://github\\.com": "github_repo",
"https?://trac\\.ietf\\.org/.*/wiki": "wiki",
"ietf\\.org.*/trac/wiki": "wiki",
"trac.*wiki": "wiki",
"www\\.ietf\\.org/mailman" : None,
"www\\.ietf\\.org/mail-archive" : None,
"mailarchive\\.ietf\\.org" : None,
"ietf\\.org/logs": "jabber_log",
"ietf\\.org/jabber/logs": "jabber_log",
"xmpp:.*?join": "jabber_room",
"bell-labs\\.com": None,
"html\\.charters": None,
"datatracker\\.ietf\\.org": None,
})
def forward(apps, schema_editor):
GroupExtResource = apps.get_model('group', 'GroupExtResource')
ExtResource = apps.get_model('extresource', 'ExtResource')
ExtResourceName = apps.get_model('name', 'ExtResourceName')
GroupUrl = apps.get_model('group', 'GroupUrl')
mapped = 0
not_mapped = 0
ignored = 0
for group_url in GroupUrl.objects.all():
match_found = False
for regext,slug in name_map.items():
@ -47,14 +67,31 @@ def forward(apps, schema_editor):
match_found = True
mapped += 1
name = ExtResourceName.objects.get(slug=slug)
# This should probably be a get_or_create if there's any value to having a single ExtResource class
ext_res = ExtResource.objects.create(name_id=slug, value= group_url.url) # TODO: validate this value against name.type
GroupExtResource.objects.create(group=group_url.group, extresource=ext_res)
GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url) # TODO: validate this value against name.type
break
if not match_found:
for regext, slug in url_map.items():
if re.search(regext, group_url.url):
match_found = True
if slug:
mapped +=1
name = ExtResourceName.objects.get(slug=slug)
# Munge the URL if it's the first github repo match
# Remove "/tree/master" substring if it exists
# Remove trailing "/issues" substring if it exists
# Remove "/blob/master/.*" pattern if present
if regext == "https?://github\\.com":
group_url.url = group_url.url.replace("/tree/master","")
group_url.url = re.sub('/issues$', '', group_url.url)
group_url.url = re.sub('/blob/master.*$', '', group_url.url)
GroupExtResource.objects.create(group=group_url.group, name_id=slug, value=group_url.url) # TODO: validate this value against name.type
else:
ignored +=1
break
if not match_found:
debug.show('("Not Mapped:",group_url.group.acronym, group_url.name, group_url.url)')
not_mapped += 1
debug.show('(mapped, not_mapped)')
debug.show('(mapped, ignored, not_mapped)')
def reverse(apps, schema_editor):
GroupExtResource = apps.get_model('group', 'GroupExtResource')
@ -64,9 +101,7 @@ class Migration(migrations.Migration):
dependencies = [
('group', '0024_extres'),
('extresource', '0001_extres'),
('name', '0011_populate_extres'),
('doc', '0033_populate_docextresources'),
]
operations = [

6
ietf/group/models.py
View file

@ -22,9 +22,8 @@ from simple_history.models import HistoricalRecords
import debug # pyflakes:ignore
from ietf.extresource.models import ExtResource
from ietf.group.colors import fg_group_colors, bg_group_colors
from ietf.name.models import GroupStateName, GroupTypeName, DocTagName, GroupMilestoneStateName, RoleName, AgendaTypeName
from ietf.name.models import GroupStateName, GroupTypeName, DocTagName, GroupMilestoneStateName, RoleName, AgendaTypeName, ExtResourceName
from ietf.person.models import Email, Person
from ietf.utils.mail import formataddr, send_mail_text
from ietf.utils import log
@ -266,7 +265,8 @@ class GroupURL(models.Model):
class GroupExtResource(models.Model):
group = ForeignKey(Group) # Should this really be to GroupInfo?
extresource = ForeignKey(ExtResource)
name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length
@python_2_unicode_compatible
class GroupMilestoneInfo(models.Model):

2
ietf/name/migrations/0011_populate_extres.py
View file

@ -32,6 +32,8 @@ def forward(apps, schema_editor):
resourcename("wiki","Wiki", "url"),
resourcename("yc_entry","Yang Catalog Entry", "url"),
resourcename("yc_impact","Yang Impact Analysis", "url"),
resourcename("jabber_room","Jabber Room", "url"),
resourcename("jabber_log","Jabber Log", "url"),
]
for name in resourcenames:

8
ietf/person/migrations/0010_extres.py
View file

@ -1,6 +1,5 @@
# Copyright The IETF Trust 2020, All Rights Reserved
# -*- coding: utf-8 -*-
# Generated by Django 1.11.29 on 2020-03-19 13:56
# Generated by Django 1.11.29 on 2020-03-22 10:49
from __future__ import unicode_literals
from django.db import migrations, models
@ -11,7 +10,7 @@ import ietf.utils.models
class Migration(migrations.Migration):
dependencies = [
('extresource', '0001_extres'),
('name', '0011_populate_extres'),
('person', '0009_auto_20190118_0725'),
]
@ -20,7 +19,8 @@ class Migration(migrations.Migration):
name='PersonExtResource',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('extresource', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='extresource.ExtResource')),
('value', models.CharField(max_length=2083)),
('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),
('person', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='person.Person')),
],
),

5
ietf/person/models.py
View file

@ -27,7 +27,7 @@ from simple_history.models import HistoricalRecords
import debug # pyflakes:ignore
from ietf.extresource.models import ExtResource
from ietf.name.models import ExtResourceName
from ietf.person.name import name_parts, initials, plain_name
from ietf.utils.mail import send_mail_preformatted
from ietf.utils.storage import NoLocationMigrationFileSystemStorage
@ -241,7 +241,8 @@ class Person(models.Model):
class PersonExtResource(models.Model):
person = ForeignKey(Person)
extresource = ForeignKey(ExtResource)
name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length
@python_2_unicode_compatible
class Alias(models.Model):

1
ietf/settings.py
View file

@ -417,7 +417,6 @@ INSTALLED_APPS = [
'ietf.community',
'ietf.dbtemplate',
'ietf.doc',
'ietf.extresource',
'ietf.group',
'ietf.idindex',
'ietf.iesg',