first run at external resources

- Legacy-Id: 17470

ietf/doc/migrations/0032_extres.py

@@ -0,0 +1,27 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-19 13:56
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('extresource', '0001_extres'),
+        ('doc', '0031_set_state_for_charters_of_replaced_groups'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DocExtResource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('doc', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='doc.Document')),
+                ('extresource', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='extresource.ExtResource')),
+            ],
+        ),
+    ]

ietf/doc/migrations/0033_populate_docextresources.py

@@ -0,0 +1,82 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-19 13:06
+from __future__ import unicode_literals
+
+import re
+
+import debug
+
+from django.db import migrations
+
+"""
+This makes me very nervous:
+
+>>> DocumentURL.objects.filter(desc__icontains='notifications').values_list('tag',flat=True).distinct()
+<QuerySet ['yang-impact-analysis', 'yang-module-metadata']>
+
+I suspect the wrong thing is happening with the map below wrt the GitHub notificaitons string.
+
+"""
+
+name_map = {
+    "Issue.*":                "tracker",
+    ".*FAQ.*":                "faq",
+    ".*Area Web Page":        "webpage",
+    ".*Wiki":                 "wiki",
+    "Home Page":              "webpage",
+    "Slack.*":                "slack",
+    "Additional .* Web Page": "webpage",
+    "Additional .* Page":     "webpage",
+    "Yang catalog entry.*":   "yc_entry",
+    "Yang impact analysis.*": "yc_impact",
+    "GitHub":                 "github_repo",
+    "Github page":            "github_repo",
+    "GitHub repo.*":          "github_repo",
+    "Github repository.*":    "github_repo",
+    "GitHub notifications":   "github_notify",
+    "GitHub org.*":           "github_org",
+    "GitHub User.*":          "github_username",
+    "GitLab User":            "gitlab_username",
+    "GitLab User Name":       "gitlab_username",
+}
+
+def forward(apps, schema_editor):
+    DocExtResource = apps.get_model('doc', 'DocExtResource')
+    ExtResource = apps.get_model('extresource', 'ExtResource')
+    ExtResourceName = apps.get_model('name', 'ExtResourceName')
+    DocumentUrl = apps.get_model('doc', 'DocumentUrl')
+
+    mapped = 0
+    not_mapped = 0
+
+    for doc_url in DocumentUrl.objects.all():
+        match_found = False
+        for regext,slug in name_map.items():
+            if re.match(regext, doc_url.desc):
+                match_found = True
+                mapped += 1
+                name = ExtResourceName.objects.get(slug=slug)
+                ext_res = ExtResource.objects.create(name_id=slug, value= doc_url.url) # TODO: validate this value against name.type
+                DocExtResource.objects.create(doc=doc_url.doc, extresource=ext_res)
+                break
+        if not match_found:
+            debug.show('("Not Mapped:",doc_url.desc, doc_url.tag.slug, doc_url.doc.name, doc_url.url)')
+            not_mapped += 1
+    debug.show('(mapped, not_mapped)')
+
+def reverse(apps, schema_editor):
+    DocExtResource = apps.get_model('doc', 'DocExtResource')
+    DocExtResource.objects.all().delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0032_extres'),
+        ('extresource', '0001_extres'),
+        ('name', '0011_populate_extres'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]

ietf/doc/models.py

@@ -22,6 +22,7 @@ from django.utils.html import mark_safe # type:ignore
 
 import debug                            # pyflakes:ignore
 
+from ietf.extresource.models import ExtResource
 from ietf.group.models import Group
 from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName,
     DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, ReviewAssignmentStateName, FormalLanguageName,
@@ -108,6 +109,7 @@ class DocumentInfo(models.Model):
     note = models.TextField(blank=True)
     internal_comments = models.TextField(blank=True)
 
+
     def file_extension(self):
         if not hasattr(self, '_cached_extension'):
             if self.uploaded_filename:
@@ -861,6 +863,10 @@ class DocumentURL(models.Model):
     desc = models.CharField(max_length=255, default='', blank=True)
     url  = models.URLField(max_length=2083) # 2083 is the legal max for URLs
 
+class DocExtResource(models.Model):
+    doc = ForeignKey(Document) # Should this really be to DocumentInfo rather than Document?
+    extresource = ForeignKey(ExtResource)
+
 @python_2_unicode_compatible
 class RelatedDocHistory(models.Model):
     source = ForeignKey('DocHistory')

ietf/extresource/__init__.py

@@ -0,0 +1 @@
+# Copyright The IETF Trust 2020, All Rights Reserved

ietf/extresource/admin.py

@@ -0,0 +1,4 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+from django.contrib import admin
+
+# Register your models here.

ietf/extresource/apps.py

@@ -0,0 +1,6 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+from django.apps import AppConfig
+
+
+class ExtresourceConfig(AppConfig):
+    name = 'extresource'

ietf/extresource/management/__init__.py

@@ -0,0 +1 @@
+# Copyright The IETF Trust ... 2020, All Rights Reserved

ietf/extresource/management/commands/__init__.py

@@ -0,0 +1 @@
+# Copyright The IETF Trust ... 2020, All Rights Reserved

ietf/extresource/management/commands/find_github_backup_info.py

@@ -0,0 +1,31 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+import json
+
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+
+from ietf.extresource.models import ExtResource
+from ietf.doc.models import DocExtResource
+from ietf.group.models import GroupExtResource
+from ietf.person.models import PersonExtResource
+
+class Command(BaseCommand):
+    help = ('Locate information about gihub repositories to backup')
+
+    def handle(self, *args, **options):
+
+        info_dict = {}
+        for repo in ExtResource.objects.filter(name__slug='github_repo'):
+            if repo not in info_dict:
+                info_dict[repo.value] = []
+
+            for username in DocExtResource.objects.filter(extresource__name__slug='github_username', doc__name__in=repo.docextresource_set.values_list('doc__name',flat=True).distinct()):
+                info_dict[repo.value].push(username.value)
+
+            for username in GroupExtResource.objects.filter(extresource__name__slug='github_username', group__acronym__in=repo.groupextresource_set.values_list('group__acronym',flat=True).distinct()):
+                info_dict[repo.value].push(username.value)
+
+            for username in PersonExtResource.objects.filter(extresource__name__slug='github_username', person_id__in=repo.personextresource_set.values_list('person__id',flat=True).distinct()):
+                info_dict[repo.value].push(username.value)
+
+        print (json.dumps(info_dict))

ietf/extresource/migrations/0001_extres.py

@@ -0,0 +1,27 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-19 13:56
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('name', '0010_extres'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ExtResource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('value', models.CharField(max_length=2083)),
+                ('name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceName')),
+            ],
+        ),
+    ]

ietf/extresource/migrations/__init__.py

@@ -0,0 +1 @@
+# Copyright The IETF Trust ... 2020, All Rights Reserved

ietf/extresource/models.py

@@ -0,0 +1,9 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+from django.db import models
+
+from ietf.name.models import ExtResourceName, ExtResourceTypeName
+
+# Create your models here.
+class ExtResource(models.Model):
+    name = models.ForeignKey(ExtResourceName, on_delete=models.CASCADE)
+    value = models.CharField(max_length=2083) # 2083 is the maximum legal URL length

ietf/extresource/tests.py

@@ -0,0 +1,4 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+from django.test import TestCase
+
+# Create your tests here.

ietf/extresource/views.py

@@ -0,0 +1,4 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+from django.shortcuts import render
+
+# Create your views here.

ietf/group/migrations/0024_extres.py

@@ -0,0 +1,27 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-19 13:56
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('extresource', '0001_extres'),
+        ('group', '0023_use_milestone_dates_default_to_true'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GroupExtResource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('extresource', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='extresource.ExtResource')),
+                ('group', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='group.Group')),
+            ],
+        ),
+    ]

ietf/group/migrations/0025_populate_groupextresources.py

@@ -0,0 +1,74 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-19 13:06
+from __future__ import unicode_literals
+
+import re
+
+import debug
+
+from django.db import migrations
+
+
+name_map = {
+    "Issue.*":                "tracker",
+    ".*FAQ.*":                "faq",
+    ".*Area Web Page":        "webpage",
+    ".*Wiki":                 "wiki",
+    "Home Page":              "webpage",
+    "Slack.*":                "slack",
+    "Additional .* Web Page": "webpage",
+    "Additional .* Page":     "webpage",
+    "Yang catalog entry.*":   "yc_entry",
+    "Yang impact analysis.*": "yc_impact",
+    "GitHub":                 "github_repo",
+    "Github page":            "github_repo",
+    "GitHub repo.*":          "github_repo",
+    "Github repository.*":    "github_repo",
+    "GitHub notifications":   "github_notify",
+    "GitHub org.*":           "github_org",
+    "GitHub User.*":          "github_username",
+    "GitLab User":            "gitlab_username",
+    "GitLab User Name":       "gitlab_username",
+}
+
+def forward(apps, schema_editor):
+    GroupExtResource = apps.get_model('group', 'GroupExtResource')
+    ExtResource = apps.get_model('extresource', 'ExtResource')
+    ExtResourceName = apps.get_model('name', 'ExtResourceName')
+    GroupUrl = apps.get_model('group', 'GroupUrl')
+
+    mapped = 0
+    not_mapped = 0
+    for group_url in GroupUrl.objects.all():
+        match_found = False
+        for regext,slug in name_map.items():
+            if re.match(regext, group_url.name):
+                match_found = True
+                mapped += 1
+                name = ExtResourceName.objects.get(slug=slug)
+                # This should probably be a get_or_create if there's any value to having a single ExtResource class
+                ext_res = ExtResource.objects.create(name_id=slug, value= group_url.url) # TODO: validate this value against name.type
+                GroupExtResource.objects.create(group=group_url.group, extresource=ext_res)
+                break
+        if not match_found:
+            debug.show('("Not Mapped:",group_url.group.acronym, group_url.name, group_url.url)')
+            not_mapped += 1
+    debug.show('(mapped, not_mapped)')
+
+def reverse(apps, schema_editor):
+    GroupExtResource = apps.get_model('group', 'GroupExtResource')
+    GroupExtResource.objects.all().delete()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('group', '0024_extres'),
+        ('extresource', '0001_extres'),
+        ('name', '0011_populate_extres'),
+        ('doc', '0033_populate_docextresources'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]

ietf/group/models.py

@@ -22,6 +22,7 @@ from simple_history.models import HistoricalRecords
 
 import debug                            # pyflakes:ignore
 
+from ietf.extresource.models import ExtResource
 from ietf.group.colors import fg_group_colors, bg_group_colors
 from ietf.name.models import GroupStateName, GroupTypeName, DocTagName, GroupMilestoneStateName, RoleName, AgendaTypeName
 from ietf.person.models import Email, Person
@@ -43,6 +44,7 @@ class GroupInfo(models.Model):
     list_archive = models.CharField(max_length=255, blank=True)
     comments = models.TextField(blank=True)
 
+
     unused_states = models.ManyToManyField('doc.State', help_text="Document states that have been disabled for the group.", blank=True)
     unused_tags = models.ManyToManyField(DocTagName, help_text="Document tags that have been disabled for the group.", blank=True)
 
@@ -262,6 +264,10 @@ class GroupURL(models.Model):
     def __str__(self):
         return u"%s (%s)" % (self.url, self.name)
 
+class GroupExtResource(models.Model):
+    group = ForeignKey(Group) # Should this really be to GroupInfo?
+    extresource = ForeignKey(ExtResource)
+
 @python_2_unicode_compatible
 class GroupMilestoneInfo(models.Model):
     group = ForeignKey(Group)

ietf/name/migrations/0010_extres.py

@@ -0,0 +1,51 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-19 13:56
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name', '0009_add_verified_errata_to_doctagname'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ExtResourceName',
+            fields=[
+                ('slug', models.CharField(max_length=32, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=255)),
+                ('desc', models.TextField(blank=True)),
+                ('used', models.BooleanField(default=True)),
+                ('order', models.IntegerField(default=0)),
+            ],
+            options={
+                'ordering': ['order', 'name'],
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ExtResourceTypeName',
+            fields=[
+                ('slug', models.CharField(max_length=32, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=255)),
+                ('desc', models.TextField(blank=True)),
+                ('used', models.BooleanField(default=True)),
+                ('order', models.IntegerField(default=0)),
+            ],
+            options={
+                'ordering': ['order', 'name'],
+                'abstract': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='extresourcename',
+            name='type',
+            field=ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='name.ExtResourceTypeName'),
+        ),
+    ]

ietf/name/migrations/0011_populate_extres.py

@@ -0,0 +1,58 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-19 11:42
+from __future__ import unicode_literals
+
+from collections import namedtuple
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    ExtResourceName = apps.get_model('name','ExtResourceName')
+    ExtResourceTypeName = apps.get_model('name','ExtResourceTypeName')
+
+    ExtResourceTypeName.objects.create(slug='email', name="Email address", desc="Email address", used=True, order=0)
+    ExtResourceTypeName.objects.create(slug='url', name="URL", desc="URL", used=True, order=0)
+    ExtResourceTypeName.objects.create(slug='string', name="string", desc="string", used=True, order=0)
+
+    # TODO: It might be better to reuse DocumentUrl.tag values for these slugs
+    resourcename = namedtuple('resourcename', ['slug', 'name', 'type'])
+    resourcenames= [
+        resourcename("webpage", "Additional Web Page", "url"),
+        resourcename("faq", "Frequently Asked Questions", "url"),
+        resourcename("github_username","GitHub Username", "string"),
+        resourcename("github_org","GitHub Organization", "url"),
+        resourcename("github_repo","GitHub Repository", "url"),
+        resourcename("gitlab_username","GitLab Username", "string"),
+        resourcename("tracker","Issuer Tracker", "url"),
+        resourcename("github_notify","GitHub Notifications Email", "email"),
+        resourcename("slack","Slack Channel", "url"),
+        resourcename("website","Website", "url"),
+        resourcename("wiki","Wiki", "url"),
+        resourcename("yc_entry","Yang Catalog Entry", "url"),
+        resourcename("yc_impact","Yang Impact Analysis", "url"),
+    ]
+
+    for name in resourcenames:
+        ExtResourceName.objects.create(slug=name.slug, name=name.name, desc=name.name, used=True, order=0, type_id=name.type)
+
+
+
+def reverse(apps, schema_editor):
+    ExtResourceName = apps.get_model('name','ExtResourceName')
+    ExtResourceTypeName = apps.get_model('name','ExtResourceTypeName')
+
+    ExtResourceName.objects.all().delete()
+    ExtResourceTypeName.objects.all().delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('name', '0010_extres'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]

ietf/name/models.py

@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2010-2019, All Rights Reserved
+# Copyright The IETF Trust 2010-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -126,4 +126,8 @@ class ImportantDateName(NameModel):
     default_offset_days = models.SmallIntegerField()
 class DocUrlTagName(NameModel):
     "Repository, Wiki, Issue Tracker, ..."
-    
+class ExtResourceTypeName(NameModel):
+    """Url, Email, String"""
+class ExtResourceName(NameModel):
+    """GitHub Repository URL, GitHub Username, ..."""
+    type = ForeignKey(ExtResourceTypeName)

ietf/person/migrations/0010_extres.py

@@ -0,0 +1,27 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-19 13:56
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('extresource', '0001_extres'),
+        ('person', '0009_auto_20190118_0725'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PersonExtResource',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('extresource', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='extresource.ExtResource')),
+                ('person', ietf.utils.models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='person.Person')),
+            ],
+        ),
+    ]

ietf/person/models.py

@@ -27,6 +27,7 @@ from simple_history.models import HistoricalRecords
 
 import debug                            # pyflakes:ignore
 
+from ietf.extresource.models import ExtResource
 from ietf.person.name import name_parts, initials, plain_name
 from ietf.utils.mail import send_mail_preformatted
 from ietf.utils.storage import NoLocationMigrationFileSystemStorage
@@ -238,6 +239,10 @@ class Person(models.Model):
         ct1['ascii']     = self.ascii
         return ct1
 
+class PersonExtResource(models.Model):
+    person = ForeignKey(Person) 
+    extresource = ForeignKey(ExtResource)
+
 @python_2_unicode_compatible
 class Alias(models.Model):
     """This is used for alternative forms of a name.  This is the

ietf/settings.py

@@ -417,6 +417,7 @@ INSTALLED_APPS = [
     'ietf.community',
     'ietf.dbtemplate',
     'ietf.doc',
+    'ietf.extresource',
     'ietf.group',
     'ietf.idindex',
     'ietf.iesg',