Merged in ^/personal/henrik/6.22.1-acctdeps which provides import of addresses subscribed to IETF mailing lists, and additional datatracker account creation requirements. Also a table and form for manual whitelisting of account logins, in order to handle cases which fall outside the default requirements. Fixed some tests.

- Legacy-Id: 11389

ietf/group/tests_info.py

@@ -396,6 +396,8 @@ class GroupEditTests(TestCase):
 
         bof_state = GroupStateName.objects.get(slug="bof")
 
+        area = Group.objects.filter(type="area").first()
+
         # normal get
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
@@ -412,21 +414,30 @@ class GroupEditTests(TestCase):
         # acronym contains non-alphanumeric
         r = self.client.post(url, dict(acronym="test...", name="Testing WG", state=bof_state.pk))
         self.assertEqual(r.status_code, 200)
+        self.assertTrue(len(q('form .has-error')) > 0)
 
         # acronym contains hyphen
         r = self.client.post(url, dict(acronym="test-wg", name="Testing WG", state=bof_state.pk))
         self.assertEqual(r.status_code, 200)
+        self.assertTrue(len(q('form .has-error')) > 0)
 
         # acronym too short
         r = self.client.post(url, dict(acronym="t", name="Testing WG", state=bof_state.pk))
         self.assertEqual(r.status_code, 200)
+        self.assertTrue(len(q('form .has-error')) > 0)
 
         # acronym doesn't start with an alpha character
         r = self.client.post(url, dict(acronym="1startwithalpha", name="Testing WG", state=bof_state.pk))
         self.assertEqual(r.status_code, 200)
+        self.assertTrue(len(q('form .has-error')) > 0)
 
-        # creation
+        # no parent group given
         r = self.client.post(url, dict(acronym="testwg", name="Testing WG", state=bof_state.pk))
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue(len(q('form .has-error')) > 0)
+
+        # Ok creation
+        r = self.client.post(url, dict(acronym="testwg", name="Testing WG", state=bof_state.pk, parent=area.pk))
         self.assertEqual(r.status_code, 302)
         self.assertEqual(len(Group.objects.filter(type="wg")), num_wgs + 1)
         group = Group.objects.get(acronym="testwg")

ietf/ietfauth/forms.py

@@ -12,6 +12,7 @@ from django.core.urlresolvers import reverse as urlreverse
 import debug                            # pyflakes:ignore
 
 from ietf.person.models import Person, Email
+from ietf.mailinglists.models import Whitelisted
 
 
 class RegistrationForm(forms.Form):
@@ -118,3 +119,9 @@ class ResetPasswordForm(forms.Form):
 class TestEmailForm(forms.Form):
     email = forms.EmailField(required=False)
 
+class WhitelistForm(ModelForm):
+    class Meta:
+        model = Whitelisted
+        exclude = ['by', 'time' ]
+
+    

ietf/ietfauth/tests.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
-import os, shutil
+from __future__ import unicode_literals
+
+import os, shutil, time
 from urlparse import urlsplit
 from pyquery import PyQuery
 from unittest import skipIf
@@ -15,6 +17,8 @@ from ietf.utils.mail import outbox, empty_outbox
 from ietf.person.models import Person, Email
 from ietf.group.models import Group, Role, RoleName
 from ietf.ietfauth.htpasswd import update_htpasswd_file
+from ietf.mailinglists.models import Subscribed
+
 import ietf.ietfauth.views
 
 if os.path.exists(settings.HTPASSWD_COMMAND):
@@ -94,7 +98,7 @@ class IetfAuthTests(TestCase):
 
         return False
 
-    def test_create_account(self):
+    def test_create_account_failure(self):
         make_test_data()
 
         url = urlreverse(ietf.ietfauth.views.create_account)
@@ -103,12 +107,21 @@ class IetfAuthTests(TestCase):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
 
-        # register email
+        # register email and verify failure
         email = 'new-account@example.com'
         empty_outbox()
         r = self.client.post(url, { 'email': email })
         self.assertEqual(r.status_code, 200)
-        self.assertTrue("Account created" in unicontent(r))
+        self.assertIn("Account creation failed", unicontent(r))
+
+    def register_and_verify(self, email):
+        url = urlreverse(ietf.ietfauth.views.create_account)
+
+        # register email
+        empty_outbox()
+        r = self.client.post(url, { 'email': email })
+        self.assertEqual(r.status_code, 200)
+        self.assertIn("Account created", unicontent(r))
         self.assertEqual(len(outbox), 1)
 
         # go to confirm page
@@ -130,6 +143,41 @@ class IetfAuthTests(TestCase):
 
         self.assertTrue(self.username_in_htpasswd_file(email))
 
+    def test_create_whitelisted_account(self):
+        email = "new-account@example.com"
+
+        # add whitelist entry
+        r = self.client.post(urlreverse(django.contrib.auth.views.login), {"username":"secretary", "password":"secretary+password"})
+        self.assertEqual(r.status_code, 302)
+        self.assertEqual(urlsplit(r["Location"])[2], urlreverse(ietf.ietfauth.views.profile))
+
+        r = self.client.get(urlreverse(ietf.ietfauth.views.add_account_whitelist))
+        self.assertEqual(r.status_code, 200)
+        self.assertIn("Add a whitelist entry", unicontent(r))
+
+        r = self.client.post(urlreverse(ietf.ietfauth.views.add_account_whitelist), {"email": email})
+        self.assertEqual(r.status_code, 200)
+        self.assertIn("Whitelist entry creation successful", unicontent(r))
+
+        # log out
+        r = self.client.get(urlreverse(django.contrib.auth.views.logout))
+        self.assertEqual(r.status_code, 200)
+
+        # register and verify whitelisted email
+        self.register_and_verify(email)
+
+
+    def test_create_subscribed_account(self):
+        # verify creation with email in subscribed list
+        saved_delay = settings.LIST_ACCOUNT_DELAY
+        settings.LIST_ACCOUNT_DELAY = 1
+        email = "subscribed@example.com"
+        s = Subscribed(email=email)
+        s.save()
+        time.sleep(1.1)
+        self.register_and_verify(email)
+        settings.LIST_ACCOUNT_DELAY = saved_delay
+
     def test_profile(self):
         make_test_data()
 

ietf/ietfauth/urls.py

@@ -3,6 +3,8 @@
 from django.conf.urls import patterns, url
 from django.contrib.auth.views import login, logout
 
+from ietf.ietfauth.views import add_account_whitelist
+
 urlpatterns = patterns('ietf.ietfauth.views',
         url(r'^$', 'index'),
 #        url(r'^login/$', 'ietf_login'),
@@ -18,4 +20,5 @@ urlpatterns = patterns('ietf.ietfauth.views',
         url(r'^reset/$', 'password_reset'),
         url(r'^reset/confirm/(?P<auth>[^/]+)/$', 'confirm_password_reset'),
         url(r'^confirmnewemail/(?P<auth>[^/]+)/$', 'confirm_new_email'),
+        (r'whitelist/add/?$', add_account_whitelist),
 )

ietf/ietfauth/views.py

@@ -32,6 +32,8 @@
 
 # Copyright The IETF Trust 2007, All Rights Reserved
 
+from datetime import datetime as DateTime, timedelta as TimeDelta
+
 from django.conf import settings
 from django.http import Http404  #, HttpResponse, HttpResponseRedirect
 from django.shortcuts import render, redirect, get_object_or_404
@@ -42,10 +44,14 @@ import django.core.signing
 from django.contrib.sites.models import Site
 from django.contrib.auth.models import User
 
+import debug                            # pyflakes:ignore
+
 from ietf.group.models import Role
-from ietf.ietfauth.forms import RegistrationForm, PasswordForm, ResetPasswordForm, TestEmailForm
+from ietf.ietfauth.forms import RegistrationForm, PasswordForm, ResetPasswordForm, TestEmailForm, WhitelistForm
 from ietf.ietfauth.forms import get_person_form, RoleEmailForm, NewEmailForm
 from ietf.ietfauth.htpasswd import update_htpasswd_file
+from ietf.ietfauth.utils import role_required
+from ietf.mailinglists.models import Subscribed, Whitelisted
 from ietf.person.models import Person, Email, Alias
 from ietf.utils.mail import send_mail
 
@@ -85,20 +91,25 @@ def create_account(request):
     if request.method == 'POST':
         form = RegistrationForm(request.POST)
         if form.is_valid():
-            to_email = form.cleaned_data['email']
+            to_email = form.cleaned_data['email'] # This will be lowercase if form.is_valid()
+            existing = Subscribed.objects.filter(email=to_email).first()
+            ok_to_create = ( Whitelisted.objects.filter(email=to_email).exists()
+                or existing and (existing.time + TimeDelta(seconds=settings.LIST_ACCOUNT_DELAY)) < DateTime.now() )
+            if ok_to_create:
+                auth = django.core.signing.dumps(to_email, salt="create_account")
 
-            auth = django.core.signing.dumps(to_email, salt="create_account")
+                domain = Site.objects.get_current().domain
+                subject = 'Confirm registration at %s' % domain
+                from_email = settings.DEFAULT_FROM_EMAIL
 
-            domain = Site.objects.get_current().domain
-            subject = 'Confirm registration at %s' % domain
-            from_email = settings.DEFAULT_FROM_EMAIL
-
-            send_mail(request, to_email, from_email, subject, 'registration/creation_email.txt', {
-                'domain': domain,
-                'auth': auth,
-                'username': to_email,
-                'expire': settings.DAYS_TO_EXPIRE_REGISTRATION_LINK,
-            })
+                send_mail(request, to_email, from_email, subject, 'registration/creation_email.txt', {
+                    'domain': domain,
+                    'auth': auth,
+                    'username': to_email,
+                    'expire': settings.DAYS_TO_EXPIRE_REGISTRATION_LINK,
+                })
+            else:
+                return render(request, 'registration/manual.html', { 'account_request_email': settings.ACCOUNT_REQUEST_EMAIL })
     else:
         form = RegistrationForm()
 
@@ -359,3 +370,22 @@ def test_email(request):
         r.set_cookie("testmailcc", cookie)
 
     return r
+
+@role_required('Secretariat')
+def add_account_whitelist(request):
+    success = False
+    if request.method == 'POST':
+        form = WhitelistForm(request.POST)
+        if form.is_valid():
+            email = form.cleaned_data['email']
+            entry = Whitelisted(email=email, by=request.user.person)
+            entry.save()
+            success = True
+    else:
+        form = WhitelistForm()
+
+    return render(request, 'ietfauth/whitelist_form.html', {
+        'form': form,
+        'success': success,
+    })
+

ietf/mailinglists/admin.py

@@ -0,0 +1,23 @@
+# Copyright The IETF Trust 2016, All Rights Reserved
+
+from django.contrib import admin
+
+from ietf.mailinglists.models import List, Subscribed, Whitelisted
+
+
+class ListAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'description', 'advertised')
+    search_fields = ('name',)
+admin.site.register(List, ListAdmin)
+
+
+class SubscribedAdmin(admin.ModelAdmin):
+    list_display = ('id', 'time', 'email')
+    raw_id_fields = ('lists',)
+    search_fields = ('email',)
+admin.site.register(Subscribed, SubscribedAdmin)
+
+
+class WhitelistedAdmin(admin.ModelAdmin):
+    list_display = ('id', 'time', 'email', 'by')
+admin.site.register(Whitelisted, WhitelistedAdmin)

ietf/mailinglists/management/commands/import_mailman_listinfo.py

@@ -0,0 +1,61 @@
+# Copyright The IETF Trust 2016, All Rights Reserved
+
+import sys
+from textwrap import dedent
+
+import debug                            # pyflakes:ignore
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ietf.mailinglists.models import List, Subscribed
+
+class Command(BaseCommand):
+    """
+    Import list information from Mailman.
+
+    Import announced list names, descriptions, and subscribers, by calling the
+    appropriate Mailman functions and adding entries to the database.
+
+    Run this from cron regularly, with sufficient permissions to access the
+    mailman database files.
+
+    """
+
+    help = dedent(__doc__).strip()
+            
+    #option_list = BaseCommand.option_list + (       )
+
+    def note(self, msg):    
+        if self.verbosity > 1:
+            self.stdout.write(msg)
+
+    def handle(self, *filenames, **options):
+        """
+
+        * Import announced lists, with appropriate meta-information.
+
+        * For each list, import the members.
+
+        """
+
+        self.verbosity = int(options.get('verbosity'))
+
+        sys.path.append(settings.MAILMAN_LIB_DIR)
+
+        from Mailman import Utils
+        from Mailman import MailList
+
+        for name in Utils.list_names():
+            mlist = MailList.MailList(name, lock=False)
+            self.note("List: %s" % mlist.internal_name())
+            if mlist.advertised:
+                list, created = List.objects.get_or_create(name=mlist.real_name, description=mlist.description, advertised=mlist.advertised)
+                # The following calls return lowercased addresses
+                members = mlist.getRegularMemberKeys() + mlist.getDigestMemberKeys()
+                known = [ s.email for s in Subscribed.objects.filter(lists__name=name) ]
+                for addr in members:
+                    if not addr in known:
+                        self.note("  Adding subscribed: %s" % (addr))
+                        new, created = Subscribed.objects.get_or_create(email=addr)
+                        new.lists.add(list)

ietf/mailinglists/migrations/0001_initial.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+from django.db import migrations
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+    ]

ietf/mailinglists/migrations/0002_list_subscribed_whitelisted.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import django.core.validators
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('person', '0013_add_plain_name_aliases'),
+        ('mailinglists', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='List',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('name', models.CharField(max_length=32)),
+                ('description', models.CharField(max_length=256)),
+                ('advertised', models.BooleanField(default=True)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='Subscribed',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('time', models.DateTimeField(auto_now_add=True)),
+                ('email', models.CharField(max_length=64, validators=[django.core.validators.EmailValidator()])),
+                ('lists', models.ManyToManyField(to='mailinglists.List')),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='Whitelisted',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('time', models.DateTimeField(auto_now_add=True)),
+                ('email', models.CharField(max_length=64, verbose_name=b'Email address', validators=[django.core.validators.EmailValidator()])),
+                ('by', models.ForeignKey(to='person.Person')),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+    ]

ietf/mailinglists/models.py

@@ -0,0 +1,28 @@
+# Copyright The IETF Trust 2016, All Rights Reserved
+
+from django.db import models
+from django.core.validators import validate_email
+
+from ietf.person.models import Person
+
+class List(models.Model):
+    name = models.CharField(max_length=32)
+    description = models.CharField(max_length=256)
+    advertised = models.BooleanField(default=True)
+    def __unicode__(self):
+        return "<List: %s>" % self.name
+
+class Subscribed(models.Model):
+    time = models.DateTimeField(auto_now_add=True)
+    email = models.CharField(max_length=64, validators=[validate_email])
+    lists = models.ManyToManyField(List)
+    def __unicode__(self):
+        return "<Subscribed: %s at %s>" % (self.email, self.time)
+
+class Whitelisted(models.Model):
+    time = models.DateTimeField(auto_now_add=True)
+    email = models.CharField("Email address", max_length=64, validators=[validate_email])
+    by = models.ForeignKey(Person)
+    def __unicode__(self):
+        return "<Whitelisted: %s at %s>" % (self.email, self.time)
+    

ietf/mailinglists/resources.py

@@ -0,0 +1,58 @@
+# Copyright The IETF Trust 2016, All Rights Reserved
+# Autogenerated by the makeresources management command 2016-06-12 12:29 PDT
+from tastypie.resources import ModelResource
+from tastypie.fields import ToManyField                 # pyflakes:ignore
+from tastypie.constants import ALL, ALL_WITH_RELATIONS  # pyflakes:ignore
+from tastypie.cache import SimpleCache
+
+from ietf import api
+from ietf.api import ToOneField                         # pyflakes:ignore
+
+from ietf.mailinglists.models import Whitelisted, List, Subscribed
+
+
+from ietf.person.resources import PersonResource
+class WhitelistedResource(ModelResource):
+    by               = ToOneField(PersonResource, 'by')
+    class Meta:
+        queryset = Whitelisted.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'whitelisted'
+        filtering = { 
+            "id": ALL,
+            "time": ALL,
+            "email": ALL,
+            "by": ALL_WITH_RELATIONS,
+        }
+api.mailinglists.register(WhitelistedResource())
+
+class ListResource(ModelResource):
+    class Meta:
+        queryset = List.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'list'
+        filtering = { 
+            "id": ALL,
+            "name": ALL,
+            "description": ALL,
+            "advertised": ALL,
+        }
+api.mailinglists.register(ListResource())
+
+class SubscribedResource(ModelResource):
+    lists            = ToManyField(ListResource, 'lists', null=True)
+    class Meta:
+        queryset = Subscribed.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'subscribed'
+        filtering = { 
+            "id": ALL,
+            "time": ALL,
+            "email": ALL,
+            "lists": ALL_WITH_RELATIONS,
+        }
+api.mailinglists.register(SubscribedResource())
+

ietf/mailinglists/tests.py

@@ -1,3 +1,5 @@
+# Copyright The IETF Trust 2016, All Rights Reserved
+
 from django.core.urlresolvers import reverse as urlreverse
 
 from pyquery import PyQuery

ietf/settings.py

@@ -658,6 +658,10 @@ MARKUP_SETTINGS = {
 
 MAILMAN_LIB_DIR = '/usr/lib/mailman'
 
+# This is the number of seconds required between subscribing to an ietf
+# mailing list and datatracker account creation being accepted
+LIST_ACCOUNT_DELAY = 60*60*25           # 25 hours
+ACCOUNT_REQUEST_EMAIL = 'account-request@ietf.org'
 
 
 # Put the production SECRET_KEY in settings_local.py, and also any other

ietf/templates/base/menu_user.html

@@ -41,6 +41,7 @@
     <li><a href="/admin/iesg/telechatagendaitem/">Management items</a></li>
     <li><a href="{% url "ietf.iesg.views.milestones_needing_review" %}">Milestones</a></li>
     <li><a href="{% url "ietf.sync.views.discrepancies" %}">Sync discrepancies</a>
+    <li><a href="{% url "ietf.ietfauth.views.add_account_whitelist" %}">Account whitelist</a>
   {% endif %}
 
   {% if user|has_role:"IANA" %}

ietf/templates/ietfauth/whitelist_form.html

@@ -0,0 +1,93 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2016, All Rights Reserved #}
+{% load origin %}
+
+{% load bootstrap3 %}
+
+{% block title %}Set up test email address{% endblock %}
+
+{% block content %}
+  {% origin %}
+  {% if success %}
+    <h1>Whitelist entry creation successful</h1>
+
+    <p>
+
+      Please ask the requestor to try the
+      <a href="{% url 'ietf.ietfauth.views.create_account'%}">account creation form</a>
+      again, with the whitelisted email address.
+
+    </p>
+
+  {% else %}
+    <h1>Add a whitelist entry for account creation.</h1>
+
+    <p>
+       When an email request comes in for assistance with account creation
+       because the automated account creation has failed, you can add the
+       address to an account creation whitelist here.
+    </p>
+    <p>
+       Before you do so, please complete the following 3 verification steps:
+       <ol>
+	  <li>
+
+	     Has the person provided relevant information in his request, or has he simply
+	     copied the text from the account creation failure message?  All genuine (non-spam)
+	     account creation requests seen between 2009 and 2016 for tools.ietf.org have
+	     contained a reasonable request message, rather than just copy-pasting the account
+	     creation failure message.  If there's no proper request message, step 2 below can
+	     be performed to make sure the request is bogus, but if that also fails, no further
+	     effort should be needed.
+
+	  </li>
+	  <li>
+
+	     Google for the person's name within the ietf.org site: "Jane Doe site:ietf.org".  If
+	     found, and the email address matches an address used in drafts or discussions,
+	     things are fine, and it's OK to add the address to the whitelist using this form,
+	     and ask the person to please try the
+	     <a href="{% url 'ietf.ietfauth.views.create_account' %}">account creation form</a> again.
+
+	  </li>
+	  <li>
+
+	     <p>
+
+	     If google finds no trace of the person being an ietf participant, he or she could
+	     still be somebody who is just getting involved in IETF work.  A datatracker account
+	     is probably not necessary, but in case this is a legitimate request, please email
+	     the person and ask:
+
+	     </p>
+	     <blockquote>
+		"Which wgs do you require a password for?"
+	     </blockquote>
+
+	     <p>
+
+	     This is a bit of a trick question, because it is very unlikely that somebody who
+	     isn't involved in IETF work will give a reasonable response, while almost any answer
+	     from somebody who is doing IETF work will show that they have some clue.
+
+	     </p>
+	     <p>
+
+	     If the answer to this question shows clue, then add the address to the whitelist
+	     using this form, and ask the person to please try the
+	     <a href="{% url 'ietf.ietfauth.views.create_account' %}"> account creation form</a> again.
+
+	     </p>
+	  </li>
+       </ol>
+    </p>
+    <form role-"form" method="post">
+      {% csrf_token %}
+      {% bootstrap_form form %}
+
+      {% buttons %}
+	<button class="btn btn-primary" type="submit">Add address to account creation whitelist</button>
+      {% endbuttons %}
+    </form>
+  {% endif %}
+{% endblock %}

ietf/templates/registration/manual.html

@@ -0,0 +1,21 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin %}
+
+{% load bootstrap3 %}
+
+{% block title %}Complete account creation{% endblock %}
+
+{% block content %}
+  {% origin %}
+
+    <h1>Account creation failed</h1>
+
+    <p>
+       Manual intervention is needed to enable account creation for you.
+       Please send an email to {{ account_request_email }}
+       and explain 1) the situation and 2) your need for an account,
+       in order to receive further assistance.
+    </p>
+
+{% endblock %}