altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

feat: decouple from mailman2 - explicitly model nonwg mailing lists (#7013)

Browse source 
* fix: remove synchronization with mailman2

* feat: manage non wg mailing lists explicitly

* chore: black

* fix: update tests for new nonwg view

* feat: drop unused models
This commit is contained in:
Robert Sparks 2024-02-05 09:28:23 -06:00 committed by GitHub
parent b4cf04a09d
commit efdaee3bb3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 688 additions and 287 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
ietf/bin/mailman_listinfo.py
View file

@ -1,50 +0,0 @@
#!/usr/bin/python2.7
# Copyright The IETF Trust 2022, All Rights Reserved
# Note the shebang. This specifically targets deployment on IETFA and intends to use its system python2.7.
# This is an adaptor to pull information out of Mailman2 using its python libraries (which are only available for python2).
# It is NOT django code, and does not have access to django.conf.settings.
import json
import sys
from collections import defaultdict
def main():
sys.path.append('/usr/lib/mailman')
have_mailman = False
try:
from Mailman import Utils
from Mailman import MailList
from Mailman import MemberAdaptor
have_mailman = True
except ImportError:
pass
if not have_mailman:
sys.stderr.write("Could not import mailman modules -- skipping import of mailman list info")
sys.exit()
names = list(Utils.list_names())
# need to emit dict of names, each name has an mlist, and each mlist has description, advertised, and members (calculated as below)
result = defaultdict(dict)
for name in names:
mlist = MailList.MailList(name, lock=False)
result[name] = dict()
result[name]['internal_name'] = mlist.internal_name()
result[name]['real_name'] = mlist.real_name
result[name]['description'] = mlist.description # Not attempting to change encoding
result[name]['advertised'] = mlist.advertised
result[name]['members'] = list()
if mlist.advertised:
members = mlist.getRegularMemberKeys() + mlist.getDigestMemberKeys()
members = set([ m for m in members if mlist.getDeliveryStatus(m) == MemberAdaptor.ENABLED ])
result[name]['members'] = list(members)
json.dump(result, sys.stdout)
if __name__ == "__main__":
main()

13
ietf/ietfauth/tests.py
View file

@ -37,7 +37,6 @@ from ietf.group.factories import GroupFactory, RoleFactory
from ietf.group.models import Group, Role, RoleName from ietf.group.models import Group, Role, RoleName
from ietf.ietfauth.htpasswd import update_htpasswd_file from ietf.ietfauth.htpasswd import update_htpasswd_file
from ietf.ietfauth.utils import has_role from ietf.ietfauth.utils import has_role
from ietf.mailinglists.models import Subscribed
from ietf.meeting.factories import MeetingFactory from ietf.meeting.factories import MeetingFactory
from ietf.nomcom.factories import NomComFactory from ietf.nomcom.factories import NomComFactory
from ietf.person.factories import PersonFactory, EmailFactory, UserFactory, PersonalApiKeyFactory from ietf.person.factories import PersonFactory, EmailFactory, UserFactory, PersonalApiKeyFactory
@ -250,18 +249,6 @@ class IetfAuthTests(TestCase):
# register and verify allowlisted email # register and verify allowlisted email
self.register_and_verify(email) self.register_and_verify(email)
def test_create_subscribed_account(self):
# verify creation with email in subscribed list
saved_delay = settings.LIST_ACCOUNT_DELAY
settings.LIST_ACCOUNT_DELAY = 1
email = "subscribed@example.com"
s = Subscribed(email=email)
s.save()
time.sleep(1.1)
self.register_and_verify(email)
settings.LIST_ACCOUNT_DELAY = saved_delay
def test_create_existing_account(self): def test_create_existing_account(self):
# create account once # create account once
email = "new-account@example.com" email = "new-account@example.com"

10
ietf/ietfauth/views.py
View file

@ -160,18 +160,8 @@ def create_account(request):
) )
new_account_email = None # Indicate to the template that we failed to create the requested account new_account_email = None # Indicate to the template that we failed to create the requested account
else: else:
# For the IETF 113 Registration period (at least) we are lowering the
# barriers for account creation to the simple email round-trip check
send_account_creation_email(request, new_account_email) send_account_creation_email(request, new_account_email)
# The following is what to revert to should that lowered barrier prove problematic
# existing = Subscribed.objects.filter(email__iexact=new_account_email).first()
# ok_to_create = ( Allowlisted.objects.filter(email__iexact=new_account_email).exists()
# or existing and (existing.time + TimeDelta(seconds=settings.LIST_ACCOUNT_DELAY)) < DateTime.now() )
# if ok_to_create:
# send_account_creation_email(request, new_account_email)
# else:
# return render(request, 'registration/manual.html', { 'account_request_email': settings.ACCOUNT_REQUEST_EMAIL })
else: else:
form = RegistrationForm() form = RegistrationForm()

17
ietf/mailinglists/admin.py
View file

@ -2,20 +2,15 @@
from django.contrib import admin from django.contrib import admin
from ietf.mailinglists.models import List, Subscribed, Allowlisted from ietf.mailinglists.models import NonWgMailingList, Allowlisted
class ListAdmin(admin.ModelAdmin):
list_display = ('id', 'name', 'description', 'advertised')
class NonWgMailingListAdmin(admin.ModelAdmin):
list_display = ('id', 'name', 'description')
search_fields = ('name',) search_fields = ('name',)
admin.site.register(List, ListAdmin) admin.site.register(NonWgMailingList, NonWgMailingListAdmin)
class SubscribedAdmin(admin.ModelAdmin):
list_display = ('id', 'time', 'email')
raw_id_fields = ('lists',)
search_fields = ('email',)
admin.site.register(Subscribed, SubscribedAdmin)
class AllowlistedAdmin(admin.ModelAdmin): class AllowlistedAdmin(admin.ModelAdmin):

8
ietf/mailinglists/factories.py
View file

@ -3,16 +3,14 @@
import factory import factory
import random
from ietf.mailinglists.models import List from ietf.mailinglists.models import NonWgMailingList
class ListFactory(factory.django.DjangoModelFactory): class NonWgMailingListFactory(factory.django.DjangoModelFactory):
class Meta: class Meta:
model = List model = NonWgMailingList
name = factory.Sequence(lambda n: "list-name-%s" % n) name = factory.Sequence(lambda n: "list-name-%s" % n)
description = factory.Faker('sentence', nb_words=10) description = factory.Faker('sentence', nb_words=10)
advertised = factory.LazyAttribute(lambda obj: random.randint(0, 1))

130
ietf/mailinglists/management/commands/import_mailman_listinfo.py
View file

@ -1,130 +0,0 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
import json
import sys
import subprocess
import time
from textwrap import dedent
import debug # pyflakes:ignore
from pathlib import Path
from django.conf import settings
from django.core.management.base import BaseCommand
from django.core.exceptions import MultipleObjectsReturned
from ietf.mailinglists.models import List, Subscribed
from ietf.utils.log import log
mark = time.time()
def import_mailman_listinfo(verbosity=0):
def note(msg):
if verbosity > 2:
sys.stdout.write(msg)
sys.stdout.write('\n')
def log_time(msg):
global mark
if verbosity > 1:
t = time.time()
log(msg+' (%.1fs)'% (t-mark))
mark = t
cmd = str(Path(settings.BASE_DIR) / "bin" / "mailman_listinfo.py")
result = subprocess.run([cmd], capture_output=True)
if result.stderr:
log("Error exporting information from mailmain")
log(result.stderr)
return
mailman_export = json.loads(result.stdout)
names = sorted(mailman_export.keys())
addr_max_length = Subscribed._meta.get_field('email').max_length
subscribed = { l.name: set(l.subscribed_set.values_list('email', flat=True)) for l in List.objects.all().prefetch_related('subscribed_set') }
for name in names:
note("List: %s" % mailman_export[name]['internal_name'])
lists = List.objects.filter(name=mailman_export[name]['real_name'])
if lists.count() > 1:
# Arbitrary choice; we'll update the remaining item next
for item in lists[1:]:
item.delete()
mmlist, created = List.objects.get_or_create(name=mailman_export[name]['real_name'])
dirty = False
desc = mailman_export[name]['description'][:256]
if mmlist.description != desc:
mmlist.description = desc
dirty = True
if mmlist.advertised != mailman_export[name]['advertised']:
mmlist.advertised = mailman_export[name]['advertised']
dirty = True
if dirty:
mmlist.save()
# The following calls return lowercased addresses
if mailman_export[name]['advertised']:
members = set(mailman_export[name]['members'])
if not mailman_export[name]['real_name'] in subscribed:
# 2022-7-29: lots of these going into the logs but being ignored...
# log("Note: didn't find '%s' in the dictionary of subscriptions" % mailman_export[name]['real_name'])
continue
known = subscribed[mailman_export[name]['real_name']]
log_time(" Fetched known list members from database")
to_remove = known - members
to_add = members - known
for addr in to_remove:
note(" Removing subscription: %s" % (addr))
old = Subscribed.objects.get(email=addr) # Intentionally leaving this as case-sensitive in postgres
old.lists.remove(mmlist)
if old.lists.count() == 0:
note(" Removing address with no subscriptions: %s" % (addr))
old.delete()
if to_remove:
log(" Removed %s addresses from %s" % (len(to_remove), name))
for addr in to_add:
if len(addr) > addr_max_length:
sys.stderr.write(" ** Email address subscribed to '%s' too long for table: <%s>\n" % (name, addr))
continue
note(" Adding subscription: %s" % (addr))
try:
new, created = Subscribed.objects.get_or_create(email=addr) # Intentionally leaving this as case-sensitive in postgres
except MultipleObjectsReturned as e:
sys.stderr.write(" ** Error handling %s in %s: %s\n" % (addr, name, e))
continue
new.lists.add(mmlist)
if to_add:
log(" Added %s addresses to %s" % (len(to_add), name))
log("Completed import of list info from Mailman")
class Command(BaseCommand):
"""
Import list information from Mailman.
Import announced list names, descriptions, and subscribers, by calling the
appropriate Mailman functions and adding entries to the database.
Run this from cron regularly, with sufficient permissions to access the
mailman database files.
"""
help = dedent(__doc__).strip()
#option_list = BaseCommand.option_list + ( )
def handle(self, *filenames, **options):
"""
* Import announced lists, with appropriate meta-information.
* For each list, import the members.
"""
verbosity = int(options.get('verbosity'))
import_mailman_listinfo(verbosity)

628
ietf/mailinglists/migrations/0002_nonwgmailinglist.py Normal file
View file

@ -0,0 +1,628 @@
# Copyright The IETF Trust 2024, All Rights Reserved
from django.db import migrations, models
def forward(apps, schema_editor):
NonWgMailingList = apps.get_model("mailinglists", "NonWgMailingList")
List = apps.get_model("mailinglists", "List")
for l in List.objects.filter(
pk__in=[
10754,
10769,
10770,
10768,
10787,
10785,
10791,
10786,
10816,
10817,
10819,
10818,
10922,
10923,
10921,
10940,
10941,
10942,
572,
10297,
182,
43,
10704,
10314,
201,
419,
282,
149,
223,
10874,
10598,
10639,
10875,
10737,
105,
65,
10781,
10771,
10946,
518,
421,
214,
285,
393,
445,
553,
183,
10725,
33,
10766,
114,
417,
10789,
10876,
4244,
10705,
10706,
10878,
10324,
10879,
10642,
10821,
547,
532,
10636,
10592,
327,
248,
10697,
288,
346,
10731,
10955,
10857,
446,
55,
10799,
10800,
10801,
10612,
73,
3,
358,
9640,
10868,
378,
462,
6595,
10914,
10915,
197,
63,
558,
10824,
124,
10881,
177,
312,
252,
185,
523,
4572,
10618,
206,
68,
10859,
560,
513,
246,
7817,
148,
10864,
10589,
10773,
10748,
364,
311,
10302,
10272,
10929,
171,
10865,
10919,
377,
469,
467,
411,
505,
6318,
10811,
10304,
10882,
10845,
568,
10883,
4774,
264,
10779,
10884,
10303,
409,
10590,
451,
10749,
10765,
486,
519,
10593,
10313,
550,
10707,
307,
10861,
10654,
10708,
10275,
134,
460,
10911,
10574,
10885,
10814,
10676,
10747,
10305,
10688,
36,
10844,
10620,
458,
10282,
10594,
10752,
389,
296,
10684,
48,
533,
443,
10739,
491,
139,
461,
10690,
424,
290,
336,
31,
10709,
382,
10866,
10724,
539,
10710,
559,
10609,
74,
10582,
133,
10621,
34,
10596,
442,
13,
56,
128,
323,
10285,
80,
315,
3520,
10949,
10950,
189,
2599,
10822,
164,
10267,
10286,
464,
440,
254,
262,
10943,
465,
75,
179,
162,
457,
10572,
372,
452,
10273,
88,
366,
331,
140,
407,
416,
91,
10632,
542,
151,
117,
431,
10628,
10271,
14,
540,
278,
352,
159,
10851,
9981,
10694,
10619,
10732,
320,
348,
338,
349,
10678,
468,
293,
350,
402,
57,
524,
141,
71,
67,
508,
7828,
10268,
10631,
10713,
10889,
345,
78,
342,
190,
10869,
46,
334,
255,
5823,
400,
10867,
23,
10666,
10685,
405,
2801,
92,
137,
10640,
10656,
104,
123,
10643,
10891,
466,
10567,
10318,
526,
30,
222,
194,
10735,
10714,
247,
493,
1162,
414,
10648,
10677,
126,
16,
422,
271,
295,
81,
10634,
544,
10850,
426,
573,
353,
10829,
538,
10913,
10566,
167,
10675,
272,
10673,
10767,
528,
284,
564,
268,
10825,
231,
520,
10645,
10872,
515,
10956,
10947,
569,
233,
10952,
195,
10938,
2809,
10591,
10665,
9639,
10775,
10760,
10715,
10716,
10667,
361,
184,
10935,
10957,
10944,
94,
449,
525,
1962,
10300,
10894,
9156,
10774,
256,
289,
218,
187,
40,
10777,
10761,
10670,
249,
10764,
420,
548,
232,
410,
196,
72,
335,
70,
146,
10287,
10299,
10311,
10895,
10617,
531,
343,
10934,
10933,
10597,
158,
10600,
10692,
8630,
556,
324,
11,
10784,
498,
10772,
478,
10833,
10691,
391,
10565,
10669,
113,
110,
7831,
10855,
10312,
10315,
10896,
10672,
10306,
438,
395,
82,
10599,
10953,
10858,
10807,
10717,
310,
10808,
119,
10595,
10718,
10317,
10898,
454,
427,
10583,
10916,
403,
10843,
10899,
291,
10812,
10900,
10794,
341,
121,
230,
136,
166,
394,
234,
10901,
2466,
10573,
10939,
221,
490,
10820,
10873,
10792,
10870,
10793,
10904,
181,
10693,
482,
10611,
125,
10568,
10788,
211,
10756,
10719,
100,
228,
5833,
251,
122,
39,
534,
437,
504,
10613,
439,
306,
10863,
10823,
10926,
76,
227,
59,
42,
455,
10927,
10928,
204,
430,
10720,
267,
396,
10849,
10308,
281,
10905,
10736,
168,
153,
385,
89,
529,
412,
215,
484,
10951,
66,
173,
10633,
10681,
3613,
10274,
10750,
367,
387,
10832,
35,
147,
10325,
10671,
565,
313,
10871,
10751,
37,
10936,
10937,
287,
496,
244,
10841,
10683,
10906,
10584,
479,
10856,
163,
10910,
257,
276,
10840,
10689,
365,
10847,
99,
77,
435,
213,
15,
10932,
58,
10722,
131,
363,
10674,
322,
180,
10917,
10918,
10738,
10954,
10581,
208,
337,
4,
571,
10668,
10291,
]
):
NonWgMailingList.objects.create(name=l.name, description=l.description)
class Migration(migrations.Migration):
dependencies = [
("mailinglists", "0001_initial"),
]
operations = [
migrations.CreateModel(
name="NonWgMailingList",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.CharField(max_length=32)),
("description", models.CharField(max_length=256)),
],
),
migrations.RunPython(forward),
]

23
ietf/mailinglists/migrations/0003_remove_subscribed_lists_delete_list_and_more.py Normal file
View file

@ -0,0 +1,23 @@
# Generated by Django 4.2.9 on 2024-02-02 23:04
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("mailinglists", "0002_nonwgmailinglist"),
]
operations = [
migrations.RemoveField(
model_name="subscribed",
name="lists",
),
migrations.DeleteModel(
name="List",
),
migrations.DeleteModel(
name="Subscribed",
),
]

17
ietf/mailinglists/models.py
View file

@ -9,25 +9,18 @@ from django.db import models
from ietf.person.models import Person from ietf.person.models import Person
from ietf.utils.models import ForeignKey from ietf.utils.models import ForeignKey
class List(models.Model):
# NonWgMailingList is a temporary bridging class to hold information known about mailman2
# while decoupling from mailman2 until we integrate with mailman3
class NonWgMailingList(models.Model):
name = models.CharField(max_length=32) name = models.CharField(max_length=32)
description = models.CharField(max_length=256) description = models.CharField(max_length=256)
advertised = models.BooleanField(default=True)
def __str__(self): def __str__(self):
return "<List: %s>" % self.name return "<NonWgMailingList: %s>" % self.name
def info_url(self): def info_url(self):
return settings.MAILING_LIST_INFO_URL % {'list_addr': self.name } return settings.MAILING_LIST_INFO_URL % {'list_addr': self.name }
class Subscribed(models.Model):
time = models.DateTimeField(auto_now_add=True)
email = models.CharField(max_length=128, validators=[validate_email])
lists = models.ManyToManyField(List)
def __str__(self):
return "<Subscribed: %s at %s>" % (self.email, self.time)
class Meta:
verbose_name_plural = "Subscribed"
class Allowlisted(models.Model): class Allowlisted(models.Model):
time = models.DateTimeField(auto_now_add=True) time = models.DateTimeField(auto_now_add=True)
email = models.CharField("Email address", max_length=64, validators=[validate_email]) email = models.CharField("Email address", max_length=64, validators=[validate_email])

27
ietf/mailinglists/resources.py
View file

@ -11,7 +11,7 @@ from tastypie.cache import SimpleCache
from ietf import api from ietf import api
from ietf.api import ToOneField # pyflakes:ignore from ietf.api import ToOneField # pyflakes:ignore
from ietf.mailinglists.models import Allowlisted, List, Subscribed from ietf.mailinglists.models import Allowlisted, NonWgMailingList
from ietf.person.resources import PersonResource from ietf.person.resources import PersonResource
@ -31,34 +31,19 @@ class AllowlistedResource(ModelResource):
} }
api.mailinglists.register(AllowlistedResource()) api.mailinglists.register(AllowlistedResource())
class ListResource(ModelResource): class NonWgMailingListResource(ModelResource):
class Meta: class Meta:
queryset = List.objects.all() queryset = NonWgMailingList.objects.all()
serializer = api.Serializer() serializer = api.Serializer()
cache = SimpleCache() cache = SimpleCache()
#resource_name = 'list' #resource_name = 'nonwgmailinglist'
ordering = ['id', ] ordering = ['id', ]
filtering = { filtering = {
"id": ALL, "id": ALL,
"name": ALL, "name": ALL,
"description": ALL, "description": ALL,
"advertised": ALL,
} }
api.mailinglists.register(ListResource()) api.mailinglists.register(NonWgMailingListResource())
class SubscribedResource(ModelResource):
lists = ToManyField(ListResource, 'lists', null=True)
class Meta:
queryset = Subscribed.objects.all()
serializer = api.Serializer()
cache = SimpleCache()
#resource_name = 'subscribed'
ordering = ['id', ]
filtering = {
"id": ALL,
"time": ALL,
"email": ALL,
"lists": ALL_WITH_RELATIONS,
}
api.mailinglists.register(SubscribedResource())

16
ietf/mailinglists/tests.py
View file

@ -9,7 +9,7 @@ from django.urls import reverse as urlreverse
import debug # pyflakes:ignore import debug # pyflakes:ignore
from ietf.group.factories import GroupFactory from ietf.group.factories import GroupFactory
from ietf.mailinglists.factories import ListFactory from ietf.mailinglists.factories import NonWgMailingListFactory
from ietf.utils.test_utils import TestCase from ietf.utils.test_utils import TestCase
@ -32,23 +32,13 @@ class MailingListTests(TestCase):
def test_nonwg(self): def test_nonwg(self):
groups = list()
groups.append(GroupFactory(type_id='wg', acronym='mars', list_archive='https://ietf.org/mars')) lists = NonWgMailingListFactory.create_batch(7)
groups.append(GroupFactory(type_id='wg', acronym='ames', state_id='conclude', list_archive='https://ietf.org/ames'))
groups.append(GroupFactory(type_id='wg', acronym='newstuff', state_id='bof', list_archive='https://ietf.org/newstuff'))
groups.append(GroupFactory(type_id='rg', acronym='research', list_archive='https://irtf.org/research'))
lists = ListFactory.create_batch(7)
url = urlreverse("ietf.mailinglists.views.nonwg") url = urlreverse("ietf.mailinglists.views.nonwg")
r = self.client.get(url) r = self.client.get(url)
for l in lists: for l in lists:
if l.advertised:
self.assertContains(r, l.name) self.assertContains(r, l.name)
self.assertContains(r, l.description) self.assertContains(r, l.description)
else:
self.assertNotContains(r, l.name, html=True)
self.assertNotContains(r, l.description, html=True)
for g in groups:
self.assertNotContains(r, g.acronym, html=True)

34
ietf/mailinglists/views.py
View file

@ -1,33 +1,25 @@
# Copyright The IETF Trust 2007-2022, All Rights Reserved # Copyright The IETF Trust 2007-2022, All Rights Reserved
import re
from django.shortcuts import render from django.shortcuts import render
import debug # pyflakes:ignore import debug # pyflakes:ignore
from ietf.group.models import Group from ietf.group.models import Group
from ietf.mailinglists.models import List from ietf.mailinglists.models import NonWgMailingList
def groups(request): def groups(request):
groups = Group.objects.filter(type__features__acts_like_wg=True, list_archive__startswith='http').exclude(state__in=('bof', 'conclude')).order_by("acronym") groups = (
Group.objects.filter(
type__features__acts_like_wg=True, list_archive__startswith="http"
)
.exclude(state__in=("bof", "conclude"))
.order_by("acronym")
)
return render(request, "mailinglists/group_archives.html", {"groups": groups})
return render(request, "mailinglists/group_archives.html", { "groups": groups } )
def nonwg(request): def nonwg(request):
groups = Group.objects.filter(type__features__acts_like_wg=True).exclude(state__in=['bof']).order_by("acronym") lists = NonWgMailingList.objects.order_by("name")
return render(request, "mailinglists/nonwg.html", {"lists": lists})
#urls = [ g.list_archive for g in groups if '.ietf.org' in g.list_archive ]
wg_lists = set()
for g in groups:
wg_lists.add(g.acronym)
match = re.search(r'^(https?://mailarchive.ietf.org/arch/(browse/|search/\?email-list=))(?P<name>[^/]*)/?$', g.list_archive)
if match:
wg_lists.add(match.group('name').lower())
lists = List.objects.filter(advertised=True)
#debug.show('lists.count()')
lists = lists.exclude(name__in=wg_lists).order_by('name')
#debug.show('lists.count()')
return render(request, "mailinglists/nonwg.html", { "lists": lists } )