diff --git a/bin/mm_hourly b/bin/mm_hourly index b66c4dfbc..0d1da2e57 100755 --- a/bin/mm_hourly +++ b/bin/mm_hourly @@ -1,6 +1,6 @@ #!/bin/bash -# Hourly datatracker jobs, run as mailman +# Hourly datatracker jobs, ***run as mailman*** # # This script is expected to be triggered by cron from # $DTDIR/etc/cron.d/datatracker which should be symlinked from @@ -13,8 +13,7 @@ export PYTHONIOENCODING=utf-8 program=${0##*/} trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR -# Note that we're using the last 2.7 release here, not the current release -DTDIR=/a/www/ietf-datatracker/py27 +DTDIR=/a/www/ietf-datatracker/web cd $DTDIR/ # Set up the virtual environment @@ -22,5 +21,4 @@ source $DTDIR/env/bin/activate logger -p user.info -t cron "Running $DTDIR/bin/mm_hourly" -## XXX commented out pending rewrite -- mailman 2 python interface is not available under Python 3 $DTDIR/ietf/manage.py import_mailman_listinfo diff --git a/ietf/bin/mailman_listinfo.py b/ietf/bin/mailman_listinfo.py new file mode 100755 index 000000000..f7e4cfe4c --- /dev/null +++ b/ietf/bin/mailman_listinfo.py @@ -0,0 +1,50 @@ +#!/usr/bin/python2.7 +# Copyright The IETF Trust 2022, All Rights Reserved +# Note the shebang. This specifically targets deployment on IETFA and intends to use its system python2.7. + +# This is an adaptor to pull information out of Mailman2 using its python libraries (which are only available for python2). +# It is NOT django code, and does not have access to django.conf.settings. + +import json +import sys + +from collections import defaultdict + +def main(): + + sys.path.append('/usr/lib/mailman') + + have_mailman = False + try: + from Mailman import Utils + from Mailman import MailList + from Mailman import MemberAdaptor + have_mailman = True + except ImportError: + pass + + + if not have_mailman: + sys.stderr.write("Could not import mailman modules -- skipping import of mailman list info") + sys.exit() + + names = list(Utils.list_names()) + + # need to emit dict of names, each name has an mlist, and each mlist has description, advertised, and members (calculated as below) + result = defaultdict(dict) + for name in names: + mlist = MailList.MailList(name, lock=False) + result[name] = dict() + result[name]['internal_name'] = mlist.internal_name() + result[name]['real_name'] = mlist.real_name + result[name]['description'] = mlist.description # Not attempting to change encoding + result[name]['advertised'] = mlist.advertised + result[name]['members'] = list() + if mlist.advertised: + members = mlist.getRegularMemberKeys() + mlist.getDigestMemberKeys() + members = set([ m for m in members if mlist.getDeliveryStatus(m) == MemberAdaptor.ENABLED ]) + result[name]['members'] = list(members) + json.dump(result, sys.stdout) + +if __name__ == "__main__": + main() diff --git a/ietf/mailinglists/management/commands/import_mailman_listinfo.py b/ietf/mailinglists/management/commands/import_mailman_listinfo.py index 6d51bd151..eae11c4da 100644 --- a/ietf/mailinglists/management/commands/import_mailman_listinfo.py +++ b/ietf/mailinglists/management/commands/import_mailman_listinfo.py @@ -1,29 +1,22 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +import json import sys +import subprocess import time from textwrap import dedent import debug # pyflakes:ignore +from pathlib import Path + from django.conf import settings from django.core.management.base import BaseCommand from django.core.exceptions import MultipleObjectsReturned -sys.path.append(settings.MAILMAN_LIB_DIR) - -have_mailman = False -try: - from Mailman import Utils - from Mailman import MailList - from Mailman import MemberAdaptor - have_mailman = True -except ImportError: - pass from ietf.mailinglists.models import List, Subscribed from ietf.utils.log import log -from ietf.utils.text import decode mark = time.time() @@ -39,13 +32,16 @@ def import_mailman_listinfo(verbosity=0): log(msg+' (%.1fs)'% (t-mark)) mark = t - if not have_mailman: - note("Could not import mailman modules -- skipping import of mailman list info") + cmd = str(Path(settings.BASE_DIR) / "bin" / "mailman_listinfo.py") + result = subprocess.run([cmd], capture_output=True) + if result.stderr: + log("Error exporting information from mailmain") + log(result.stderr) return + mailman_export = json.loads(result.stdout) log("Starting import of list info from Mailman") - names = list(Utils.list_names()) - names.sort() + names = sorted(mailman_export.keys()) log_time("Fetched list of mailman list names") addr_max_length = Subscribed._meta.get_field('email').max_length @@ -53,37 +49,33 @@ def import_mailman_listinfo(verbosity=0): log_time("Computed dictionary of list members") for name in names: - mlist = MailList.MailList(name, lock=False) - note("List: %s" % mlist.internal_name()) + note("List: %s" % mailman_export[name]['internal_name']) log_time("Fetched Mailman list object for %s" % name) - lists = List.objects.filter(name=mlist.real_name) + lists = List.objects.filter(name=mailman_export[name]['real_name']) if lists.count() > 1: # Arbitrary choice; we'll update the remaining item next for item in lists[1:]: item.delete() - mmlist, created = List.objects.get_or_create(name=mlist.real_name) + mmlist, created = List.objects.get_or_create(name=mailman_export[name]['real_name']) dirty = False - desc = decode(mlist.description)[:256] + desc = mailman_export[name]['description'][:256] if mmlist.description != desc: mmlist.description = desc dirty = True - if mmlist.advertised != mlist.advertised: - mmlist.advertised = mlist.advertised + if mmlist.advertised != mailman_export[name]['advertised']: + mmlist.advertised = mailman_export[name]['advertised'] dirty = True if dirty: mmlist.save() log_time(" Updated database List object for %s" % name) # The following calls return lowercased addresses - if mlist.advertised: - members = mlist.getRegularMemberKeys() + mlist.getDigestMemberKeys() - log_time(" Fetched list of list members") - members = set([ m for m in members if mlist.getDeliveryStatus(m) == MemberAdaptor.ENABLED ]) - log_time(" Filtered list of list members") - if not mlist.real_name in subscribed: - log("Note: didn't find '%s' in the dictionary of subscriptions" % mlist.real_name) + if mailman_export[name]['advertised']: + members = set(mailman_export[name]['members']) + if not mailman_export[name]['real_name'] in subscribed: + log("Note: didn't find '%s' in the dictionary of subscriptions" % mailman_export[name]['real_name']) continue - known = subscribed[mlist.real_name] + known = subscribed[mailman_export[name]['real_name']] log_time(" Fetched known list members from database") to_remove = known - members to_add = members - known