# Copyright The IETF Trust 2023, All Rights Reserved from collections import namedtuple import csv import datetime import os import re import shutil from django.conf import settings from django.core.management import BaseCommand from pathlib import Path from zoneinfo import ZoneInfo from ietf.doc.models import DocEvent, Document from ietf.meeting.models import ( Meeting, SchedTimeSessAssignment, Schedule, SchedulingEvent, Session, TimeSlot, ) from ietf.name.models import DocTypeName def add_time_of_day(bare_datetime): """Add a time for the iesg meeting based on a date and make it tzaware From the secretariat - the telechats happened at these times: 2015-04-09 to present: 0700 PT America/Los Angeles 1993-02-01 to 2015-03-12: 1130 ET America/New York 1991-07-30 to 1993-01-25: 1200 ET America/New York """ dt = None if bare_datetime.year > 2015: dt = bare_datetime.replace(hour=7).replace( tzinfo=ZoneInfo("America/Los_Angeles") ) elif bare_datetime.year == 2015: if bare_datetime.month >= 4: dt = bare_datetime.replace(hour=7).replace( tzinfo=ZoneInfo("America/Los_Angeles") ) else: dt = bare_datetime.replace(hour=11, minute=30).replace( tzinfo=ZoneInfo("America/New_York") ) elif bare_datetime.year > 1993: dt = bare_datetime.replace(hour=11, minute=30).replace( tzinfo=ZoneInfo("America/New_York") ) elif bare_datetime.year == 1993: if bare_datetime.month >= 2: dt = bare_datetime.replace(hour=11, minute=30).replace( tzinfo=ZoneInfo("America/New_York") ) else: dt = bare_datetime.replace(hour=12).replace( tzinfo=ZoneInfo("America/New_York") ) else: dt = bare_datetime.replace(hour=12).replace(tzinfo=ZoneInfo("America/New_York")) return dt.astimezone(datetime.timezone.utc) def build_bof_coord_data(): CoordTuple = namedtuple("CoordTuple", "meeting_number source_name") def utc_from_la_time(time): return time.replace(tzinfo=ZoneInfo("America/Los_Angeles")).astimezone( datetime.timezone.utc ) data = dict() data[utc_from_la_time(datetime.datetime(2016, 6, 10, 7, 0))] = CoordTuple( 96, "2015/bof-minutes-ietf-96.txt" ) data[utc_from_la_time(datetime.datetime(2016, 10, 6, 7, 0))] = CoordTuple( 97, "2016/BoF-Minutes-2016-10-06.txt" ) data[utc_from_la_time(datetime.datetime(2017, 2, 15, 8, 0))] = CoordTuple( 98, "2017/bof-minutes-ietf-98.txt" ) data[utc_from_la_time(datetime.datetime(2017, 6, 7, 8, 0))] = CoordTuple( 99, "2017/bof-minutes-ietf-99.txt" ) data[utc_from_la_time(datetime.datetime(2017, 10, 5, 7, 0))] = CoordTuple( 100, "2017/bof-minutes-ietf-100.txt" ) data[utc_from_la_time(datetime.datetime(2018, 2, 5, 11, 0))] = CoordTuple( 101, "2018/bof-minutes-ietf-101.txt" ) data[utc_from_la_time(datetime.datetime(2018, 6, 5, 8, 0))] = CoordTuple( 102, "2018/bof-minutes-ietf-102.txt" ) data[utc_from_la_time(datetime.datetime(2018, 9, 26, 7, 0))] = CoordTuple( 103, "2018/bof-minutes-ietf-103.txt" ) data[utc_from_la_time(datetime.datetime(2019, 2, 15, 9, 0))] = CoordTuple( 104, "2019/bof-minutes-ietf-104.txt" ) data[utc_from_la_time(datetime.datetime(2019, 6, 11, 7, 30))] = CoordTuple( 105, "2019/bof-minutes-ietf-105.txt" ) data[utc_from_la_time(datetime.datetime(2019, 10, 9, 6, 30))] = CoordTuple( 106, "2019/bof-minutes-ietf-106.txt" ) data[utc_from_la_time(datetime.datetime(2020, 2, 13, 8, 0))] = CoordTuple( 107, "2020/bof-minutes-ietf-107.txt" ) data[utc_from_la_time(datetime.datetime(2020, 6, 15, 8, 0))] = CoordTuple( 108, "2020/bof-minutes-ietf-108.txt" ) data[utc_from_la_time(datetime.datetime(2020, 10, 9, 7, 0))] = CoordTuple( 109, "2020/bof-minutes-ietf-109.txt" ) data[utc_from_la_time(datetime.datetime(2021, 1, 14, 13, 30))] = CoordTuple( 110, "2021/bof-minutes-ietf-110.txt" ) data[utc_from_la_time(datetime.datetime(2021, 6, 1, 8, 0))] = CoordTuple( 111, "2021/bof-minutes-ietf-111.txt" ) data[utc_from_la_time(datetime.datetime(2021, 9, 15, 9, 0))] = CoordTuple( 112, "2021/bof-minutes-ietf-112.txt" ) data[utc_from_la_time(datetime.datetime(2022, 1, 28, 7, 0))] = CoordTuple( 113, "2022/bof-minutes-ietf-113.txt" ) data[utc_from_la_time(datetime.datetime(2022, 6, 2, 10, 0))] = CoordTuple( 114, "2022/bof-minutes-ietf-114.txt" ) data[utc_from_la_time(datetime.datetime(2022, 9, 13, 9, 0))] = CoordTuple( 115, "2022/bof-minutes-ietf-115.txt" ) data[utc_from_la_time(datetime.datetime(2023, 2, 1, 9, 0))] = CoordTuple( 116, "2023/bof-minutes-ietf-116.txt" ) data[utc_from_la_time(datetime.datetime(2023, 6, 1, 7, 0))] = CoordTuple( 117, "2023/bof-minutes-ietf-117.txt" ) data[utc_from_la_time(datetime.datetime(2023, 9, 15, 8, 0))] = CoordTuple( 118, "2023/bof-minutes-ietf-118.txt" ) return data class Command(BaseCommand): help = "Performs a one-time import of IESG minutes, creating Meetings to attach them to" def handle(self, *args, **options): old_minutes_root = ( "/a/www/www6/iesg/minutes" if settings.SERVER_MODE == "production" else "/assets/www6/iesg/minutes" ) minutes_dir = Path(old_minutes_root) date_re = re.compile(r"\d{4}-\d{2}-\d{2}") meeting_times = set() redirects = [] for file_prefix in ["minutes", "narrative"]: paths = list(minutes_dir.glob(f"[12][09][0129][0-9]/{file_prefix}*.txt")) paths.extend( list(minutes_dir.glob(f"[12][09][0129][0-9]/{file_prefix}*.html")) ) for path in paths: s = date_re.search(path.name) if s: meeting_times.add( add_time_of_day( datetime.datetime.strptime(s.group(), "%Y-%m-%d") ) ) bof_coord_data = build_bof_coord_data() bof_times = set(bof_coord_data.keys()) assert len(bof_times.intersection(meeting_times)) == 0 meeting_times.update(bof_times) year_seen = None for dt in sorted(meeting_times): if dt.year != year_seen: counter = 1 year_seen = dt.year meeting_name = f"interim-{dt.year}-iesg-{counter:02d}" meeting = Meeting.objects.create( number=meeting_name, type_id="interim", date=dt.date(), days=1, time_zone=dt.tzname(), ) schedule = Schedule.objects.create( meeting=meeting, owner_id=1, # the "(System)" person visible=True, public=True, ) meeting.schedule = schedule meeting.save() session = Session.objects.create( meeting=meeting, group_id=2, # The IESG group type_id="regular", purpose_id="regular", name=( f"IETF {bof_coord_data[dt].meeting_number} BOF Coordination Call" if dt in bof_times else "Formal Telechat" ), ) SchedulingEvent.objects.create( session=session, status_id="sched", by_id=1, # (System) ) timeslot = TimeSlot.objects.create( meeting=meeting, type_id="regular", time=dt, duration=datetime.timedelta(seconds=2 * 60 * 60), ) SchedTimeSessAssignment.objects.create( timeslot=timeslot, session=session, schedule=schedule ) if dt in bof_times: source = minutes_dir / bof_coord_data[dt].source_name if source.exists(): doc_name = ( f"minutes-interim-{dt.year}-iesg-{counter:02d}-{dt:%Y%m%d%H%M}" ) doc_filename = f"{doc_name}-00.txt" doc = Document.objects.create( name=doc_name, type_id="minutes", title=f"Minutes IETF {bof_coord_data[dt].meeting_number} BOF coordination {meeting_name} {dt:%Y-%m-%d %H:%M}", group_id=2, # the IESG group rev="00", uploaded_filename=doc_filename, ) e = DocEvent.objects.create( type="comment", doc=doc, rev="00", by_id=1, # "(System)" desc="Minutes moved into datatracker", ) doc.save_with_history([e]) session.presentations.create(document=doc, rev=doc.rev) dest = ( Path(settings.AGENDA_PATH) / meeting_name / "minutes" / doc_filename ) if dest.exists(): self.stdout.write( f"WARNING: {dest} already exists - not overwriting it." ) else: os.makedirs(dest.parent, exist_ok=True) shutil.copy(source, dest) redirects.append( [ f"www6.ietf.org/iesg/minutes/{dt.year}/{bof_coord_data[dt].source_name}", f"https://datatracker.ietf.org/doc/{doc_name}", 302, ] ) else: for type_id in ["minutes", "narrativeminutes"]: source_file_prefix = ( "minutes" if type_id == "minutes" else "narrative-minutes" ) txt_source = ( minutes_dir / f"{dt.year}" / f"{source_file_prefix}-{dt:%Y-%m-%d}.txt" ) html_source = ( minutes_dir / f"{dt.year}" / f"{source_file_prefix}-{dt:%Y-%m-%d}.html" ) if txt_source.exists() and html_source.exists(): self.stdout.write( f"WARNING: Both {txt_source} and {html_source} exist." ) if txt_source.exists() or html_source.exists(): prefix = DocTypeName.objects.get(slug=type_id).prefix doc_name = f"{prefix}-interim-{dt.year}-iesg-{counter:02d}-{dt:%Y%m%d%H%M}" suffix = "html" if html_source.exists() else "txt" doc_filename = f"{doc_name}-00.{suffix}" verbose_type = ( "Minutes" if type_id == "minutes" else "Narrative Minutes" ) doc = Document.objects.create( name=doc_name, type_id=type_id, title=f"{verbose_type} {meeting_name} {dt:%Y-%m-%d %H:%M}", group_id=2, # the IESG group rev="00", uploaded_filename=doc_filename, ) e = DocEvent.objects.create( type="comment", doc=doc, rev="00", by_id=1, # "(System)" desc=f"{verbose_type} moved into datatracker", ) doc.save_with_history([e]) session.presentations.create(document=doc, rev=doc.rev) dest = ( Path(settings.AGENDA_PATH) / meeting_name / type_id / doc_filename ) if dest.exists(): self.stdout.write( f"WARNING: {dest} already exists - not overwriting it." ) else: os.makedirs(dest.parent, exist_ok=True) if html_source.exists(): html_content = html_source.read_text(encoding="utf-8") html_content = html_content.replace( f'href="IESGnarrative-{dt:%Y-%m-%d}.html#', 'href="#', ) html_content = re.sub( r']*>([^<]*)', r"\1", html_content, ) html_content = re.sub( r'([^<]*)', r"\1", html_content, ) html_content = re.sub( '