altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

chore: migrate timestamps for use with USE_TZ=True (#4370)

Browse source 
* chore: add migration to change timestamps to UTC

* chore: fill in Meeting.time_zone where it is blank

Nearly all interim meetings on or before 2016-07-01 have blank
time_zone values. This migration fills these in with PST8PDT.

* chore: disallow blank Meeting.time_zone value

* refactor: no need to handle blank time_zone case in TZ migration

* refactor: remove now-unnecessary checks that meeting has time_zone

* chore: renumber migrations

* chore: update timestamp conversion migration

The django-celery-beat package introduces tables with timestamp
columns. These columns are stored in CELERY_TIMEZONE. Because we run with
this set to UTC, the migration ignores these columns.

* chore: fix pytz-related change in migration

* chore: be explicit that Meeting.vtimezone can return None

* refactor: remove unnecessary save()
This commit is contained in:
Jennifer Richards 2022-08-26 13:03:19 -03:00 committed by GitHub
parent ebebdbed3e
commit 42203d7a9c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 361 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
ietf/meeting/migrations/0057_fill_in_empty_meeting_time_zone.py Normal file
View file

@ -0,0 +1,47 @@
# Generated by Django 2.2.28 on 2022-08-08 11:37
import datetime
from django.db import migrations
# date of last meeting with an empty time_zone before this migration
LAST_EMPTY_TZ = datetime.date(2022, 7, 1)
def forward(apps, schema_editor):
Meeting = apps.get_model('meeting', 'Meeting')
# Check that we will be able to identify the migrated meetings later
old_meetings_in_pst8pdt = Meeting.objects.filter(type_id='interim', time_zone='PST8PDT', date__lte=LAST_EMPTY_TZ)
assert old_meetings_in_pst8pdt.count() == 0, 'not expecting interim meetings in PST8PDT time_zone'
meetings_with_empty_tz = Meeting.objects.filter(time_zone='')
# check our expected conditions
for mtg in meetings_with_empty_tz:
assert mtg.type_id == 'interim', 'was not expecting non-interim meetings to be affected'
assert mtg.date <= LAST_EMPTY_TZ, 'affected meeting outside expected date range'
mtg.time_zone = 'PST8PDT'
# commit the changes
Meeting.objects.bulk_update(meetings_with_empty_tz, ['time_zone'])
def reverse(apps, schema_editor):
Meeting = apps.get_model('meeting', 'Meeting')
meetings_to_restore = Meeting.objects.filter(time_zone='PST8PDT', date__lte=LAST_EMPTY_TZ)
for mtg in meetings_to_restore:
mtg.time_zone = ''
# commit the changes
Meeting.objects.bulk_update(meetings_to_restore, ['time_zone'])
class Migration(migrations.Migration):
dependencies = [
('meeting', '0056_use_timezone_now_for_meeting_models'),
]
operations = [
migrations.RunPython(forward, reverse),
]

18
ietf/meeting/migrations/0058_meeting_time_zone_not_blank.py Normal file
View file

File diff suppressed because one or more lines are too long

49
ietf/meeting/models.py
View file

@ -86,7 +86,7 @@ class Meeting(models.Model):
# We can't derive time-zone from country, as there are some that have
# more than one timezone, and the pytz module doesn't provide timezone
# lookup information for all relevant city/country combinations.
time_zone = models.CharField(blank=True, max_length=255, choices=timezones)
time_zone = models.CharField(max_length=255, choices=timezones, default='UTC')
idsubmit_cutoff_day_offset_00 = models.IntegerField(blank=True,
default=settings.IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_00,
help_text = "The number of days before the meeting start date when the submission of -00 drafts will be closed.")
@ -355,19 +355,18 @@ class Meeting(models.Model):
# timeslot = ts)
def vtimezone(self):
if self.time_zone:
try:
tzfn = os.path.join(settings.TZDATA_ICS_PATH, self.time_zone + ".ics")
if os.path.exists(tzfn):
with io.open(tzfn) as tzf:
icstext = tzf.read()
vtimezone = re.search("(?sm)(\nBEGIN:VTIMEZONE.*\nEND:VTIMEZONE\n)", icstext).group(1).strip()
if vtimezone:
vtimezone += "\n"
return vtimezone
except IOError:
pass
return ''
try:
tzfn = os.path.join(settings.TZDATA_ICS_PATH, self.time_zone + ".ics")
if os.path.exists(tzfn):
with io.open(tzfn) as tzf:
icstext = tzf.read()
vtimezone = re.search("(?sm)(\nBEGIN:VTIMEZONE.*\nEND:VTIMEZONE\n)", icstext).group(1).strip()
if vtimezone:
vtimezone += "\n"
return vtimezone
except IOError:
pass
return None
def set_official_schedule(self, schedule):
if self.schedule != schedule:
@ -606,24 +605,15 @@ class TimeSlot(models.Model):
def tz(self):
if not hasattr(self, '_cached_tz'):
if self.meeting.time_zone:
self._cached_tz = pytz.timezone(self.meeting.time_zone)
else:
self._cached_tz = None
self._cached_tz = pytz.timezone(self.meeting.time_zone)
return self._cached_tz
def tzname(self):
if self.tz():
return self.tz().tzname(self.time)
else:
return ""
return self.tz().tzname(self.time)
def utc_start_time(self):
if self.tz():
local_start_time = self.tz().localize(self.time)
return local_start_time.astimezone(pytz.utc)
else:
return None
local_start_time = self.tz().localize(self.time)
return local_start_time.astimezone(pytz.utc)
def utc_end_time(self):
utc_start = self.utc_start_time()
@ -631,10 +621,7 @@ class TimeSlot(models.Model):
return None if utc_start is None else utc_start + self.duration
def local_start_time(self):
if self.tz():
return self.tz().localize(self.time)
else:
return None
return self.tz().localize(self.time)
def local_end_time(self):
local_start = self.local_start_time()

18
ietf/meeting/tests_models.py
View file

@ -3,6 +3,8 @@
"""Tests of models in the Meeting application"""
import datetime
from mock import patch
from ietf.meeting.factories import MeetingFactory, SessionFactory
from ietf.stats.factories import MeetingRegistrationFactory
from ietf.utils.test_utils import TestCase
@ -52,6 +54,22 @@ class MeetingTests(TestCase):
self.assertEqual(attendance.online, 0)
self.assertEqual(attendance.onsite, 5)
def test_vtimezone(self):
# normal time zone that should have a zoneinfo file
meeting = MeetingFactory(type_id='ietf', time_zone='America/Los_Angeles')
vtz = meeting.vtimezone()
self.assertIsNotNone(vtz)
self.assertGreater(len(vtz), 0)
# time zone that does not have a zoneinfo file should return None
meeting = MeetingFactory(type_id='ietf', time_zone='Fake/Time_Zone')
vtz = meeting.vtimezone()
self.assertIsNone(vtz)
# ioerror trying to read zoneinfo should return None
meeting = MeetingFactory(type_id='ietf', time_zone='America/Los_Angeles')
with patch('ietf.meeting.models.io.open', side_effect=IOError):
vtz = meeting.vtimezone()
self.assertIsNone(vtz)
class SessionTests(TestCase):
def test_chat_archive_url_with_jabber(self):

5
ietf/meeting/views.py
View file

@ -3662,9 +3662,12 @@ def upcoming_ical(request):
ietfs = [m for m in meetings if m.type_id == 'ietf']
preprocess_meeting_important_dates(ietfs)
meeting_vtz = {meeting.vtimezone() for meeting in meetings}
meeting_vtz.discard(None)
# icalendar response file should have '\r\n' line endings per RFC5545
response = render_to_string('meeting/upcoming.ics', {
'vtimezones': ''.join(sorted(list({meeting.vtimezone() for meeting in meetings if meeting.vtimezone()}))),
'vtimezones': ''.join(sorted(meeting_vtz)),
'assignments': assignments,
'ietfs': ietfs,
}, request=request)

256
ietf/utils/migrations/0002_convert_timestamps_to_utc.py Normal file
View file

@ -0,0 +1,256 @@
# Generated by Django 2.2.28 on 2022-06-21 11:44
from django.conf import settings
from django.db import migrations, connection
# to generate the expected list:
#
# from django.db import connection
# from pprint import pp
# cursor = connection.cursor()
# cursor.execute("""
# SELECT table_name, column_name
# FROM information_schema.columns
# WHERE table_schema='ietf_utf8'
# AND column_type LIKE 'datetime%'
# AND NOT table_name LIKE 'django_celery_beat_%'
# ORDER BY table_name, column_name;
# """)
# pp(cursor.fetchall())
#
expected_datetime_columns = (
('auth_user', 'date_joined'),
('auth_user', 'last_login'),
('community_documentchangedates', 'new_version_date'),
('community_documentchangedates', 'normal_change_date'),
('community_documentchangedates', 'significant_change_date'),
('django_admin_log', 'action_time'),
('django_migrations', 'applied'),
('django_session', 'expire_date'),
('doc_ballotpositiondocevent', 'comment_time'),
('doc_ballotpositiondocevent', 'discuss_time'),
('doc_deletedevent', 'time'),
('doc_docevent', 'time'),
('doc_dochistory', 'expires'),
('doc_dochistory', 'time'),
('doc_docreminder', 'due'),
('doc_document', 'expires'),
('doc_document', 'time'),
('doc_documentactionholder', 'time_added'),
('doc_initialreviewdocevent', 'expires'),
('doc_irsgballotdocevent', 'duedate'),
('doc_lastcalldocevent', 'expires'),
('group_group', 'time'),
('group_groupevent', 'time'),
('group_grouphistory', 'time'),
('group_groupmilestone', 'time'),
('group_groupmilestonehistory', 'time'),
('ipr_iprdisclosurebase', 'time'),
('ipr_iprevent', 'response_due'),
('ipr_iprevent', 'time'),
('liaisons_liaisonstatementevent', 'time'),
('mailinglists_subscribed', 'time'),
('mailinglists_whitelisted', 'time'),
('meeting_floorplan', 'modified'),
('meeting_room', 'modified'),
('meeting_schedtimesessassignment', 'modified'),
('meeting_schedulingevent', 'time'),
('meeting_session', 'modified'),
('meeting_session', 'scheduled'),
('meeting_slidesubmission', 'time'),
('meeting_timeslot', 'modified'),
('meeting_timeslot', 'time'),
('message_message', 'sent'),
('message_message', 'time'),
('message_sendqueue', 'send_at'),
('message_sendqueue', 'sent_at'),
('message_sendqueue', 'time'),
('nomcom_feedback', 'time'),
('nomcom_feedbacklastseen', 'time'),
('nomcom_nomination', 'time'),
('nomcom_nomineeposition', 'time'),
('nomcom_topicfeedbacklastseen', 'time'),
('oidc_provider_code', 'expires_at'),
('oidc_provider_token', 'expires_at'),
('oidc_provider_userconsent', 'date_given'),
('oidc_provider_userconsent', 'expires_at'),
('person_email', 'time'),
('person_historicalemail', 'history_date'),
('person_historicalemail', 'time'),
('person_historicalperson', 'history_date'),
('person_historicalperson', 'time'),
('person_person', 'time'),
('person_personalapikey', 'created'),
('person_personalapikey', 'latest'),
('person_personevent', 'time'),
('request_profiler_profilingrecord', 'end_ts'),
('request_profiler_profilingrecord', 'start_ts'),
('review_historicalreviewassignment', 'assigned_on'),
('review_historicalreviewassignment', 'completed_on'),
('review_historicalreviewassignment', 'history_date'),
('review_historicalreviewersettings', 'history_date'),
('review_historicalreviewrequest', 'history_date'),
('review_historicalreviewrequest', 'time'),
('review_historicalunavailableperiod', 'history_date'),
('review_reviewassignment', 'assigned_on'),
('review_reviewassignment', 'completed_on'),
('review_reviewrequest', 'time'),
('review_reviewwish', 'time'),
('south_migrationhistory', 'applied'),
('submit_preapproval', 'time'),
('submit_submissioncheck', 'time'),
('submit_submissionevent', 'time'),
('tastypie_apikey', 'created'),
('utils_dumpinfo', 'date'),
('utils_versioninfo', 'time'),
)
def forward(apps, schema_editor):
# Check that we can safely ignore celery beat columns - it defaults to UTC if CELERY_TIMEZONE is not set.
celery_timezone = getattr(settings, 'CELERY_TIMEZONE', None)
assert celery_timezone in ('UTC', None), 'update migration, celery is not using UTC'
# If the CELERY_ENABLE_UTC flag is set, abort because someone is using a strange configuration.
assert not hasattr(settings, 'CELERY_ENABLE_UTC'), 'update migration, settings.CELERY_ENABLE_UTC was not expected'
with connection.cursor() as cursor:
# Check that we have timezones.
# If these assertions fail, the DB does not know all the necessary time zones.
# To load timezones,
# $ mysql_tzinfo_to_sql /usr/share/zoneinfo | mysql -u root mysql
# (on a dev system, first connect to the db image with `docker compose exec db bash`)
cursor.execute("SELECT CONVERT_TZ('2022-06-22T17:43:00', 'PST8PDT', 'UTC');")
assert not any(None in row for row in cursor.fetchall()), 'database does not recognize PST8PDT'
cursor.execute(
"SELECT CONVERT_TZ('2022-06-22T17:43:00', time_zone, 'UTC') FROM meeting_meeting WHERE time_zone != '';"
)
assert not any(None in row for row in cursor.fetchall()), 'database does not recognize a meeting time zone'
# Check that we have all and only the expected datetime columns to work with.
# If this fails, figure out what changed and decide how to proceed safely.
cursor.execute("""
SELECT table_name, column_name
FROM information_schema.columns
WHERE table_schema='ietf_utf8'
AND column_type LIKE 'datetime%'
AND NOT table_name LIKE 'django_celery_beat_%'
ORDER BY table_name, column_name;
""")
assert cursor.fetchall() == expected_datetime_columns, 'unexpected or missing datetime columns in db'
class Migration(migrations.Migration):
dependencies = [
('utils', '0001_initial'),
('meeting', '0058_meeting_time_zone_not_blank'),
]
# To generate the queries:
#
# pst8pdt_columns = [e for e in expected_datetime_columns if e != ('meeting_timeslot', 'time')]
# queries = []
# for table, column in pst8pdt_columns:
# queries.append(f"UPDATE {table} SET {column} = CONVERT_TZ({column}, 'PST8PDT', 'UTC');")
#
# queries.append("""
# UPDATE meeting_timeslot
# JOIN meeting_meeting
# ON meeting_meeting.id = meeting_id
# SET time = CONVERT_TZ(time, time_zone, 'UTC');
# """)
#
# print("\n".join(queries))
#
operations = [
migrations.RunPython(forward),
migrations.RunSQL("""
UPDATE auth_user SET date_joined = CONVERT_TZ(date_joined, 'PST8PDT', 'UTC');
UPDATE auth_user SET last_login = CONVERT_TZ(last_login, 'PST8PDT', 'UTC');
UPDATE community_documentchangedates SET new_version_date = CONVERT_TZ(new_version_date, 'PST8PDT', 'UTC');
UPDATE community_documentchangedates SET normal_change_date = CONVERT_TZ(normal_change_date, 'PST8PDT', 'UTC');
UPDATE community_documentchangedates SET significant_change_date = CONVERT_TZ(significant_change_date, 'PST8PDT', 'UTC');
UPDATE django_admin_log SET action_time = CONVERT_TZ(action_time, 'PST8PDT', 'UTC');
UPDATE django_migrations SET applied = CONVERT_TZ(applied, 'PST8PDT', 'UTC');
UPDATE django_session SET expire_date = CONVERT_TZ(expire_date, 'PST8PDT', 'UTC');
UPDATE doc_ballotpositiondocevent SET comment_time = CONVERT_TZ(comment_time, 'PST8PDT', 'UTC');
UPDATE doc_ballotpositiondocevent SET discuss_time = CONVERT_TZ(discuss_time, 'PST8PDT', 'UTC');
UPDATE doc_deletedevent SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE doc_docevent SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE doc_dochistory SET expires = CONVERT_TZ(expires, 'PST8PDT', 'UTC');
UPDATE doc_dochistory SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE doc_docreminder SET due = CONVERT_TZ(due, 'PST8PDT', 'UTC');
UPDATE doc_document SET expires = CONVERT_TZ(expires, 'PST8PDT', 'UTC');
UPDATE doc_document SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE doc_documentactionholder SET time_added = CONVERT_TZ(time_added, 'PST8PDT', 'UTC');
UPDATE doc_initialreviewdocevent SET expires = CONVERT_TZ(expires, 'PST8PDT', 'UTC');
UPDATE doc_irsgballotdocevent SET duedate = CONVERT_TZ(duedate, 'PST8PDT', 'UTC');
UPDATE doc_lastcalldocevent SET expires = CONVERT_TZ(expires, 'PST8PDT', 'UTC');
UPDATE group_group SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE group_groupevent SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE group_grouphistory SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE group_groupmilestone SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE group_groupmilestonehistory SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE ipr_iprdisclosurebase SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE ipr_iprevent SET response_due = CONVERT_TZ(response_due, 'PST8PDT', 'UTC');
UPDATE ipr_iprevent SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE liaisons_liaisonstatementevent SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE mailinglists_subscribed SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE mailinglists_whitelisted SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE meeting_floorplan SET modified = CONVERT_TZ(modified, 'PST8PDT', 'UTC');
UPDATE meeting_room SET modified = CONVERT_TZ(modified, 'PST8PDT', 'UTC');
UPDATE meeting_schedtimesessassignment SET modified = CONVERT_TZ(modified, 'PST8PDT', 'UTC');
UPDATE meeting_schedulingevent SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE meeting_session SET modified = CONVERT_TZ(modified, 'PST8PDT', 'UTC');
UPDATE meeting_session SET scheduled = CONVERT_TZ(scheduled, 'PST8PDT', 'UTC');
UPDATE meeting_slidesubmission SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE meeting_timeslot SET modified = CONVERT_TZ(modified, 'PST8PDT', 'UTC');
UPDATE message_message SET sent = CONVERT_TZ(sent, 'PST8PDT', 'UTC');
UPDATE message_message SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE message_sendqueue SET send_at = CONVERT_TZ(send_at, 'PST8PDT', 'UTC');
UPDATE message_sendqueue SET sent_at = CONVERT_TZ(sent_at, 'PST8PDT', 'UTC');
UPDATE message_sendqueue SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE nomcom_feedback SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE nomcom_feedbacklastseen SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE nomcom_nomination SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE nomcom_nomineeposition SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE nomcom_topicfeedbacklastseen SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE oidc_provider_code SET expires_at = CONVERT_TZ(expires_at, 'PST8PDT', 'UTC');
UPDATE oidc_provider_token SET expires_at = CONVERT_TZ(expires_at, 'PST8PDT', 'UTC');
UPDATE oidc_provider_userconsent SET date_given = CONVERT_TZ(date_given, 'PST8PDT', 'UTC');
UPDATE oidc_provider_userconsent SET expires_at = CONVERT_TZ(expires_at, 'PST8PDT', 'UTC');
UPDATE person_email SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE person_historicalemail SET history_date = CONVERT_TZ(history_date, 'PST8PDT', 'UTC');
UPDATE person_historicalemail SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE person_historicalperson SET history_date = CONVERT_TZ(history_date, 'PST8PDT', 'UTC');
UPDATE person_historicalperson SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE person_person SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE person_personalapikey SET created = CONVERT_TZ(created, 'PST8PDT', 'UTC');
UPDATE person_personalapikey SET latest = CONVERT_TZ(latest, 'PST8PDT', 'UTC');
UPDATE person_personevent SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE request_profiler_profilingrecord SET end_ts = CONVERT_TZ(end_ts, 'PST8PDT', 'UTC');
UPDATE request_profiler_profilingrecord SET start_ts = CONVERT_TZ(start_ts, 'PST8PDT', 'UTC');
UPDATE review_historicalreviewassignment SET assigned_on = CONVERT_TZ(assigned_on, 'PST8PDT', 'UTC');
UPDATE review_historicalreviewassignment SET completed_on = CONVERT_TZ(completed_on, 'PST8PDT', 'UTC');
UPDATE review_historicalreviewassignment SET history_date = CONVERT_TZ(history_date, 'PST8PDT', 'UTC');
UPDATE review_historicalreviewersettings SET history_date = CONVERT_TZ(history_date, 'PST8PDT', 'UTC');
UPDATE review_historicalreviewrequest SET history_date = CONVERT_TZ(history_date, 'PST8PDT', 'UTC');
UPDATE review_historicalreviewrequest SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE review_historicalunavailableperiod SET history_date = CONVERT_TZ(history_date, 'PST8PDT', 'UTC');
UPDATE review_reviewassignment SET assigned_on = CONVERT_TZ(assigned_on, 'PST8PDT', 'UTC');
UPDATE review_reviewassignment SET completed_on = CONVERT_TZ(completed_on, 'PST8PDT', 'UTC');
UPDATE review_reviewrequest SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE review_reviewwish SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE south_migrationhistory SET applied = CONVERT_TZ(applied, 'PST8PDT', 'UTC');
UPDATE submit_preapproval SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE submit_submissioncheck SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE submit_submissionevent SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE tastypie_apikey SET created = CONVERT_TZ(created, 'PST8PDT', 'UTC');
UPDATE utils_dumpinfo SET date = CONVERT_TZ(date, 'PST8PDT', 'UTC');
UPDATE utils_versioninfo SET time = CONVERT_TZ(time, 'PST8PDT', 'UTC');
UPDATE meeting_timeslot
JOIN meeting_meeting
ON meeting_meeting.id = meeting_id
SET time = CONVERT_TZ(time, time_zone, 'UTC');
"""),
]