fix: record and interpret RFC pub dates in correct timezone (#4421)

* fix: use PST8PDT for published_rfc event timestamps

* fix: find RFCs by PST8PDT year in RfcFeed

* refactor: add const RPC_TZINFO to represent RFC publication timezone

* chore: remove (rather than fix) unused template tags

* fix: always return RPC_TZINFO-local date from Document.pub_date()

* refactor: convert 'published' flag to a Boolean to reflect its usage

* fix: display doc publication dates in correct time zones

* fix: fix various small issues breaking tests

ietf/doc/feeds.py

@@ -16,6 +16,7 @@ from django.utils.html import strip_tags
 from ietf.doc.models import Document, State, LastCallDocEvent, DocEvent
 from ietf.doc.utils import augment_events_with_revision
 from ietf.doc.templatetags.ietf_filters import format_textarea
+from ietf.utils.timezone import RPC_TZINFO
 
 
 def strip_control_characters(s):
@@ -134,7 +135,14 @@ class RfcFeed(Feed):
     
     def items(self):
         if self.year:
-            rfc_events = DocEvent.objects.filter(type='published_rfc',time__year=self.year).order_by('-time')
+            # Find published RFCs based on their official publication year
+            start_of_year = datetime.datetime(int(self.year), 1, 1, tzinfo=RPC_TZINFO)
+            start_of_next_year = datetime.datetime(int(self.year) + 1, 1, 1, tzinfo=RPC_TZINFO)
+            rfc_events = DocEvent.objects.filter(
+                type='published_rfc',
+                time__gte=start_of_year,
+                time__lt=start_of_next_year,
+            ).order_by('-time')
         else:
             cutoff = timezone.now() - datetime.timedelta(days=8)
             rfc_events = DocEvent.objects.filter(type='published_rfc',time__gte=cutoff).order_by('-time')

ietf/doc/models.py

@@ -36,6 +36,7 @@ from ietf.utils.decorators import memoize
 from ietf.utils.validators import validate_no_control_chars
 from ietf.utils.mail import formataddr
 from ietf.utils.models import ForeignKey
+from ietf.utils.timezone import RPC_TZINFO
 if TYPE_CHECKING:
     # importing other than for type checking causes errors due to cyclic imports
     from ietf.meeting.models import ProceedingsMaterial, Session
@@ -925,13 +926,18 @@ class Document(DocumentInfo):
         return s
 
     def pub_date(self):
-        """This is the rfc publication date (datetime) for RFCs, 
+        """Get the publication date for this document
-        and the new-revision datetime for other documents."""
+
+        This is the rfc publication date for RFCs, and the new-revision date for other documents.
+        """
         if self.get_state_slug() == "rfc":
+            # As of Sept 2022, in ietf.sync.rfceditor.update_docs_from_rfc_index() `published_rfc` events are
+            # created with a timestamp whose date *in the PST8PDT timezone* is the official publication date
+            # assigned by the RFC editor.
             event = self.latest_event(type='published_rfc')
         else:
             event = self.latest_event(type='new_revision')
-        return event.time
+        return event.time.astimezone(RPC_TZINFO).date() if event else None
 
     def is_dochistory(self):
         return False

ietf/doc/tests.py

@@ -58,7 +58,7 @@ from ietf.utils.mail import outbox, empty_outbox
 from ietf.utils.test_utils import login_testing_unauthorized, unicontent, reload_db_objects
 from ietf.utils.test_utils import TestCase
 from ietf.utils.text import normalize_text
-from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO
+from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO, RPC_TZINFO
 
 
 class SearchTests(TestCase):
@@ -1431,7 +1431,7 @@ Man                    Expires September 22, 2015               [Page 3]
 
     def test_draft_group_link(self):
         """Link to group 'about' page should have correct format"""
-        event_datetime = datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo('America/Los_Angeles'))
+        event_datetime = datetime.datetime(2010, 10, 10, tzinfo=RPC_TZINFO)
 
         for group_type_id in ['wg', 'rg', 'ag']:
             group = GroupFactory(type_id=group_type_id)
@@ -1890,13 +1890,13 @@ class DocTestCase(TestCase):
                   #other_aliases = ['rfc6020',],
                   states = [('draft','rfc'),('draft-iesg','pub')],
                   std_level_id = 'ps',
-                  time = datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo('America/Los_Angeles')),
+                  time = datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE)),
               )
         num = rfc.rfc_number()
         DocEventFactory.create(
             doc=rfc,
             type='published_rfc',
-            time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo('America/Los_Angeles')),
+            time=datetime.datetime(2010, 10, 10, tzinfo=RPC_TZINFO),
         )
         #
         url = urlreverse('ietf.doc.views_doc.document_bibtex', kwargs=dict(name=rfc.name))
@@ -1915,13 +1915,13 @@ class DocTestCase(TestCase):
                   stream_id =       'ise',
                   states =          [('draft','rfc'),('draft-iesg','pub')],
                   std_level_id =    'inf',
-                  time =            datetime.datetime(1990, 4, 1, tzinfo=ZoneInfo('America/Los_Angeles')),
+                  time =            datetime.datetime(1990, 4, 1, tzinfo=ZoneInfo(settings.TIME_ZONE)),
               )
         num = april1.rfc_number()
         DocEventFactory.create(
             doc=april1,
             type='published_rfc',
-            time=datetime.datetime(1990, 4, 1, tzinfo=ZoneInfo('America/Los_Angeles')),
+            time=datetime.datetime(1990, 4, 1, tzinfo=RPC_TZINFO),
         )
         #
         url = urlreverse('ietf.doc.views_doc.document_bibtex', kwargs=dict(name=april1.name))
@@ -2057,8 +2057,7 @@ class GenerateDraftAliasesTests(TestCase):
        super().tearDown()
 
    def testManagementCommand(self):
-       tz = ZoneInfo('America/Los_Angeles')
+       a_month_ago = (timezone.now() - datetime.timedelta(30)).astimezone(RPC_TZINFO)
-       a_month_ago = (timezone.now() - datetime.timedelta(30)).astimezone(tz)
        a_month_ago = a_month_ago.replace(hour=0, minute=0, second=0, microsecond=0)
        ad = RoleFactory(name_id='ad', group__type_id='area', group__state_id='active').person
        shepherd = PersonFactory()
@@ -2075,8 +2074,8 @@ class GenerateDraftAliasesTests(TestCase):
        doc2 = WgDraftFactory(name='draft-ietf-mars-test', group__acronym='mars', authors=[author2], ad=ad)
        doc3 = WgRfcFactory.create(name='draft-ietf-mars-finished', group__acronym='mars', authors=[author3], ad=ad, std_level_id='ps', states=[('draft','rfc'),('draft-iesg','pub')], time=a_month_ago)
        DocEventFactory.create(doc=doc3, type='published_rfc', time=a_month_ago)
-       doc4 = WgRfcFactory.create(authors=[author4,author5], ad=ad, std_level_id='ps', states=[('draft','rfc'),('draft-iesg','pub')], time=datetime.datetime(2010,10,10, tzinfo=tz))
+       doc4 = WgRfcFactory.create(authors=[author4,author5], ad=ad, std_level_id='ps', states=[('draft','rfc'),('draft-iesg','pub')], time=datetime.datetime(2010,10,10, tzinfo=ZoneInfo(settings.TIME_ZONE)))
-       DocEventFactory.create(doc=doc4, type='published_rfc', time=datetime.datetime(2010, 10, 10, tzinfo=tz))
+       DocEventFactory.create(doc=doc4, type='published_rfc', time=datetime.datetime(2010, 10, 10, tzinfo=RPC_TZINFO))
        doc5 = IndividualDraftFactory(authors=[author6])
 
        args = [ ]

ietf/doc/utils.py

@@ -968,6 +968,7 @@ def make_rev_history(doc):
                         history[url]['pages'] = d.history_set.filter(rev=e.newrevisiondocevent.rev).first().pages
 
     if doc.type_id == "draft":
+        # e.time.date() agrees with RPC publication date when shown in the RPC_TZINFO time zone
         e = doc.latest_event(type='published_rfc')
     else:
         e = doc.latest_event(type='iesg_approved')

ietf/doc/utils_search.py

@@ -5,6 +5,10 @@ import re
 import datetime
 import debug                            # pyflakes:ignore
 
+from zoneinfo import ZoneInfo
+
+from django.conf import settings
+
 from ietf.doc.models import Document, DocAlias, RelatedDocument, DocEvent, TelechatDocEvent, BallotDocEvent
 from ietf.doc.expire import expirable_drafts
 from ietf.doc.utils import augment_docs_and_user_with_user_info
@@ -204,7 +208,7 @@ def prepare_document_table(request, docs, query=None, max_results=200):
         if sort_key == "title":
             res.append(d.title)
         elif sort_key == "date":
-            res.append(str(d.latest_revision_date))
+            res.append(str(d.latest_revision_date.astimezone(ZoneInfo(settings.TIME_ZONE))))
         elif sort_key == "status":
             if rfc_num != None:
                 res.append(num(rfc_num))

ietf/doc/views_doc.py

@@ -967,7 +967,7 @@ def document_bibtex(request, name, rev=None):
 
     latest_revision = doc.latest_event(NewRevisionDocEvent, type="new_revision")
     replaced_by = [d.name for d in doc.related_that("replaces")]
-    published = doc.latest_event(type="published_rfc")
+    published = doc.latest_event(type="published_rfc") is not None
     rfc = latest_revision.doc if latest_revision and latest_revision.doc.get_state_slug() == "rfc" else None
 
     if rev != None and rev != doc.rev:

ietf/secr/proceedings/templatetags/ams_filters.py

@@ -38,20 +38,6 @@ def display_duration(value):
         x=int(value)
         return "%d Hours %d Minutes %d Seconds"%(x//3600,(x%3600)//60,x%60)
 
-@register.filter
-def get_published_date(doc):
-    '''
-    Returns the published date for a RFC Document
-    '''
-    event = doc.latest_event(type='published_rfc')
-    if event:
-        return event.time
-    event = doc.latest_event(type='new_revision')
-    if event:
-        return event.time
-    else:
-        return None
-        
 @register.filter
 def is_ppt(value):
     '''

ietf/secr/sreq/templatetags/ams_filters.py

@@ -42,20 +42,6 @@ def display_duration(value):
     else:
         return "%d Hours %d Minutes %d Seconds"%(value//3600,(value%3600)//60,value%60)
 
-@register.filter
-def get_published_date(doc):
-    '''
-    Returns the published date for a RFC Document
-    '''
-    event = doc.latest_event(type='published_rfc')
-    if event:
-        return event.time
-    event = doc.latest_event(type='new_revision')
-    if event:
-        return event.time
-    else:
-        return None
-        
 @register.filter
 def is_ppt(value):
     '''

ietf/stats/views.py

@@ -40,7 +40,7 @@ from ietf.stats.models import MeetingRegistration, CountryAlias
 from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries, compute_hirsch_index
 from ietf.ietfauth.utils import has_role
 from ietf.utils.response import permission_denied
-from ietf.utils.timezone import date_today, DEADLINE_TZINFO
+from ietf.utils.timezone import date_today, DEADLINE_TZINFO, RPC_TZINFO
 
 
 def stats_index(request):
@@ -625,8 +625,9 @@ def document_stats(request, stats_type=None):
                 type__in=["published_rfc", "new_revision"],
             ).values_list("doc", "time").order_by("doc")
 
-            for doc, time in docevent_qs.iterator():
+            for doc_id, time in docevent_qs.iterator():
-                doc_years[doc].add(time.year)
+                # RPC_TZINFO is used to match the timezone handling in Document.pub_date()
+                doc_years[doc_id].add(time.astimezone(RPC_TZINFO).year)
 
             person_qs = Person.objects.filter(person_filters)
 

ietf/sync/rfceditor.py

@@ -25,7 +25,7 @@ from ietf.name.models import StdLevelName, StreamName
 from ietf.person.models import Person
 from ietf.utils.log import log
 from ietf.utils.mail import send_mail_text
-from ietf.utils.timezone import datetime_from_date
+from ietf.utils.timezone import datetime_from_date, RPC_TZINFO
 
 #QUEUE_URL = "https://www.rfc-editor.org/queue2.xml"
 #INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
@@ -333,9 +333,12 @@ def parse_index(response):
 
 
 def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=None):
-    """Given parsed data from the RFC Editor index, update the documents
+    """Given parsed data from the RFC Editor index, update the documents in the database
-    in the database. Yields a list of change descriptions for each
+
-    document, if any."""
+    Yields a list of change descriptions for each document, if any.
+
+    The skip_older_than_date is a bare date, not a datetime.
+    """
 
     errata = {}
     for item in errata_data:
@@ -373,7 +376,7 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
 
     for rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract in index_data:
 
-        if skip_older_than_date and datetime_from_date(rfc_published_date) < datetime_from_date(skip_older_than_date):
+        if skip_older_than_date and rfc_published_date < skip_older_than_date:
             # speed up the process by skipping old entries
             continue
 
@@ -444,8 +447,16 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
             # unfortunately, rfc_published_date doesn't include the correct day
             # at the moment because the data only has month/year, so
             # try to deduce it
-            d = datetime_from_date(rfc_published_date)
+            #
-            synthesized = timezone.now()
+            # Note: This is in done PST8PDT to preserve compatibility with events created when
+            # USE_TZ was False. The published_rfc event was created with a timestamp whose
+            # server-local datetime (PST8PDT) matched the publication date from the RFC index.
+            # When switching to USE_TZ=True, the timestamps were migrated so they still
+            # matched the publication date in PST8PDT. When interpreting the event timestamp
+            # as a publication date, you must treat it in the PST8PDT time zone. The
+            # RPC_TZINFO constant in ietf.utils.timezone is defined for this purpose.
+            d = datetime_from_date(rfc_published_date, RPC_TZINFO)
+            synthesized = timezone.now().astimezone(RPC_TZINFO)
             if abs(d - synthesized) > datetime.timedelta(days=60):
                 synthesized = d
             else:

ietf/sync/tests.py

@@ -23,7 +23,7 @@ from ietf.sync import iana, rfceditor
 from ietf.utils.mail import outbox, empty_outbox
 from ietf.utils.test_utils import login_testing_unauthorized
 from ietf.utils.test_utils import TestCase
-from ietf.utils.timezone import date_today
+from ietf.utils.timezone import date_today, RPC_TZINFO
 
 
 class IANASyncTests(TestCase):
@@ -354,7 +354,7 @@ class RFCSyncTests(TestCase):
         self.assertEqual(events[0].type, "sync_from_rfc_editor")
         self.assertEqual(events[1].type, "changed_action_holders")
         self.assertEqual(events[2].type, "published_rfc")
-        self.assertEqual(events[2].time.date(), today)
+        self.assertEqual(events[2].time.astimezone(RPC_TZINFO).date(), today)
         self.assertTrue("errata" in doc.tags.all().values_list("slug", flat=True))
         self.assertTrue(DocAlias.objects.filter(name="rfc1234", docs=doc))
         self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=doc))

ietf/templates/doc/document_draft.html

@@ -13,7 +13,7 @@
           title="Document changes"
           href="/feed/document-changes/{{ name }}/">
     <meta name="description"
-          content="{{ doc.title }} {% if doc.get_state_slug == 'rfc' and not snapshot %}(RFC {{ rfc_number }}{% if published %}, {{ published.time|date:'F Y' }}{% endif %}{% if obsoleted_by %}; obsoleted by {{ obsoleted_by|join:', ' }}{% endif %}){% else %}(Internet-Draft, {{ doc.time|date:'Y' }}){% endif %}">
+          content="{{ doc.title }} {% if doc.get_state_slug == 'rfc' and not snapshot %}(RFC {{ rfc_number }}{% if published %}, {{ doc.pub_date|date:'F Y' }}{% endif %}{% if obsoleted_by %}; obsoleted by {{ obsoleted_by|join:', ' }}{% endif %}){% else %}(Internet-Draft, {{ doc.time|date:'Y' }}){% endif %}">
 {% endblock %}
 {% block morecss %}.inline { display: inline; }{% endblock %}
 {% block title %}
@@ -47,7 +47,7 @@
                     {% if doc.get_state_slug == "rfc" and not snapshot %}
                         <span class="text-success">RFC - {{ doc.std_level }}</span>
                         {% if published %}
-                            ({{ published.time|date:"F Y" }})
+                            ({{ doc.pub_date|date:"F Y" }})
                         {% else %}
                             (Publication date unknown)
                         {% endif %}

ietf/utils/timezone.py

@@ -7,9 +7,17 @@ from django.conf import settings
 from django.utils import timezone
 
 
+# Timezone constants - tempting to make these settings, but changing them will
+# require code changes.
+#
 # Default time zone for deadlines / expiration dates.
 DEADLINE_TZINFO = ZoneInfo('PST8PDT')
 
+# Time zone for dates from the RPC. This value is baked into the timestamps on DocEvents
+# of type="published_rfc" - see Document.pub_date() and ietf.sync.refceditor.update_docs_from_rfc_index()
+# for more information about how that works.
+RPC_TZINFO = ZoneInfo('PST8PDT')
+
 
 def make_aware(dt, tzinfo):
     """Assign timezone to a naive datetime