Refactor expirable_drafts() and its usage to speed up doc.views_search.recent_drafts view (and discourage inefficient use of the method). Commit ready for merge.

- Legacy-Id: 19490

ietf/bin/expire-ids

@@ -27,15 +27,22 @@ django.setup()
 from ietf.utils.log import logger
 
 try:
-    from ietf.doc.expire import ( in_draft_expire_freeze, get_expired_drafts, expirable_draft,
+    from ietf.doc.expire import ( in_draft_expire_freeze, get_expired_drafts, expirable_drafts,
         send_expire_notice_for_draft, expire_draft, clean_up_draft_files )
+    from ietf.doc.models import Document
 
     if not in_draft_expire_freeze():
         syslog.syslog("Expiring drafts ...")
         for doc in get_expired_drafts():
             # verify expirability -- it might have changed after get_expired_drafts() was run
             # (this whole loop took about 2 minutes on 04 Jan 2018)
-            if expirable_draft(doc) and doc.expires < datetime.datetime.today() + datetime.timedelta(1):
+            # N.B., re-running expirable_drafts() repeatedly is fairly expensive. Where possible,
+            # it's much faster to run it once on a superset query of the objects you are going
+            # to test and keep its results. That's not desirable here because it would defeat
+            # the purpose of double-checking that a document is still expirable when it is actually
+            # being marked as expired.
+            if (expirable_drafts(Document.objects.filter(pk=doc.pk)).exists()
+                    and doc.expires < datetime.datetime.today() + datetime.timedelta(1)):
                 send_expire_notice_for_draft(doc)
                 expire_draft(doc)
                 syslog.syslog("  Expired draft %s-%s" % (doc.name, doc.rev))

ietf/doc/expire.py

@@ -8,7 +8,7 @@ from django.conf import settings
 import datetime, os, shutil, glob, re
 from pathlib import Path
 
-from typing import List, Tuple      # pyflakes:ignore
+from typing import List, Optional      # pyflakes:ignore
 
 from ietf.utils import log
 from ietf.utils.mail import send_mail
@@ -19,15 +19,7 @@ from ietf.doc.utils import add_state_change_event, update_action_holders
 from ietf.mailtrigger.utils import gather_address_lists
 
 
-def expirable_draft(draft):
-    """Return whether draft is in an expirable state or not. This is
-    the single draft version of the logic in expirable_drafts. These
-    two functions need to be kept in sync."""
-    if draft.type_id != 'draft':
-        return False
-    return bool(expirable_drafts(Document.objects.filter(pk=draft.pk)))
-
-nonexpirable_states = []                # type: List[State]
+nonexpirable_states: Optional[List[State]] = None
 
 def expirable_drafts(queryset=None):
     """Return a queryset with expirable drafts."""
@@ -39,31 +31,25 @@ def expirable_drafts(queryset=None):
         queryset = Document.objects.all()
 
     # Populate this first time through (but after django has been set up)
-    if nonexpirable_states == []:
+    if nonexpirable_states is None:
         # all IESG states except I-D Exists, AD Watching, and Dead block expiry
-        nonexpirable_states += list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists","watching", "dead")))
+        nonexpirable_states = list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists","watching", "dead")))
         # sent to RFC Editor and RFC Published block expiry (the latter
         # shouldn't be possible for an active draft, though)
         nonexpirable_states += list(State.objects.filter(used=True, type__in=("draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"), slug__in=("rfc-edit", "pub")))
         # other IRTF states that block expiration
         nonexpirable_states += list(State.objects.filter(used=True, type_id="draft-stream-irtf", slug__in=("irsgpoll", "iesg-rev",)))
 
-    d = queryset.filter(states__type="draft", states__slug="active")
-    if not d.exists():
-        return d
+    return queryset.filter(
+        states__type="draft", states__slug="active"
+    ).exclude(
+        expires=None
+    ).exclude(
+        states__in=nonexpirable_states
+    ).exclude(
+        tags="rfc-rev"  # under review by the RFC Editor blocks expiry
+    ).distinct()
 
-    d = d.exclude(expires=None)
-    if not d.exists():
-        return d
-
-    d = d.exclude(states__in=nonexpirable_states)
-    if not d.exists():
-        return d
-
-    # under review by the RFC Editor blocks expiry
-    d = d.exclude(tags="rfc-rev")
-
-    return d.distinct()
 
 def get_soon_to_expire_drafts(days_of_warning):
     start_date = datetime.date.today() - datetime.timedelta(1)

ietf/doc/utils_search.py

@@ -6,7 +6,7 @@ import datetime
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import Document, DocAlias, RelatedDocument, DocEvent, TelechatDocEvent, BallotDocEvent
-from ietf.doc.expire import expirable_draft
+from ietf.doc.expire import expirable_drafts
 from ietf.doc.utils import augment_docs_and_user_with_user_info
 from ietf.meeting.models import SessionPresentation, Meeting, Session
 
@@ -80,6 +80,7 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False):
     fill_in_document_sessions(docs, doc_dict, doc_ids)
 
     # misc
+    expirable_pks = expirable_drafts(Document.objects.filter(pk__in=doc_ids)).values_list('pk', flat=True)
     for d in docs:
         # emulate canonical name which is used by a lot of the utils
         d.canonical_name = wrap_value(rfc_aliases[d.pk] if d.pk in rfc_aliases else d.name)
@@ -102,7 +103,7 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False):
             else:
                 d.search_heading = "%s Internet-Draft" % d.get_state()
                 if state_slug == "active":
-                    d.expirable = expirable_draft(d)
+                    d.expirable = d.pk in expirable_pks
                 else:
                     d.expirable = False
         else: