refactor: expire last calls via celery (#7417)

* feat: expire_last_calls_task

* feat: create periodic task for last calls

* test: test new task

* chore: remove expire-last-calls script

bin/daily

@@ -40,9 +40,5 @@ $DTDIR/ietf/manage.py populate_yang_model_dirs -v0
 # Re-run yang checks on active documents
 $DTDIR/ietf/manage.py run_yang_model_checks -v0
 
-# Expire last calls
-# Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner:
-$DTDIR/ietf/bin/expire-last-calls
-
 # Purge older PersonApiKeyEvents
 $DTDIR/ietf/manage.py purge_old_personal_api_key_events 14

ietf/bin/expire-last-calls

@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-
-# This script requires that the proper virtual python environment has been
-# invoked before start
-
-import os
-import sys
-import syslog
-
-# boilerplate
-basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
-sys.path = [ basedir ] + sys.path
-os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
-
-virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
-if os.path.exists(virtualenv_activation):
-    execfile(virtualenv_activation, dict(__file__=virtualenv_activation))
-
-syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER)
-
-import django
-django.setup()
-
-# ----------------------------------------------------------------------
-
-from ietf.doc.lastcall import get_expired_last_calls, expire_last_call
-
-drafts = get_expired_last_calls()
-for doc in drafts:
-    try:
-        expire_last_call(doc)
-        syslog.syslog("Expired last call for %s (id=%s)" % (doc.file_tag(), doc.pk))
-    except Exception as e:
-        syslog.syslog(syslog.LOG_ERR, "ERROR: Failed to expire last call for %s (id=%s)" % (doc.file_tag(), doc.pk))

ietf/doc/tasks.py

@@ -23,6 +23,7 @@ from .expire import (
     get_soon_to_expire_drafts,
     send_expire_warning_for_draft,
 )
+from .lastcall import get_expired_last_calls, expire_last_call
 from .models import Document
 from .utils import generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted
 
@@ -61,6 +62,17 @@ def notify_expirations_task(notify_days=14):
 
 
 @shared_task
+def expire_last_calls_task():
+    for doc in get_expired_last_calls():
+        try:
+            expire_last_call(doc)
+        except Exception:
+            log.log(f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})")
+        else:
+            log.log(f"Expired last call for {doc.file_tag()} (id={doc.pk})")
+
+
+@shared_task            
 def generate_idnits2_rfc_status_task():
     outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status"
     blob = generate_idnits2_rfc_status()

ietf/doc/tests_tasks.py

@@ -12,6 +12,7 @@ from .factories import DocumentFactory
 from .models import Document
 from .tasks import (
     expire_ids_task,
+    expire_last_calls_task,
     generate_idnits2_rfcs_obsoleted_task,
     generate_idnits2_rfc_status_task,
     notify_expirations_task,
@@ -71,6 +72,29 @@ class TaskTests(TestCase):
         self.assertEqual(send_warning_mock.call_count, 1)
         self.assertEqual(send_warning_mock.call_args[0], ("sentinel",))
 
+    @mock.patch("ietf.doc.tasks.expire_last_call")
+    @mock.patch("ietf.doc.tasks.get_expired_last_calls")
+    def test_expire_last_calls_task(self, mock_get_expired, mock_expire):
+        docs = DocumentFactory.create_batch(3)
+        mock_get_expired.return_value = docs
+        expire_last_calls_task()
+        self.assertTrue(mock_get_expired.called)
+        self.assertEqual(mock_expire.call_count, 3)
+        self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0]))
+        self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1]))
+        self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2]))
+    
+        # Check that it runs even if exceptions occur
+        mock_get_expired.reset_mock()
+        mock_expire.reset_mock()
+        mock_expire.side_effect = ValueError
+        expire_last_calls_task()
+        self.assertTrue(mock_get_expired.called)
+        self.assertEqual(mock_expire.call_count, 3)
+        self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0]))
+        self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1]))
+        self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2]))
+
     @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status")
     def test_generate_idnits2_rfc_status_task(self, mock_generate):
         mock_generate.return_value = "dåtå"
@@ -90,4 +114,3 @@ class TaskTests(TestCase):
             "dåtå".encode("utf8"),
             (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(),
         )
-    

ietf/utils/management/commands/periodic_tasks.py

@@ -141,6 +141,16 @@ class Command(BaseCommand):
             ),
         )
 
+        PeriodicTask.objects.get_or_create(
+            name="Expire Last Calls",
+            task="ietf.doc.tasks.expire_last_calls_task",
+            defaults=dict(
+                enabled=False,
+                crontab=self.crontabs["daily"],
+                description="Move docs whose last call has expired to their next states",
+            ),
+        )
+
         PeriodicTask.objects.get_or_create(
             name="Sync with IANA changes",
             task="ietf.sync.tasks.iana_changes_update_task",