b4cf04a09d
* refactor: Change import style for clarity * feat: Add iana_changes_updates_task() * chore: Squelch lint warning My linter does not like variables defined outside of __init__() * feat: Add PeriodicTask for iana_changes_updates_task * refactor: tasks instead of scripts on sync.views.notify() * test: Test iana_changes_updates_task * refactor: rename task for consistency * feat: Add iana_protocols_update_task * feat: Add PeriodicTask for iana protocols sync * refactor: Use protocol sync task instead of script in view * refactor: itertools.batched() not available until py312 * test: test iana_protocols_update_task * feat: Add idindex_update_task() Calls idindex generation functions and does the file update dance to put them in place. * chore: Add comments to bin/hourly * fix: annotate types and fix bug * feat: Create PeriodicTask for idindex_update_task * refactor: Move helpers into a class More testable this way * refactor: Make TempFileManager a context mgr * test: Test idindex_update_task * test: Test TempFileManager * fix: Fix bug in TestFileManager yay testing * feat: Add expire_ids_task() * feat: Create PeriodicTask for expire_ids_task * test: Test expire_ids_task * test: Test request timeout in iana_protocols_update_task * refactor: do not re-raise timeout exception Not sure this is the right thing to do, but it's the same as rfc_editor_index_update_task * feat: Add notify_expirations_task * feat: Add "weekly" celery beat crontab * refactor: Reorder crontab fields This matches the crontab file field order * feat: Add PeriodicTask for notify_expirations * test: Test notify_expirations_task * test: Add annotation to satisfy mypy
86 lines
3.4 KiB
Python
86 lines
3.4 KiB
Python
# Copyright The IETF Trust 2024, All Rights Reserved
|
|
#
|
|
# Celery task definitions
|
|
#
|
|
import shutil
|
|
|
|
import debug # pyflakes:ignore
|
|
|
|
from celery import shared_task
|
|
from contextlib import AbstractContextManager
|
|
from pathlib import Path
|
|
from tempfile import NamedTemporaryFile
|
|
|
|
from .index import all_id_txt, all_id2_txt, id_index_txt
|
|
|
|
|
|
class TempFileManager(AbstractContextManager):
|
|
def __init__(self, tmpdir=None) -> None:
|
|
self.cleanup_list: set[Path] = set()
|
|
self.dir = tmpdir
|
|
|
|
def make_temp_file(self, content):
|
|
with NamedTemporaryFile(mode="wt", delete=False, dir=self.dir) as tf:
|
|
tf_path = Path(tf.name)
|
|
self.cleanup_list.add(tf_path)
|
|
tf.write(content)
|
|
return tf_path
|
|
|
|
def move_into_place(self, src_path: Path, dest_path: Path):
|
|
shutil.move(src_path, dest_path)
|
|
dest_path.chmod(0o644)
|
|
self.cleanup_list.remove(src_path)
|
|
|
|
def cleanup(self):
|
|
for tf_path in self.cleanup_list:
|
|
tf_path.unlink(missing_ok=True)
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
self.cleanup()
|
|
return False # False: do not suppress the exception
|
|
|
|
|
|
@shared_task
|
|
def idindex_update_task():
|
|
"""Update I-D indexes"""
|
|
id_path = Path("/a/ietfdata/doc/draft/repository")
|
|
derived_path = Path("/a/ietfdata/derived")
|
|
download_path = Path("/a/www/www6s/download")
|
|
|
|
with TempFileManager("/a/tmp") as tmp_mgr:
|
|
# Generate copies of new contents
|
|
all_id_content = all_id_txt()
|
|
all_id_tmpfile = tmp_mgr.make_temp_file(all_id_content)
|
|
derived_all_id_tmpfile = tmp_mgr.make_temp_file(all_id_content)
|
|
download_all_id_tmpfile = tmp_mgr.make_temp_file(all_id_content)
|
|
|
|
id_index_content = id_index_txt()
|
|
id_index_tmpfile = tmp_mgr.make_temp_file(id_index_content)
|
|
derived_id_index_tmpfile = tmp_mgr.make_temp_file(id_index_content)
|
|
download_id_index_tmpfile = tmp_mgr.make_temp_file(id_index_content)
|
|
|
|
id_abstracts_content = id_index_txt(with_abstracts=True)
|
|
id_abstracts_tmpfile = tmp_mgr.make_temp_file(id_abstracts_content)
|
|
derived_id_abstracts_tmpfile = tmp_mgr.make_temp_file(id_abstracts_content)
|
|
download_id_abstracts_tmpfile = tmp_mgr.make_temp_file(id_abstracts_content)
|
|
|
|
all_id2_content = all_id2_txt()
|
|
all_id2_tmpfile = tmp_mgr.make_temp_file(all_id2_content)
|
|
derived_all_id2_tmpfile = tmp_mgr.make_temp_file(all_id2_content)
|
|
|
|
# Move temp files as-atomically-as-possible into place
|
|
tmp_mgr.move_into_place(all_id_tmpfile, id_path / "all_id.txt")
|
|
tmp_mgr.move_into_place(derived_all_id_tmpfile, derived_path / "all_id.txt")
|
|
tmp_mgr.move_into_place(download_all_id_tmpfile, download_path / "id-all.txt")
|
|
|
|
tmp_mgr.move_into_place(id_index_tmpfile, id_path / "1id-index.txt")
|
|
tmp_mgr.move_into_place(derived_id_index_tmpfile, derived_path / "1id-index.txt")
|
|
tmp_mgr.move_into_place(download_id_index_tmpfile, download_path / "id-index.txt")
|
|
|
|
tmp_mgr.move_into_place(id_abstracts_tmpfile, id_path / "1id-abstracts.txt")
|
|
tmp_mgr.move_into_place(derived_id_abstracts_tmpfile, derived_path / "1id-abstracts.txt")
|
|
tmp_mgr.move_into_place(download_id_abstracts_tmpfile, download_path / "id-abstract.txt")
|
|
|
|
tmp_mgr.move_into_place(all_id2_tmpfile, id_path / "all_id2.txt")
|
|
tmp_mgr.move_into_place(derived_all_id2_tmpfile, derived_path / "all_id2.txt")
|