b4cf04a09d
* refactor: Change import style for clarity * feat: Add iana_changes_updates_task() * chore: Squelch lint warning My linter does not like variables defined outside of __init__() * feat: Add PeriodicTask for iana_changes_updates_task * refactor: tasks instead of scripts on sync.views.notify() * test: Test iana_changes_updates_task * refactor: rename task for consistency * feat: Add iana_protocols_update_task * feat: Add PeriodicTask for iana protocols sync * refactor: Use protocol sync task instead of script in view * refactor: itertools.batched() not available until py312 * test: test iana_protocols_update_task * feat: Add idindex_update_task() Calls idindex generation functions and does the file update dance to put them in place. * chore: Add comments to bin/hourly * fix: annotate types and fix bug * feat: Create PeriodicTask for idindex_update_task * refactor: Move helpers into a class More testable this way * refactor: Make TempFileManager a context mgr * test: Test idindex_update_task * test: Test TempFileManager * fix: Fix bug in TestFileManager yay testing * feat: Add expire_ids_task() * feat: Create PeriodicTask for expire_ids_task * test: Test expire_ids_task * test: Test request timeout in iana_protocols_update_task * refactor: do not re-raise timeout exception Not sure this is the right thing to do, but it's the same as rfc_editor_index_update_task * feat: Add notify_expirations_task * feat: Add "weekly" celery beat crontab * refactor: Reorder crontab fields This matches the crontab file field order * feat: Add PeriodicTask for notify_expirations * test: Test notify_expirations_task * test: Add annotation to satisfy mypy
105 lines
3.3 KiB
Bash
Executable file
105 lines
3.3 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
# Hourly datatracker jobs
|
|
#
|
|
# This script is expected to be triggered by cron from
|
|
# /etc/cron.d/datatracker
|
|
export LANG=en_US.UTF-8
|
|
export PYTHONIOENCODING=utf-8
|
|
|
|
# Make sure we stop if something goes wrong:
|
|
program=${0##*/}
|
|
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
|
|
|
|
DTDIR=/a/www/ietf-datatracker/web
|
|
cd $DTDIR/
|
|
|
|
# Set up the virtual environment
|
|
source $DTDIR/env/bin/activate
|
|
|
|
logger -p user.info -t cron "Running $DTDIR/bin/hourly"
|
|
|
|
# *** Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner: ***
|
|
|
|
# # Update community lists. Remove once the community rewrite (will be around 6.20.0 )
|
|
# $DTDIR/ietf/manage.py update_community_lists
|
|
#
|
|
# # Polling backup for iana and rfc-editory post APIs
|
|
$DTDIR/ietf/bin/iana-changes-updates
|
|
$DTDIR/ietf/bin/iana-protocols-updates
|
|
# $DTDIR/ietf/bin/rfc-editor-index-updates
|
|
# $DTDIR/ietf/bin/rfc-editor-queue-updates
|
|
#
|
|
# # Generate alias and virtual files for draft email aliases
|
|
# $DTDIR/ietf/bin/generate-draft-aliases && \
|
|
# ( cd /a/postfix; /usr/sbin/postalias -o draft-aliases; ) && \
|
|
# ( cd /a/postfix; /usr/sbin/postmap -o draft-virtual; )
|
|
#
|
|
# # Generate alias and virtual files for group email aliases
|
|
# $DTDIR/ietf/bin/generate-wg-aliases && \
|
|
# ( cd /a/postfix; /usr/sbin/postalias -o group-aliases; ) && \
|
|
# ( cd /a/postfix; /usr/sbin/postmap -o group-virtual; )
|
|
#
|
|
# Generate some static files
|
|
ID=/a/ietfdata/doc/draft/repository
|
|
DERIVED=/a/ietfdata/derived
|
|
DOWNLOAD=/a/www/www6s/download
|
|
|
|
## Start of script refactored into idindex_update_task() ===
|
|
export TMPDIR=/a/tmp
|
|
|
|
TMPFILE1=`mktemp` || exit 1
|
|
TMPFILE2=`mktemp` || exit 1
|
|
TMPFILE3=`mktemp` || exit 1
|
|
TMPFILE4=`mktemp` || exit 1
|
|
TMPFILE5=`mktemp` || exit 1
|
|
TMPFILE6=`mktemp` || exit 1
|
|
TMPFILE7=`mktemp` || exit 1
|
|
TMPFILE8=`mktemp` || exit 1
|
|
TMPFILE9=`mktemp` || exit 1
|
|
TMPFILEA=`mktemp` || exit 1
|
|
TMPFILEB=`mktemp` || exit 1
|
|
|
|
chmod a+r $TMPFILE1 $TMPFILE2 $TMPFILE3 $TMPFILE4 $TMPFILE5 $TMPFILE6 $TMPFILE7 $TMPFILE8 $TMPFILE9 $TMPFILEA $TMPFILEB
|
|
|
|
python -m ietf.idindex.generate_all_id_txt >> $TMPFILE1
|
|
python -m ietf.idindex.generate_id_index_txt >> $TMPFILE2
|
|
python -m ietf.idindex.generate_id_abstracts_txt >> $TMPFILE3
|
|
cp $TMPFILE1 $TMPFILE4
|
|
cp $TMPFILE2 $TMPFILE5
|
|
cp $TMPFILE3 $TMPFILE6
|
|
cp $TMPFILE1 $TMPFILE8
|
|
cp $TMPFILE2 $TMPFILE9
|
|
cp $TMPFILE3 $TMPFILEA
|
|
python -m ietf.idindex.generate_all_id2_txt >> $TMPFILE7
|
|
cp $TMPFILE7 $TMPFILEB
|
|
|
|
mv $TMPFILE1 $ID/all_id.txt
|
|
mv $TMPFILE2 $ID/1id-index.txt
|
|
mv $TMPFILE3 $ID/1id-abstracts.txt
|
|
mv $TMPFILE4 $DOWNLOAD/id-all.txt
|
|
mv $TMPFILE5 $DOWNLOAD/id-index.txt
|
|
mv $TMPFILE6 $DOWNLOAD/id-abstract.txt
|
|
mv $TMPFILE7 $ID/all_id2.txt
|
|
mv $TMPFILE8 $DERIVED/all_id.txt
|
|
mv $TMPFILE9 $DERIVED/1id-index.txt
|
|
mv $TMPFILEA $DERIVED/1id-abstracts.txt
|
|
mv $TMPFILEB $DERIVED/all_id2.txt
|
|
|
|
## End of script refactored into idindex_update_task() ===
|
|
|
|
$DTDIR/ietf/manage.py generate_idnits2_rfc_status
|
|
$DTDIR/ietf/manage.py generate_idnits2_rfcs_obsoleted
|
|
|
|
CHARTER=/a/www/ietf-ftp/charter
|
|
wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt
|
|
wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt
|
|
|
|
# Regenerate the last week of bibxml-ids
|
|
$DTDIR/ietf/manage.py generate_draft_bibxml_files
|
|
|
|
# Create and update group wikis
|
|
#$DTDIR/ietf/manage.py create_group_wikis
|
|
|
|
# exit 0
|