diff --git a/ietf/bin/rfc-editor-index-updates b/ietf/bin/rfc-editor-index-updates index 0519eae8d..3595ac3c4 100755 --- a/ietf/bin/rfc-editor-index-updates +++ b/ietf/bin/rfc-editor-index-updates @@ -2,6 +2,7 @@ import os, sys, re, json, datetime import syslog +import traceback syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER) @@ -13,6 +14,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings") from django.conf import settings from optparse import OptionParser +from django.core.mail import mail_admins parser = OptionParser() parser.add_option("-d", dest="skip_date", @@ -24,7 +26,9 @@ skip_date = datetime.date.today() - datetime.timedelta(days=365) if options.skip_date: skip_date = datetime.datetime.strptime(options.skip_date, "%Y-%m-%d").date() +from ietf.utils.pipe import pipe from ietf.sync.rfceditor import * +from ietf.doc.utils import rebuild_reference_relations syslog.syslog("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_QUEUE_URL) @@ -35,6 +39,28 @@ if len(data) < MIN_INDEX_RESULTS: syslog.syslog("Not enough results, only %s" % len(data)) sys.exit(1) -changed = update_docs_from_rfc_index(data, skip_older_than_date=skip_date) +changed, new_rfcs = update_docs_from_rfc_index(data, skip_older_than_date=skip_date) + for c in changed: syslog.syslog(c) + +# This can be called while processing a notifying POST from the RFC Editor +# Spawn a child to sync the rfcs and calculate new reference relationships +# so that the POST + +newpid = os.fork() + +if newpid == 0: + try: + pipe("%s -a %s %s" % (settings.RSYNC_BINARY,settings.RFC_TEXT_RSYNC_SOURCE,settings.RFC_PATH)) + for rfc in new_rfcs: + rebuild_reference_relations(rfc) + syslog.syslog("Updated references for %s"%rfc.canonical_name()) + except: + subject = "Exception in updating references for new rfcs: %s : %s" % (sys.exc_info()[0],sys.exc_info()[1]) + msg = "%s\n%s\n----\n%s"%(sys.exc_info()[0],sys.exc_info()[1],traceback.format_tb(sys.exc_info()[2])) + mail_admins(subject,msg,fail_silently=True) + syslog.syslog(subject) + os._exit(0) +else: + sys.exit(0) diff --git a/ietf/settings.py b/ietf/settings.py index 47d3ed463..7d17ddeeb 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -319,6 +319,8 @@ IANA_SYNC_PASSWORD = "secret" IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/" +RFC_TEXT_RSYNC_SOURCE="ftp.rfc-editor.org::rfcs-text-only" + RFC_EDITOR_SYNC_PASSWORD="secret" RFC_EDITOR_SYNC_NOTIFICATION_URL = "http://www.rfc-editor.org/parser/parser.php" RFC_EDITOR_QUEUE_URL = "http://www.rfc-editor.org/queue2.xml" @@ -375,6 +377,7 @@ IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB DOT_BINARY = '/usr/bin/dot' UNFLATTEN_BINARY= '/usr/bin/unflatten' PS2PDF_BINARY = '/usr/bin/ps2pdf' +RSYNC_BINARY = '/usr/bin/rsync' # Account settings DAYS_TO_EXPIRE_REGISTRATION_LINK = 3 diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index d241655d0..0c4535f8c 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -317,6 +317,7 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None): system = Person.objects.get(name="(System)") results = [] + new_rfcs = [] for rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract in data: @@ -399,6 +400,7 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None): other_changes = True results.append("Added RFC published event: %s" % e.time.strftime("%Y-%m-%d")) + new_rfcs.append(doc) for t in ("draft-iesg", "draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"): slug = doc.get_state_slug(t) @@ -461,7 +463,7 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None): doc.time = datetime.datetime.now() doc.save() - return results + return results, new_rfcs def post_approved_draft(url, name): diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index ee6b1b70d..7fce09716 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -306,7 +306,7 @@ class RFCSyncTests(TestCase): draft_filename = "%s-%s.txt" % (doc.name, doc.rev) self.write_draft_file(draft_filename, 5000) - changed = rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30)) + changed,_ = rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30)) doc = Document.objects.get(name=doc.name) @@ -329,7 +329,7 @@ class RFCSyncTests(TestCase): self.assertTrue(os.path.exists(os.path.join(self.archive_dir, draft_filename))) # make sure we can apply it again with no changes - changed = rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30)) + changed,_ = rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30)) self.assertEqual(len(changed), 0)