Merged [7557] from rjsparks@nostrum.com: Update reference relationships (asyncronously) for new rfcs found when syncing with the rfc index. Fixes ticket #1347.

- Legacy-Id: 7572
Note: SVN reference [7557] has been migrated to Git commit 9b43e159623ed1fb53438541b25a1605922943dd

ietf/bin/rfc-editor-index-updates

@@ -2,6 +2,7 @@
 
 import os, sys, re, json, datetime
 import syslog
+import traceback
 
 syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER)
 
@@ -13,6 +14,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings")
 
 from django.conf import settings
 from optparse import OptionParser
+from django.core.mail import mail_admins
 
 parser = OptionParser()
 parser.add_option("-d", dest="skip_date",
@@ -24,7 +26,9 @@ skip_date = datetime.date.today() - datetime.timedelta(days=365)
 if options.skip_date:
     skip_date = datetime.datetime.strptime(options.skip_date, "%Y-%m-%d").date()
 
+from ietf.utils.pipe import pipe
 from ietf.sync.rfceditor import *
+from ietf.doc.utils import rebuild_reference_relations
 
 syslog.syslog("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_QUEUE_URL)
 
@@ -35,6 +39,28 @@ if len(data) < MIN_INDEX_RESULTS:
     syslog.syslog("Not enough results, only %s" % len(data))
     sys.exit(1)
 
-changed = update_docs_from_rfc_index(data, skip_older_than_date=skip_date)
+changed, new_rfcs  = update_docs_from_rfc_index(data, skip_older_than_date=skip_date)
+
 for c in changed:
     syslog.syslog(c)
+
+# This can be called while processing a notifying POST from the RFC Editor
+# Spawn a child to sync the rfcs and calculate new reference relationships
+# so that the POST
+
+newpid = os.fork()
+
+if newpid == 0:
+    try:
+        pipe("%s -a %s %s" % (settings.RSYNC_BINARY,settings.RFC_TEXT_RSYNC_SOURCE,settings.RFC_PATH))
+        for rfc in new_rfcs:
+            rebuild_reference_relations(rfc)
+            syslog.syslog("Updated references for %s"%rfc.canonical_name())
+    except:
+        subject = "Exception in updating references for new rfcs: %s : %s" % (sys.exc_info()[0],sys.exc_info()[1])
+        msg = "%s\n%s\n----\n%s"%(sys.exc_info()[0],sys.exc_info()[1],traceback.format_tb(sys.exc_info()[2])) 
+        mail_admins(subject,msg,fail_silently=True)
+        syslog.syslog(subject)
+    os._exit(0)
+else:
+    sys.exit(0)

ietf/settings.py

@@ -319,6 +319,8 @@ IANA_SYNC_PASSWORD = "secret"
 IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes"
 IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/"
 
+RFC_TEXT_RSYNC_SOURCE="ftp.rfc-editor.org::rfcs-text-only"
+
 RFC_EDITOR_SYNC_PASSWORD="secret"
 RFC_EDITOR_SYNC_NOTIFICATION_URL = "http://www.rfc-editor.org/parser/parser.php"
 RFC_EDITOR_QUEUE_URL = "http://www.rfc-editor.org/queue2.xml"
@@ -375,6 +377,7 @@ IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB
 DOT_BINARY = '/usr/bin/dot'
 UNFLATTEN_BINARY= '/usr/bin/unflatten'
 PS2PDF_BINARY = '/usr/bin/ps2pdf'
+RSYNC_BINARY = '/usr/bin/rsync'
 
 # Account settings
 DAYS_TO_EXPIRE_REGISTRATION_LINK = 3

ietf/sync/rfceditor.py

@@ -317,6 +317,7 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None):
     system = Person.objects.get(name="(System)")
 
     results = []
+    new_rfcs = []
 
     for rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract in data:
 
@@ -399,6 +400,7 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None):
             other_changes = True
 
             results.append("Added RFC published event: %s" % e.time.strftime("%Y-%m-%d"))
+            new_rfcs.append(doc)
 
         for t in ("draft-iesg", "draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"):
             slug = doc.get_state_slug(t)
@@ -461,7 +463,7 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None):
             doc.time = datetime.datetime.now()
             doc.save()
 
-    return results
+    return results, new_rfcs
 
 
 def post_approved_draft(url, name):

ietf/sync/tests.py

@@ -306,7 +306,7 @@ class RFCSyncTests(TestCase):
         draft_filename = "%s-%s.txt" % (doc.name, doc.rev)
         self.write_draft_file(draft_filename, 5000)
 
-        changed = rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30))
+        changed,_ = rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30))
 
         doc = Document.objects.get(name=doc.name)
 
@@ -329,7 +329,7 @@ class RFCSyncTests(TestCase):
         self.assertTrue(os.path.exists(os.path.join(self.archive_dir, draft_filename)))
 
         # make sure we can apply it again with no changes
-        changed = rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30))
+        changed,_ = rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30))
         self.assertEqual(len(changed), 0)