Added exception logging for ietf/bin/rfc-editor-queue-updates. Changed the logging in the ietf/bin/rfc-editor-* scripts to use ietf.utils.log.log(). Reordered some imports.
- Legacy-Id: 15318
This commit is contained in:
Henrik Levkowetz
parent
85c24b8450
commit
d6565f0450
|
|
@ -1,7 +1,6 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os, sys, datetime
|
||||
import syslog
|
||||
import traceback
|
||||
|
||||
# boilerplate
|
||||
|
|
@ -13,8 +12,6 @@ virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
|
|||
if os.path.exists(virtualenv_activation):
|
||||
execfile(virtualenv_activation, dict(__file__=virtualenv_activation))
|
||||
|
||||
syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER)
|
||||
|
||||
import django
|
||||
django.setup()
|
||||
|
||||
|
|
@ -22,6 +19,13 @@ from django.conf import settings
|
|||
from optparse import OptionParser
|
||||
from django.core.mail import mail_admins
|
||||
|
||||
from ietf.doc.utils import rebuild_reference_relations
|
||||
from ietf.utils.log import log
|
||||
from ietf.utils.pipe import pipe
|
||||
|
||||
import ietf.sync.rfceditor
|
||||
|
||||
|
||||
parser = OptionParser()
|
||||
parser.add_option("-d", dest="skip_date",
|
||||
help="To speed up processing skip RFCs published before this date (default is one year ago)", metavar="YYYY-MM-DD")
|
||||
|
|
@ -32,17 +36,13 @@ skip_date = datetime.date.today() - datetime.timedelta(days=365)
|
|||
if options.skip_date:
|
||||
skip_date = datetime.datetime.strptime(options.skip_date, "%Y-%m-%d").date()
|
||||
|
||||
from ietf.utils.pipe import pipe
|
||||
from ietf.doc.utils import rebuild_reference_relations
|
||||
import ietf.sync.rfceditor
|
||||
|
||||
syslog.syslog("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_INDEX_URL)
|
||||
log("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_INDEX_URL)
|
||||
|
||||
response = ietf.sync.rfceditor.fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
|
||||
data = ietf.sync.rfceditor.parse_index(response)
|
||||
|
||||
if len(data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS:
|
||||
syslog.syslog("Not enough results, only %s" % len(data))
|
||||
log("Not enough results, only %s" % len(data))
|
||||
sys.exit(1)
|
||||
|
||||
new_rfcs = []
|
||||
|
|
@ -51,8 +51,7 @@ for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_inde
|
|||
new_rfcs.append(doc)
|
||||
|
||||
for c in changes:
|
||||
syslog.syslog("%s: %s" % (doc.name, c))
|
||||
print "%s: %s" % (doc.name, c)
|
||||
log("%s: %s" % (doc.name, c))
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
|
@ -67,12 +66,12 @@ if newpid == 0:
|
|||
pipe("%s -a %s %s" % (settings.RSYNC_BINARY,settings.RFC_TEXT_RSYNC_SOURCE,settings.RFC_PATH))
|
||||
for rfc in new_rfcs:
|
||||
rebuild_reference_relations(rfc)
|
||||
syslog.syslog("Updated references for %s"%rfc.canonical_name())
|
||||
log("Updated references for %s"%rfc.canonical_name())
|
||||
except:
|
||||
subject = "Exception in updating references for new rfcs: %s : %s" % (sys.exc_info()[0],sys.exc_info()[1])
|
||||
msg = "%s\n%s\n----\n%s"%(sys.exc_info()[0],sys.exc_info()[1],traceback.format_tb(sys.exc_info()[2]))
|
||||
mail_admins(subject,msg,fail_silently=True)
|
||||
syslog.syslog(subject)
|
||||
log(subject)
|
||||
os._exit(0)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os, sys, re, json, datetime
|
||||
import syslog
|
||||
import os, sys
|
||||
|
||||
# boilerplate
|
||||
basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
|
||||
|
|
@ -12,28 +11,28 @@ virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
|
|||
if os.path.exists(virtualenv_activation):
|
||||
execfile(virtualenv_activation, dict(__file__=virtualenv_activation))
|
||||
|
||||
syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER)
|
||||
|
||||
import django
|
||||
django.setup()
|
||||
|
||||
from django.conf import settings
|
||||
from ietf.sync.rfceditor import *
|
||||
|
||||
syslog.syslog("Updating RFC Editor queue states from %s" % settings.RFC_EDITOR_QUEUE_URL)
|
||||
from ietf.sync.rfceditor import fetch_queue_xml, parse_queue, MIN_QUEUE_RESULTS, update_drafts_from_queue
|
||||
from ietf.utils.log import log
|
||||
|
||||
log("Updating RFC Editor queue states from %s" % settings.RFC_EDITOR_QUEUE_URL)
|
||||
|
||||
response = fetch_queue_xml(settings.RFC_EDITOR_QUEUE_URL)
|
||||
drafts, warnings = parse_queue(response)
|
||||
for w in warnings:
|
||||
syslog.syslog(u"WARNING: %s" % w)
|
||||
log(u"Warning: %s" % w)
|
||||
|
||||
if len(drafts) < MIN_QUEUE_RESULTS:
|
||||
syslog.syslog("Not enough results, only %s" % len(drafts))
|
||||
log("Not enough results, only %s" % len(drafts))
|
||||
sys.exit(1)
|
||||
|
||||
changed, warnings = update_drafts_from_queue(drafts)
|
||||
for w in warnings:
|
||||
syslog.syslog(u"WARNING: %s" % w)
|
||||
log(u"Warning: %s" % w)
|
||||
|
||||
for c in changed:
|
||||
syslog.syslog(u"Updated %s" % c)
|
||||
log(u"Updated %s" % c)
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ from xml.dom import pulldom, Node
|
|||
|
||||
from django.conf import settings
|
||||
|
||||
import debug # pyflakes:ignore
|
||||
|
||||
from ietf.doc.models import ( Document, DocAlias, State, StateType, DocEvent, DocRelationshipName,
|
||||
DocTagName, DocTypeName, RelatedDocument )
|
||||
from ietf.doc.expire import move_draft_files_to_archive
|
||||
|
|
@ -45,6 +47,7 @@ def parse_queue(response):
|
|||
stream = None
|
||||
|
||||
for event, node in events:
|
||||
try:
|
||||
if event == pulldom.START_ELEMENT and node.tagName == "entry":
|
||||
events.expandNode(node)
|
||||
node.normalize()
|
||||
|
|
@ -106,6 +109,10 @@ def parse_queue(response):
|
|||
else:
|
||||
stream = None
|
||||
warnings.append("unrecognized section " + name)
|
||||
except Exception as e:
|
||||
log("Exception when processing an RFC queue entry: %s" % e)
|
||||
log("node: %s" % node)
|
||||
raise
|
||||
|
||||
return drafts, warnings
|
||||
|
||||
|
|
@ -243,6 +250,7 @@ def parse_index(response):
|
|||
data = []
|
||||
events = pulldom.parse(response)
|
||||
for event, node in events:
|
||||
try:
|
||||
if event == pulldom.START_ELEMENT and node.tagName in ["bcp-entry", "fyi-entry", "std-entry"]:
|
||||
events.expandNode(node)
|
||||
node.normalize()
|
||||
|
|
@ -303,7 +311,10 @@ def parse_index(response):
|
|||
has_errata = 0
|
||||
|
||||
data.append((rfc_number,title,authors,rfc_published_date,current_status,updates,updated_by,obsoletes,obsoleted_by,[],draft,has_errata,stream,wg,file_formats,pages,abstract))
|
||||
|
||||
except Exception as e:
|
||||
log("Exception when processing an RFC index entry: %s" % e)
|
||||
log("node: %s" % node)
|
||||
raise
|
||||
for d in data:
|
||||
k = "RFC%04d" % d[0]
|
||||
if k in also_list:
|
||||
|
|
|
|||
Loading…
Reference in a new issue