Added fetch and use of the RFC-Editor's errata.json file in order to set a new 'verified-errata' document tag, and use that to show the link to inline-errata documents only when there is verified errata. Fixes issue #2861.
- Legacy-Id: 17212
This commit is contained in:
Henrik Levkowetz
parent
a3edd91698
commit
ba3010e3b0
|
|
@ -35,6 +35,7 @@ COPYRIGHT
|
|||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
|
@ -150,15 +151,18 @@ def get_first_commit(path):
|
|||
cmd = 'svn log %s' % path
|
||||
if opt_verbose > 1:
|
||||
note("Running '%s' ..." % cmd)
|
||||
commit_log = pipe(cmd)
|
||||
commit_log = commit_log.splitlines()
|
||||
commit_log.reverse()
|
||||
for line in commit_log:
|
||||
if re.search(loginfo_format, line):
|
||||
rev, who, when = split_loginfo(line)
|
||||
break
|
||||
else:
|
||||
pass
|
||||
try:
|
||||
commit_log = pipe(cmd)
|
||||
commit_log = commit_log.splitlines()
|
||||
commit_log.reverse()
|
||||
for line in commit_log:
|
||||
if re.search(loginfo_format, line):
|
||||
rev, who, when = split_loginfo(line)
|
||||
break
|
||||
else:
|
||||
pass
|
||||
except OSError:
|
||||
rev, who, when = None, None, datetime.datetime.now()
|
||||
return { path: { 'rev': rev, 'who': who, 'date': when.strftime('%Y-%m-%d %H:%M:%S'), }, }
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,13 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os, sys, datetime
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from six.moves.urllib.request import urlopen
|
||||
|
||||
# boilerplate
|
||||
basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
|
||||
sys.path = [ basedir ] + sys.path
|
||||
|
|
@ -38,15 +43,22 @@ if options.skip_date:
|
|||
|
||||
log("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_INDEX_URL)
|
||||
|
||||
response = ietf.sync.rfceditor.fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
|
||||
data = ietf.sync.rfceditor.parse_index(response)
|
||||
rfc_index_xml = ietf.sync.rfceditor.fetch_index_xml(settings.RFC_EDITOR_INDEX_URL)
|
||||
index_data = ietf.sync.rfceditor.parse_index(rfc_index_xml)
|
||||
|
||||
if len(data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS:
|
||||
log("Not enough results, only %s" % len(data))
|
||||
rfc_errata_json = urlopen(settings.RFC_EDITOR_ERRATA_JSON_URL)
|
||||
errata_data = json.load(rfc_errata_json)
|
||||
|
||||
if len(index_data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS:
|
||||
log("Not enough index entries, only %s" % len(index_data))
|
||||
sys.exit(1)
|
||||
|
||||
if len(errata_data) < ietf.sync.rfceditor.MIN_ERRATA_RESULTS:
|
||||
log("Not enough errata entries, only %s" % len(errata_data))
|
||||
sys.exit(1)
|
||||
|
||||
new_rfcs = []
|
||||
for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(data, skip_older_than_date=skip_date):
|
||||
for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=skip_date):
|
||||
if rfc_published:
|
||||
new_rfcs.append(doc)
|
||||
|
||||
|
|
|
|||
|
|
@ -229,7 +229,7 @@ def document_main(request, name, rev=None):
|
|||
|
||||
if "txt" in found_types:
|
||||
file_urls.append(("htmlized", settings.TOOLS_ID_HTML_URL + name))
|
||||
if doc.tags.filter(slug="errata"):
|
||||
if doc.tags.filter(slug="verified-errata").exists():
|
||||
file_urls.append(("with errata", settings.RFC_EDITOR_INLINE_ERRATA_URL.format(rfc_number=rfc_number)))
|
||||
|
||||
if not found_types:
|
||||
|
|
|
|||
|
|
@ -9100,6 +9100,16 @@
|
|||
"model": "name.doctagname",
|
||||
"pk": "sheph-u"
|
||||
},
|
||||
{
|
||||
"fields": {
|
||||
"desc": "",
|
||||
"name": "Has verified errata",
|
||||
"order": 0,
|
||||
"used": true
|
||||
},
|
||||
"model": "name.doctagname",
|
||||
"pk": "verified-errata"
|
||||
},
|
||||
{
|
||||
"fields": {
|
||||
"desc": "",
|
||||
|
|
@ -14216,7 +14226,7 @@
|
|||
"fields": {
|
||||
"command": "xym",
|
||||
"switch": "--version",
|
||||
"time": "2019-12-18T00:13:51.087",
|
||||
"time": "2020-01-09T00:13:22.771",
|
||||
"used": true,
|
||||
"version": "xym 0.4"
|
||||
},
|
||||
|
|
@ -14227,7 +14237,7 @@
|
|||
"fields": {
|
||||
"command": "pyang",
|
||||
"switch": "--version",
|
||||
"time": "2019-12-18T00:13:52.218",
|
||||
"time": "2020-01-09T00:13:23.631",
|
||||
"used": true,
|
||||
"version": "pyang 2.1"
|
||||
},
|
||||
|
|
@ -14238,7 +14248,7 @@
|
|||
"fields": {
|
||||
"command": "yanglint",
|
||||
"switch": "--version",
|
||||
"time": "2019-12-18T00:13:52.367",
|
||||
"time": "2020-01-09T00:13:23.912",
|
||||
"used": true,
|
||||
"version": "yanglint 0.14.80"
|
||||
},
|
||||
|
|
@ -14249,9 +14259,9 @@
|
|||
"fields": {
|
||||
"command": "xml2rfc",
|
||||
"switch": "--version",
|
||||
"time": "2019-12-18T00:13:53.390",
|
||||
"time": "2020-01-09T00:13:24.917",
|
||||
"used": true,
|
||||
"version": "xml2rfc 2.36.0"
|
||||
"version": "xml2rfc 2.37.2"
|
||||
},
|
||||
"model": "utils.versioninfo",
|
||||
"pk": 4
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
# Copyright The IETF Trust 2020, All Rights Reserved
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
def forward(apps, schema_editor):
|
||||
DocTagName = apps.get_model('name','DocTagName')
|
||||
DocTagName.objects.get_or_create(slug='verified-errata', name='Has verified errata', desc='', used=True, order=0)
|
||||
|
||||
def reverse(apps, schema_editor):
|
||||
DocTagName = apps.get_model('name','DocTagName')
|
||||
DocTagName.objects.filter(slug='verified-errata').delete()
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('name', '0008_reviewerqueuepolicyname'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RunPython(forward, reverse)
|
||||
]
|
||||
|
|
@ -734,6 +734,7 @@ RFC_EDITOR_GROUP_NOTIFICATION_EMAIL = "webmaster@rfc-editor.org"
|
|||
#RFC_EDITOR_GROUP_NOTIFICATION_URL = "https://www.rfc-editor.org/notification/group.php"
|
||||
RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml"
|
||||
RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
|
||||
RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json"
|
||||
RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}&rec_status=0"
|
||||
RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright The IETF Trust 2012-2019, All Rights Reserved
|
||||
# Copyright The IETF Trust 2012-2020, All Rights Reserved
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
|
|
@ -32,8 +32,9 @@ from ietf.utils.mail import send_mail_text
|
|||
#INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
|
||||
#POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php"
|
||||
|
||||
MIN_ERRATA_RESULTS = 5000
|
||||
MIN_INDEX_RESULTS = 8000
|
||||
MIN_QUEUE_RESULTS = 10
|
||||
MIN_INDEX_RESULTS = 5000
|
||||
|
||||
def get_child_text(parent_node, tag_name):
|
||||
text = []
|
||||
|
|
@ -330,11 +331,18 @@ def parse_index(response):
|
|||
return data
|
||||
|
||||
|
||||
def update_docs_from_rfc_index(data, skip_older_than_date=None):
|
||||
def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=None):
|
||||
"""Given parsed data from the RFC Editor index, update the documents
|
||||
in the database. Yields a list of change descriptions for each
|
||||
document, if any."""
|
||||
|
||||
errata = {}
|
||||
for item in errata_data:
|
||||
name = item['doc-id']
|
||||
if not name in errata:
|
||||
errata[name] = []
|
||||
errata[name].append(item)
|
||||
|
||||
std_level_mapping = {
|
||||
"Standard": StdLevelName.objects.get(slug="std"),
|
||||
"Internet Standard": StdLevelName.objects.get(slug="std"),
|
||||
|
|
@ -356,12 +364,13 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None):
|
|||
}
|
||||
|
||||
tag_has_errata = DocTagName.objects.get(slug='errata')
|
||||
tag_has_verified_errata = DocTagName.objects.get(slug='verified-errata')
|
||||
relationship_obsoletes = DocRelationshipName.objects.get(slug="obs")
|
||||
relationship_updates = DocRelationshipName.objects.get(slug="updates")
|
||||
|
||||
system = Person.objects.get(name="(System)")
|
||||
|
||||
for rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract in data:
|
||||
for rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract in index_data:
|
||||
|
||||
if skip_older_than_date and rfc_published_date < skip_older_than_date:
|
||||
# speed up the process by skipping old entries
|
||||
|
|
@ -493,13 +502,20 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None):
|
|||
changes.append("created alias %s" % prettify_std_name(a))
|
||||
|
||||
if has_errata:
|
||||
if not doc.tags.filter(pk=tag_has_errata.pk):
|
||||
if not doc.tags.filter(pk=tag_has_errata.pk).exists():
|
||||
doc.tags.add(tag_has_errata)
|
||||
changes.append("added Errata tag")
|
||||
has_verified_errata = name.upper() in errata and any([ e['errata_status_code']=='Verified' for e in errata[name.upper()] ])
|
||||
if has_verified_errata and not doc.tags.filter(pk=tag_has_verified_errata.pk).exists():
|
||||
doc.tags.add(tag_has_verified_errata)
|
||||
changes.append("added Verified Errata tag")
|
||||
else:
|
||||
if doc.tags.filter(pk=tag_has_errata.pk):
|
||||
doc.tags.remove(tag_has_errata)
|
||||
changes.append("removed Errata tag")
|
||||
if doc.tags.filter(pk=tag_has_verified_errata.pk):
|
||||
doc.tags.remove(tag_has_verified_errata)
|
||||
changes.append("removed Verified Errata tag")
|
||||
|
||||
if changes:
|
||||
events.append(DocEvent.objects.create(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright The IETF Trust 2012-2019, All Rights Reserved
|
||||
# Copyright The IETF Trust 2012-2020, All Rights Reserved
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
|
|
@ -315,6 +315,22 @@ class RFCSyncTests(TestCase):
|
|||
area=doc.group.parent.acronym,
|
||||
group=doc.group.acronym)
|
||||
|
||||
errata = [{
|
||||
"errata_id":1,
|
||||
"doc-id":"RFC123",
|
||||
"errata_status_code":"Verified",
|
||||
"errata_type_code":"Editorial",
|
||||
"section": "4.1",
|
||||
"orig_text":" S: 220-smtp.example.com ESMTP Server",
|
||||
"correct_text":" S: 220 smtp.example.com ESMTP Server",
|
||||
"notes":"There are 3 instances of this (one on p. 7 and two on p. 8). \n",
|
||||
"submit_date":"2007-07-19",
|
||||
"submitter_name":"Rob Siemborski",
|
||||
"verifier_id":99,
|
||||
"verifier_name":None,
|
||||
"update_date":"2019-09-10 09:09:03"},
|
||||
]
|
||||
|
||||
data = rfceditor.parse_index(io.StringIO(t))
|
||||
self.assertEqual(len(data), 1)
|
||||
|
||||
|
|
@ -339,7 +355,7 @@ class RFCSyncTests(TestCase):
|
|||
self.write_draft_file(draft_filename, 5000)
|
||||
|
||||
changes = []
|
||||
for cs, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30)):
|
||||
for cs, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)):
|
||||
changes.append(cs)
|
||||
|
||||
doc = Document.objects.get(name=doc.name)
|
||||
|
|
@ -364,7 +380,7 @@ class RFCSyncTests(TestCase):
|
|||
self.assertTrue(os.path.exists(os.path.join(self.archive_dir, draft_filename)))
|
||||
|
||||
# make sure we can apply it again with no changes
|
||||
changed = list(rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30)))
|
||||
changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)))
|
||||
self.assertEqual(len(changed), 0)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue