diff --git a/bin/check-copyright b/bin/check-copyright index 0e6c0a36c..1d52e5d3a 100755 --- a/bin/check-copyright +++ b/bin/check-copyright @@ -35,6 +35,7 @@ COPYRIGHT """ from __future__ import print_function +import datetime import os import sys import time @@ -150,15 +151,18 @@ def get_first_commit(path): cmd = 'svn log %s' % path if opt_verbose > 1: note("Running '%s' ..." % cmd) - commit_log = pipe(cmd) - commit_log = commit_log.splitlines() - commit_log.reverse() - for line in commit_log: - if re.search(loginfo_format, line): - rev, who, when = split_loginfo(line) - break - else: - pass + try: + commit_log = pipe(cmd) + commit_log = commit_log.splitlines() + commit_log.reverse() + for line in commit_log: + if re.search(loginfo_format, line): + rev, who, when = split_loginfo(line) + break + else: + pass + except OSError: + rev, who, when = None, None, datetime.datetime.now() return { path: { 'rev': rev, 'who': who, 'date': when.strftime('%Y-%m-%d %H:%M:%S'), }, } diff --git a/ietf/bin/rfc-editor-index-updates b/ietf/bin/rfc-editor-index-updates index 6354229da..281d8afe0 100755 --- a/ietf/bin/rfc-editor-index-updates +++ b/ietf/bin/rfc-editor-index-updates @@ -1,8 +1,13 @@ #!/usr/bin/env python -import os, sys, datetime +import datetime +import json +import os +import sys import traceback +from six.moves.urllib.request import urlopen + # boilerplate basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) sys.path = [ basedir ] + sys.path @@ -38,15 +43,22 @@ if options.skip_date: log("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_INDEX_URL) -response = ietf.sync.rfceditor.fetch_index_xml(settings.RFC_EDITOR_INDEX_URL) -data = ietf.sync.rfceditor.parse_index(response) +rfc_index_xml = ietf.sync.rfceditor.fetch_index_xml(settings.RFC_EDITOR_INDEX_URL) +index_data = ietf.sync.rfceditor.parse_index(rfc_index_xml) -if len(data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS: - log("Not enough results, only %s" % len(data)) +rfc_errata_json = urlopen(settings.RFC_EDITOR_ERRATA_JSON_URL) +errata_data = json.load(rfc_errata_json) + +if len(index_data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS: + log("Not enough index entries, only %s" % len(index_data)) + sys.exit(1) + +if len(errata_data) < ietf.sync.rfceditor.MIN_ERRATA_RESULTS: + log("Not enough errata entries, only %s" % len(errata_data)) sys.exit(1) new_rfcs = [] -for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(data, skip_older_than_date=skip_date): +for changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=skip_date): if rfc_published: new_rfcs.append(doc) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 561f34894..73cab0089 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -229,7 +229,7 @@ def document_main(request, name, rev=None): if "txt" in found_types: file_urls.append(("htmlized", settings.TOOLS_ID_HTML_URL + name)) - if doc.tags.filter(slug="errata"): + if doc.tags.filter(slug="verified-errata").exists(): file_urls.append(("with errata", settings.RFC_EDITOR_INLINE_ERRATA_URL.format(rfc_number=rfc_number))) if not found_types: diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 43429486d..a5ea76b91 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -9100,6 +9100,16 @@ "model": "name.doctagname", "pk": "sheph-u" }, + { + "fields": { + "desc": "", + "name": "Has verified errata", + "order": 0, + "used": true + }, + "model": "name.doctagname", + "pk": "verified-errata" + }, { "fields": { "desc": "", @@ -14216,7 +14226,7 @@ "fields": { "command": "xym", "switch": "--version", - "time": "2019-12-18T00:13:51.087", + "time": "2020-01-09T00:13:22.771", "used": true, "version": "xym 0.4" }, @@ -14227,7 +14237,7 @@ "fields": { "command": "pyang", "switch": "--version", - "time": "2019-12-18T00:13:52.218", + "time": "2020-01-09T00:13:23.631", "used": true, "version": "pyang 2.1" }, @@ -14238,7 +14248,7 @@ "fields": { "command": "yanglint", "switch": "--version", - "time": "2019-12-18T00:13:52.367", + "time": "2020-01-09T00:13:23.912", "used": true, "version": "yanglint 0.14.80" }, @@ -14249,9 +14259,9 @@ "fields": { "command": "xml2rfc", "switch": "--version", - "time": "2019-12-18T00:13:53.390", + "time": "2020-01-09T00:13:24.917", "used": true, - "version": "xml2rfc 2.36.0" + "version": "xml2rfc 2.37.2" }, "model": "utils.versioninfo", "pk": 4 diff --git a/ietf/name/migrations/0009_add_verified_errata_to_doctagname.py b/ietf/name/migrations/0009_add_verified_errata_to_doctagname.py new file mode 100644 index 000000000..ba64796a8 --- /dev/null +++ b/ietf/name/migrations/0009_add_verified_errata_to_doctagname.py @@ -0,0 +1,24 @@ +# Copyright The IETF Trust 2020, All Rights Reserved +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +from django.db import migrations + +def forward(apps, schema_editor): + DocTagName = apps.get_model('name','DocTagName') + DocTagName.objects.get_or_create(slug='verified-errata', name='Has verified errata', desc='', used=True, order=0) + +def reverse(apps, schema_editor): + DocTagName = apps.get_model('name','DocTagName') + DocTagName.objects.filter(slug='verified-errata').delete() + +class Migration(migrations.Migration): + + dependencies = [ + ('name', '0008_reviewerqueuepolicyname'), + ] + + operations = [ + migrations.RunPython(forward, reverse) + ] diff --git a/ietf/settings.py b/ietf/settings.py index f980f57f6..5cc1b3475 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -734,6 +734,7 @@ RFC_EDITOR_GROUP_NOTIFICATION_EMAIL = "webmaster@rfc-editor.org" #RFC_EDITOR_GROUP_NOTIFICATION_URL = "https://www.rfc-editor.org/notification/group.php" RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" +RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json" RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}&rec_status=0" RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html" diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index 9fb094c2f..29c81b843 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2012-2019, All Rights Reserved +# Copyright The IETF Trust 2012-2020, All Rights Reserved # -*- coding: utf-8 -*- @@ -32,8 +32,9 @@ from ietf.utils.mail import send_mail_text #INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" #POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php" +MIN_ERRATA_RESULTS = 5000 +MIN_INDEX_RESULTS = 8000 MIN_QUEUE_RESULTS = 10 -MIN_INDEX_RESULTS = 5000 def get_child_text(parent_node, tag_name): text = [] @@ -330,11 +331,18 @@ def parse_index(response): return data -def update_docs_from_rfc_index(data, skip_older_than_date=None): +def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=None): """Given parsed data from the RFC Editor index, update the documents in the database. Yields a list of change descriptions for each document, if any.""" + errata = {} + for item in errata_data: + name = item['doc-id'] + if not name in errata: + errata[name] = [] + errata[name].append(item) + std_level_mapping = { "Standard": StdLevelName.objects.get(slug="std"), "Internet Standard": StdLevelName.objects.get(slug="std"), @@ -356,12 +364,13 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None): } tag_has_errata = DocTagName.objects.get(slug='errata') + tag_has_verified_errata = DocTagName.objects.get(slug='verified-errata') relationship_obsoletes = DocRelationshipName.objects.get(slug="obs") relationship_updates = DocRelationshipName.objects.get(slug="updates") system = Person.objects.get(name="(System)") - for rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract in data: + for rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract in index_data: if skip_older_than_date and rfc_published_date < skip_older_than_date: # speed up the process by skipping old entries @@ -493,13 +502,20 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None): changes.append("created alias %s" % prettify_std_name(a)) if has_errata: - if not doc.tags.filter(pk=tag_has_errata.pk): + if not doc.tags.filter(pk=tag_has_errata.pk).exists(): doc.tags.add(tag_has_errata) changes.append("added Errata tag") + has_verified_errata = name.upper() in errata and any([ e['errata_status_code']=='Verified' for e in errata[name.upper()] ]) + if has_verified_errata and not doc.tags.filter(pk=tag_has_verified_errata.pk).exists(): + doc.tags.add(tag_has_verified_errata) + changes.append("added Verified Errata tag") else: if doc.tags.filter(pk=tag_has_errata.pk): doc.tags.remove(tag_has_errata) changes.append("removed Errata tag") + if doc.tags.filter(pk=tag_has_verified_errata.pk): + doc.tags.remove(tag_has_verified_errata) + changes.append("removed Verified Errata tag") if changes: events.append(DocEvent.objects.create( diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index fa7e70386..bbdcf7149 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2012-2019, All Rights Reserved +# Copyright The IETF Trust 2012-2020, All Rights Reserved # -*- coding: utf-8 -*- @@ -315,6 +315,22 @@ class RFCSyncTests(TestCase): area=doc.group.parent.acronym, group=doc.group.acronym) + errata = [{ + "errata_id":1, + "doc-id":"RFC123", + "errata_status_code":"Verified", + "errata_type_code":"Editorial", + "section": "4.1", + "orig_text":" S: 220-smtp.example.com ESMTP Server", + "correct_text":" S: 220 smtp.example.com ESMTP Server", + "notes":"There are 3 instances of this (one on p. 7 and two on p. 8). \n", + "submit_date":"2007-07-19", + "submitter_name":"Rob Siemborski", + "verifier_id":99, + "verifier_name":None, + "update_date":"2019-09-10 09:09:03"}, + ] + data = rfceditor.parse_index(io.StringIO(t)) self.assertEqual(len(data), 1) @@ -339,7 +355,7 @@ class RFCSyncTests(TestCase): self.write_draft_file(draft_filename, 5000) changes = [] - for cs, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30)): + for cs, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)): changes.append(cs) doc = Document.objects.get(name=doc.name) @@ -364,7 +380,7 @@ class RFCSyncTests(TestCase): self.assertTrue(os.path.exists(os.path.join(self.archive_dir, draft_filename))) # make sure we can apply it again with no changes - changed = list(rfceditor.update_docs_from_rfc_index(data, today - datetime.timedelta(days=30))) + changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30))) self.assertEqual(len(changed), 0)