From b7eec8ae26a83869044baaa79b1e4b5b470e1c31 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 14 Dec 2021 20:56:26 +0000 Subject: [PATCH 1/3] Build and maintain a full set of bibxml-ids documents. - Legacy-Id: 19782 --- bin/hourly | 3 +++ .../commands/generate_draft_bibxml_files.py | 2 +- ietf/settings.py | 4 ++-- ietf/submit/tests.py | 17 ++++++++++++++++- ietf/submit/utils.py | 8 ++++++++ 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/bin/hourly b/bin/hourly index ddda01bec..77310302c 100755 --- a/bin/hourly +++ b/bin/hourly @@ -92,6 +92,9 @@ CHARTER=/a/www/ietf-ftp/charter wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt +# Regenerate the last week of bibxml-ids +$DTDIR/ietf/manage.py generate_draft_bibxml_files + # Create and update group wikis #$DTDIR/ietf/manage.py create_group_wikis diff --git a/ietf/doc/management/commands/generate_draft_bibxml_files.py b/ietf/doc/management/commands/generate_draft_bibxml_files.py index 1e1a5e3dd..524384eee 100644 --- a/ietf/doc/management/commands/generate_draft_bibxml_files.py +++ b/ietf/doc/management/commands/generate_draft_bibxml_files.py @@ -61,7 +61,7 @@ class Command(BaseCommand): process_all = options.get("all") days = options.get("days") # - bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml3') + bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') if not os.path.exists(bibxmldir): os.makedirs(bibxmldir) # diff --git a/ietf/settings.py b/ietf/settings.py index 1ba557aea..88d49ca39 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1016,8 +1016,8 @@ HTPASSWD_FILE = "/www/htpasswd" # Generation of pdf files GHOSTSCRIPT_COMMAND = "/usr/bin/gs" -# Generation of bibxml files for xml2rfc -BIBXML_BASE_PATH = '/a/www/ietf-ftp/xml2rfc' +# Generation of bibxml files for internet-drafts +BIBXML_BASE_PATH = '/a/ietfdata/derived/bibxml' # Timezone files for iCalendar TZDATA_ICS_PATH = BASE_DIR + '/../vzic/zoneinfo/' diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 5ca74b351..02f1e9ea7 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -39,7 +39,7 @@ from ietf.submit.mail import add_submission_email, process_response_email from ietf.utils.accesstoken import generate_access_token from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.models import VersionInfo -from ietf.utils.test_utils import login_testing_unauthorized, TestCase +from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent from ietf.utils.draft import Draft @@ -256,6 +256,12 @@ class SubmitTests(BaseSubmitTestCase): return confirmation_url + def verify_bibxml_ids_creation(self, draft): + url = urlreverse('ietf.doc.views_doc.document_bibxml', kwargs=dict(name=draft.name, rev=draft.rev)) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertIn(draft.name, unicontent(r)) + def submit_new_wg(self, formats): # submit new -> supply submitter info -> approve GroupFactory(type_id='wg',acronym='ames') @@ -374,6 +380,8 @@ class SubmitTests(BaseSubmitTestCase): self.assertContains(r, 'Yang Validation') self.assertContains(r, 'WG Document') + self.verify_bibxml_ids_creation(draft) + def test_submit_new_wg_txt(self): self.submit_new_wg(["txt"]) @@ -685,6 +693,7 @@ class SubmitTests(BaseSubmitTestCase): self.assertContains(r, draft.title) # Check submission settings self.assertEqual(draft.submission().xml_version, "3" if 'xml' in formats else None) + self.verify_bibxml_ids_creation(draft) def test_submit_existing_txt(self): self.submit_existing(["txt"]) @@ -839,6 +848,7 @@ class SubmitTests(BaseSubmitTestCase): new_revision = draft.latest_event() self.assertEqual(new_revision.type, "new_revision") self.assertEqual(new_revision.by.name, "Submitter Name") + self.verify_bibxml_ids_creation(draft) def test_submit_new_individual_txt(self): self.submit_new_individual(["txt"]) @@ -879,6 +889,7 @@ class SubmitTests(BaseSubmitTestCase): self.assertEqual(docauth.person, author) self.assertEqual(docauth.affiliation, '') self.assertEqual(docauth.country, '') + self.verify_bibxml_ids_creation(doc) def test_submit_new_draft_no_org_or_address_txt(self): self.submit_new_draft_no_org_or_address(['txt']) @@ -1015,6 +1026,7 @@ class SubmitTests(BaseSubmitTestCase): # Check submission settings self.assertEqual(draft.submission().xml_version, "3" if 'xml' in formats else None) + self.verify_bibxml_ids_creation(draft) def test_submit_new_logged_in_txt(self): self.submit_new_individual_logged_in(["txt"]) @@ -1058,6 +1070,7 @@ class SubmitTests(BaseSubmitTestCase): [str(r) for r in resources], ) self._assert_extresource_change_event(draft, is_present=True) + self.verify_bibxml_ids_creation(draft) def test_submit_update_individual(self): IndividualDraftFactory(name='draft-ietf-random-thing', states=[('draft','rfc')], other_aliases=['rfc9999',], pages=5) @@ -1115,6 +1128,7 @@ class SubmitTests(BaseSubmitTestCase): self.assertContains(r, draft.name) self.assertContains(r, draft.title) self._assert_extresource_change_event(draft, is_present=False) + self.verify_bibxml_ids_creation(draft) def submit_existing_with_extresources(self, group_type, stream_type='ietf'): """Submit a draft with external resources @@ -1390,6 +1404,7 @@ class SubmitTests(BaseSubmitTestCase): draft = Document.objects.get(docalias__name=name) self.assertEqual(draft.rev, rev) self.assertEqual(draft.docextresource_set.count(), 0) + self.verify_bibxml_ids_creation(draft) def test_search_for_submission_and_edit_as_secretariat(self): # submit -> edit diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index b4a8de8df..501f21889 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -17,6 +17,7 @@ from django.core.validators import validate_email from django.db import transaction from django.http import HttpRequest # pyflakes:ignore from django.utils.module_loading import import_string +from django.template.loader import render_to_string import debug # pyflakes:ignore @@ -449,6 +450,13 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): submission.save() create_submission_event(request, submission, approved_subm_desc) + + # Create bibxml-ids entry + ref_text = '%s' % render_to_string('doc/bibxml.xml', {'name':draft.name, 'doc': draft, 'doc_bibtype':'I-D'}) + ref_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s.xml' % (draft.name[6:], )) + with io.open(ref_file_name, "w", encoding='utf-8') as f: + f.write(ref_text) + log.log(f"{submission.name}: done") From f6035850d857cd1a37803a87547733464033d944 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 14 Dec 2021 21:53:10 +0000 Subject: [PATCH 2/3] update bibxml-ids generation on draft submission. - Legacy-Id: 19783 --- ietf/submit/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 501f21889..7015a3b04 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -456,6 +456,9 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): ref_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s.xml' % (draft.name[6:], )) with io.open(ref_file_name, "w", encoding='utf-8') as f: f.write(ref_text) + ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name[6:], draft.rev )) + with io.open(ref_rev_file_name, "w", encoding='utf-8') as f: + f.write(ref_text) log.log(f"{submission.name}: done") From 865dc62676f452aff7df4f63de7d0b5fcc71a881 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Dec 2021 22:33:47 +0000 Subject: [PATCH 3/3] Change strategy to only creating a single file per doc/rev, and not creating the 'draft-'-less or version-less variants. Corrected calculation of date in the management command. - Legacy-Id: 19789 --- .../commands/generate_draft_bibxml_files.py | 14 ++++++++------ ietf/settings.py | 2 +- ietf/submit/tests.py | 16 +++++++++++----- ietf/submit/utils.py | 14 +++++++++----- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/ietf/doc/management/commands/generate_draft_bibxml_files.py b/ietf/doc/management/commands/generate_draft_bibxml_files.py index 524384eee..8bdaa0a86 100644 --- a/ietf/doc/management/commands/generate_draft_bibxml_files.py +++ b/ietf/doc/management/commands/generate_draft_bibxml_files.py @@ -75,19 +75,21 @@ class Command(BaseCommand): for e in doc_events: self.mutter('%s %s' % (e.time, e.doc.name)) try: - e.doc.date = e.time.date() doc = e.doc if e.rev != doc.rev: for h in doc.history_set.order_by("-time"): if e.rev == h.rev: doc = h break + doc.date = e.time.date() ref_text = '%s' % render_to_string('doc/bibxml.xml', {'name':doc.name, 'doc': doc, 'doc_bibtype':'I-D'}) - if e.rev == e.doc.rev: - ref_file_name = os.path.join(bibxmldir, 'reference.I-D.%s.xml' % (doc.name[6:], )) - self.write(ref_file_name, ref_text) - else: - self.note("Skipping %s; outdated revision: %s" % (os.path.basename(ref_file_name), e.rev)) + # if e.rev == e.doc.rev: + # for name in (doc.name, doc.name[6:]): + # ref_file_name = os.path.join(bibxmldir, 'reference.I-D.%s.xml' % (name, )) + # self.write(ref_file_name, ref_text) + # for name in (doc.name, doc.name[6:]): + # ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (name, doc.rev)) + # self.write(ref_rev_file_name, ref_text) ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (doc.name, doc.rev)) self.write(ref_rev_file_name, ref_text) except Exception as ee: diff --git a/ietf/settings.py b/ietf/settings.py index 88d49ca39..a48cbebd6 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1016,7 +1016,7 @@ HTPASSWD_FILE = "/www/htpasswd" # Generation of pdf files GHOSTSCRIPT_COMMAND = "/usr/bin/gs" -# Generation of bibxml files for internet-drafts +# Generation of bibxml files (currently only for internet drafts) BIBXML_BASE_PATH = '/a/ietfdata/derived/bibxml' # Timezone files for iCalendar diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 02f1e9ea7..949c75d66 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -39,7 +39,7 @@ from ietf.submit.mail import add_submission_email, process_response_email from ietf.utils.accesstoken import generate_access_token from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.models import VersionInfo -from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent +from ietf.utils.test_utils import login_testing_unauthorized, TestCase from ietf.utils.draft import Draft @@ -50,6 +50,7 @@ class BaseSubmitTestCase(TestCase): 'SUBMIT_YANG_DRAFT_MODEL_DIR', 'SUBMIT_YANG_IANA_MODEL_DIR', 'SUBMIT_YANG_CATALOG_DIR', + 'BIBXML_BASE_PATH', ] def setUp(self): @@ -59,6 +60,7 @@ class BaseSubmitTestCase(TestCase): # old drafts may not be moved out of the way properly. self.saved_repository_path = settings.IDSUBMIT_REPOSITORY_PATH settings.IDSUBMIT_REPOSITORY_PATH = settings.INTERNET_DRAFT_PATH + os.mkdir(os.path.join(settings.BIBXML_BASE_PATH,'bibxml-ids')) def tearDown(self): settings.IDSUBMIT_REPOSITORY_PATH = self.saved_repository_path @@ -257,10 +259,14 @@ class SubmitTests(BaseSubmitTestCase): return confirmation_url def verify_bibxml_ids_creation(self, draft): - url = urlreverse('ietf.doc.views_doc.document_bibxml', kwargs=dict(name=draft.name, rev=draft.rev)) - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - self.assertIn(draft.name, unicontent(r)) + # for name in (draft.name, draft.name[6:]): + # ref_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s.xml' % (name, )) + # self.assertTrue(os.path.exists(ref_file_name)) + # ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (name, draft.rev )) + # self.assertTrue(os.path.exists(ref_rev_file_name)) + ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev )) + self.assertTrue(os.path.exists(ref_rev_file_name)) + def submit_new_wg(self, formats): # submit new -> supply submitter info -> approve diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 7015a3b04..9e9ef274c 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -453,12 +453,16 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): # Create bibxml-ids entry ref_text = '%s' % render_to_string('doc/bibxml.xml', {'name':draft.name, 'doc': draft, 'doc_bibtype':'I-D'}) - ref_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s.xml' % (draft.name[6:], )) - with io.open(ref_file_name, "w", encoding='utf-8') as f: - f.write(ref_text) - ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name[6:], draft.rev )) + # for name in (draft.name, draft.name[6:]): + # ref_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s.xml' % (name, )) + # with io.open(ref_file_name, "w", encoding='utf-8') as f: + # f.write(ref_text) + # ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (name, draft.rev )) + # with io.open(ref_rev_file_name, "w", encoding='utf-8') as f: + # f.write(ref_text) + ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev )) with io.open(ref_rev_file_name, "w", encoding='utf-8') as f: - f.write(ref_text) + f.write(ref_text) log.log(f"{submission.name}: done")