diff --git a/bin/hourly b/bin/hourly index ddda01bec..77310302c 100755 --- a/bin/hourly +++ b/bin/hourly @@ -92,6 +92,9 @@ CHARTER=/a/www/ietf-ftp/charter wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt +# Regenerate the last week of bibxml-ids +$DTDIR/ietf/manage.py generate_draft_bibxml_files + # Create and update group wikis #$DTDIR/ietf/manage.py create_group_wikis diff --git a/ietf/doc/management/commands/generate_draft_bibxml_files.py b/ietf/doc/management/commands/generate_draft_bibxml_files.py index 1e1a5e3dd..8bdaa0a86 100644 --- a/ietf/doc/management/commands/generate_draft_bibxml_files.py +++ b/ietf/doc/management/commands/generate_draft_bibxml_files.py @@ -61,7 +61,7 @@ class Command(BaseCommand): process_all = options.get("all") days = options.get("days") # - bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml3') + bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') if not os.path.exists(bibxmldir): os.makedirs(bibxmldir) # @@ -75,19 +75,21 @@ class Command(BaseCommand): for e in doc_events: self.mutter('%s %s' % (e.time, e.doc.name)) try: - e.doc.date = e.time.date() doc = e.doc if e.rev != doc.rev: for h in doc.history_set.order_by("-time"): if e.rev == h.rev: doc = h break + doc.date = e.time.date() ref_text = '%s' % render_to_string('doc/bibxml.xml', {'name':doc.name, 'doc': doc, 'doc_bibtype':'I-D'}) - if e.rev == e.doc.rev: - ref_file_name = os.path.join(bibxmldir, 'reference.I-D.%s.xml' % (doc.name[6:], )) - self.write(ref_file_name, ref_text) - else: - self.note("Skipping %s; outdated revision: %s" % (os.path.basename(ref_file_name), e.rev)) + # if e.rev == e.doc.rev: + # for name in (doc.name, doc.name[6:]): + # ref_file_name = os.path.join(bibxmldir, 'reference.I-D.%s.xml' % (name, )) + # self.write(ref_file_name, ref_text) + # for name in (doc.name, doc.name[6:]): + # ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (name, doc.rev)) + # self.write(ref_rev_file_name, ref_text) ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (doc.name, doc.rev)) self.write(ref_rev_file_name, ref_text) except Exception as ee: diff --git a/ietf/settings.py b/ietf/settings.py index 1ba557aea..a48cbebd6 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1016,8 +1016,8 @@ HTPASSWD_FILE = "/www/htpasswd" # Generation of pdf files GHOSTSCRIPT_COMMAND = "/usr/bin/gs" -# Generation of bibxml files for xml2rfc -BIBXML_BASE_PATH = '/a/www/ietf-ftp/xml2rfc' +# Generation of bibxml files (currently only for internet drafts) +BIBXML_BASE_PATH = '/a/ietfdata/derived/bibxml' # Timezone files for iCalendar TZDATA_ICS_PATH = BASE_DIR + '/../vzic/zoneinfo/' diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 5ca74b351..949c75d66 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -50,6 +50,7 @@ class BaseSubmitTestCase(TestCase): 'SUBMIT_YANG_DRAFT_MODEL_DIR', 'SUBMIT_YANG_IANA_MODEL_DIR', 'SUBMIT_YANG_CATALOG_DIR', + 'BIBXML_BASE_PATH', ] def setUp(self): @@ -59,6 +60,7 @@ class BaseSubmitTestCase(TestCase): # old drafts may not be moved out of the way properly. self.saved_repository_path = settings.IDSUBMIT_REPOSITORY_PATH settings.IDSUBMIT_REPOSITORY_PATH = settings.INTERNET_DRAFT_PATH + os.mkdir(os.path.join(settings.BIBXML_BASE_PATH,'bibxml-ids')) def tearDown(self): settings.IDSUBMIT_REPOSITORY_PATH = self.saved_repository_path @@ -256,6 +258,16 @@ class SubmitTests(BaseSubmitTestCase): return confirmation_url + def verify_bibxml_ids_creation(self, draft): + # for name in (draft.name, draft.name[6:]): + # ref_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s.xml' % (name, )) + # self.assertTrue(os.path.exists(ref_file_name)) + # ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (name, draft.rev )) + # self.assertTrue(os.path.exists(ref_rev_file_name)) + ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev )) + self.assertTrue(os.path.exists(ref_rev_file_name)) + + def submit_new_wg(self, formats): # submit new -> supply submitter info -> approve GroupFactory(type_id='wg',acronym='ames') @@ -374,6 +386,8 @@ class SubmitTests(BaseSubmitTestCase): self.assertContains(r, 'Yang Validation') self.assertContains(r, 'WG Document') + self.verify_bibxml_ids_creation(draft) + def test_submit_new_wg_txt(self): self.submit_new_wg(["txt"]) @@ -685,6 +699,7 @@ class SubmitTests(BaseSubmitTestCase): self.assertContains(r, draft.title) # Check submission settings self.assertEqual(draft.submission().xml_version, "3" if 'xml' in formats else None) + self.verify_bibxml_ids_creation(draft) def test_submit_existing_txt(self): self.submit_existing(["txt"]) @@ -839,6 +854,7 @@ class SubmitTests(BaseSubmitTestCase): new_revision = draft.latest_event() self.assertEqual(new_revision.type, "new_revision") self.assertEqual(new_revision.by.name, "Submitter Name") + self.verify_bibxml_ids_creation(draft) def test_submit_new_individual_txt(self): self.submit_new_individual(["txt"]) @@ -879,6 +895,7 @@ class SubmitTests(BaseSubmitTestCase): self.assertEqual(docauth.person, author) self.assertEqual(docauth.affiliation, '') self.assertEqual(docauth.country, '') + self.verify_bibxml_ids_creation(doc) def test_submit_new_draft_no_org_or_address_txt(self): self.submit_new_draft_no_org_or_address(['txt']) @@ -1015,6 +1032,7 @@ class SubmitTests(BaseSubmitTestCase): # Check submission settings self.assertEqual(draft.submission().xml_version, "3" if 'xml' in formats else None) + self.verify_bibxml_ids_creation(draft) def test_submit_new_logged_in_txt(self): self.submit_new_individual_logged_in(["txt"]) @@ -1058,6 +1076,7 @@ class SubmitTests(BaseSubmitTestCase): [str(r) for r in resources], ) self._assert_extresource_change_event(draft, is_present=True) + self.verify_bibxml_ids_creation(draft) def test_submit_update_individual(self): IndividualDraftFactory(name='draft-ietf-random-thing', states=[('draft','rfc')], other_aliases=['rfc9999',], pages=5) @@ -1115,6 +1134,7 @@ class SubmitTests(BaseSubmitTestCase): self.assertContains(r, draft.name) self.assertContains(r, draft.title) self._assert_extresource_change_event(draft, is_present=False) + self.verify_bibxml_ids_creation(draft) def submit_existing_with_extresources(self, group_type, stream_type='ietf'): """Submit a draft with external resources @@ -1390,6 +1410,7 @@ class SubmitTests(BaseSubmitTestCase): draft = Document.objects.get(docalias__name=name) self.assertEqual(draft.rev, rev) self.assertEqual(draft.docextresource_set.count(), 0) + self.verify_bibxml_ids_creation(draft) def test_search_for_submission_and_edit_as_secretariat(self): # submit -> edit diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index b4a8de8df..9e9ef274c 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -17,6 +17,7 @@ from django.core.validators import validate_email from django.db import transaction from django.http import HttpRequest # pyflakes:ignore from django.utils.module_loading import import_string +from django.template.loader import render_to_string import debug # pyflakes:ignore @@ -449,6 +450,20 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): submission.save() create_submission_event(request, submission, approved_subm_desc) + + # Create bibxml-ids entry + ref_text = '%s' % render_to_string('doc/bibxml.xml', {'name':draft.name, 'doc': draft, 'doc_bibtype':'I-D'}) + # for name in (draft.name, draft.name[6:]): + # ref_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s.xml' % (name, )) + # with io.open(ref_file_name, "w", encoding='utf-8') as f: + # f.write(ref_text) + # ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (name, draft.rev )) + # with io.open(ref_rev_file_name, "w", encoding='utf-8') as f: + # f.write(ref_text) + ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev )) + with io.open(ref_rev_file_name, "w", encoding='utf-8') as f: + f.write(ref_text) + log.log(f"{submission.name}: done")