altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Provide and maintain an rsyncable bibxml-ids dataset. Commit ready for merge.

Browse source 
- Legacy-Id: 19790
This commit is contained in:
Robert Sparks 2021-12-16 22:36:57 +00:00
parent c1649f8f0b 865dc62676
commit 28e270b1cd
5 changed files with 50 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
bin/hourly
View file

@ -92,6 +92,9 @@ CHARTER=/a/www/ietf-ftp/charter
wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt
wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt
# Regenerate the last week of bibxml-ids
$DTDIR/ietf/manage.py generate_draft_bibxml_files
# Create and update group wikis
#$DTDIR/ietf/manage.py create_group_wikis

16
ietf/doc/management/commands/generate_draft_bibxml_files.py
View file

@ -61,7 +61,7 @@ class Command(BaseCommand):
process_all = options.get("all")
days = options.get("days")
#
bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml3')
bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids')
if not os.path.exists(bibxmldir):
os.makedirs(bibxmldir)
#
@ -75,19 +75,21 @@ class Command(BaseCommand):
for e in doc_events:
self.mutter('%s %s' % (e.time, e.doc.name))
try:
e.doc.date = e.time.date()
doc = e.doc
if e.rev != doc.rev:
for h in doc.history_set.order_by("-time"):
if e.rev == h.rev:
doc = h
break
doc.date = e.time.date()
ref_text = '%s' % render_to_string('doc/bibxml.xml', {'name':doc.name, 'doc': doc, 'doc_bibtype':'I-D'})
if e.rev == e.doc.rev:
ref_file_name = os.path.join(bibxmldir, 'reference.I-D.%s.xml' % (doc.name[6:], ))
self.write(ref_file_name, ref_text)
else:
self.note("Skipping %s; outdated revision: %s" % (os.path.basename(ref_file_name), e.rev))
# if e.rev == e.doc.rev:
# for name in (doc.name, doc.name[6:]):
# ref_file_name = os.path.join(bibxmldir, 'reference.I-D.%s.xml' % (name, ))
# self.write(ref_file_name, ref_text)
# for name in (doc.name, doc.name[6:]):
# ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (name, doc.rev))
# self.write(ref_rev_file_name, ref_text)
ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (doc.name, doc.rev))
self.write(ref_rev_file_name, ref_text)
except Exception as ee:

4
ietf/settings.py
View file

@ -1016,8 +1016,8 @@ HTPASSWD_FILE = "/www/htpasswd"
# Generation of pdf files
GHOSTSCRIPT_COMMAND = "/usr/bin/gs"
# Generation of bibxml files for xml2rfc
BIBXML_BASE_PATH = '/a/www/ietf-ftp/xml2rfc'
# Generation of bibxml files (currently only for internet drafts)
BIBXML_BASE_PATH = '/a/ietfdata/derived/bibxml'
# Timezone files for iCalendar
TZDATA_ICS_PATH = BASE_DIR + '/../vzic/zoneinfo/'

21
ietf/submit/tests.py
View file

@ -50,6 +50,7 @@ class BaseSubmitTestCase(TestCase):
'SUBMIT_YANG_DRAFT_MODEL_DIR',
'SUBMIT_YANG_IANA_MODEL_DIR',
'SUBMIT_YANG_CATALOG_DIR',
'BIBXML_BASE_PATH',
]
def setUp(self):
@ -59,6 +60,7 @@ class BaseSubmitTestCase(TestCase):
# old drafts may not be moved out of the way properly.
self.saved_repository_path = settings.IDSUBMIT_REPOSITORY_PATH
settings.IDSUBMIT_REPOSITORY_PATH = settings.INTERNET_DRAFT_PATH
os.mkdir(os.path.join(settings.BIBXML_BASE_PATH,'bibxml-ids'))
def tearDown(self):
settings.IDSUBMIT_REPOSITORY_PATH = self.saved_repository_path
@ -256,6 +258,16 @@ class SubmitTests(BaseSubmitTestCase):
return confirmation_url
def verify_bibxml_ids_creation(self, draft):
# for name in (draft.name, draft.name[6:]):
# ref_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s.xml' % (name, ))
# self.assertTrue(os.path.exists(ref_file_name))
# ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (name, draft.rev ))
# self.assertTrue(os.path.exists(ref_rev_file_name))
ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev ))
self.assertTrue(os.path.exists(ref_rev_file_name))
def submit_new_wg(self, formats):
# submit new -> supply submitter info -> approve
GroupFactory(type_id='wg',acronym='ames')
@ -374,6 +386,8 @@ class SubmitTests(BaseSubmitTestCase):
self.assertContains(r, 'Yang Validation')
self.assertContains(r, 'WG Document')
self.verify_bibxml_ids_creation(draft)
def test_submit_new_wg_txt(self):
self.submit_new_wg(["txt"])
@ -685,6 +699,7 @@ class SubmitTests(BaseSubmitTestCase):
self.assertContains(r, draft.title)
# Check submission settings
self.assertEqual(draft.submission().xml_version, "3" if 'xml' in formats else None)
self.verify_bibxml_ids_creation(draft)
def test_submit_existing_txt(self):
self.submit_existing(["txt"])
@ -839,6 +854,7 @@ class SubmitTests(BaseSubmitTestCase):
new_revision = draft.latest_event()
self.assertEqual(new_revision.type, "new_revision")
self.assertEqual(new_revision.by.name, "Submitter Name")
self.verify_bibxml_ids_creation(draft)
def test_submit_new_individual_txt(self):
self.submit_new_individual(["txt"])
@ -879,6 +895,7 @@ class SubmitTests(BaseSubmitTestCase):
self.assertEqual(docauth.person, author)
self.assertEqual(docauth.affiliation, '')
self.assertEqual(docauth.country, '')
self.verify_bibxml_ids_creation(doc)
def test_submit_new_draft_no_org_or_address_txt(self):
self.submit_new_draft_no_org_or_address(['txt'])
@ -1015,6 +1032,7 @@ class SubmitTests(BaseSubmitTestCase):
# Check submission settings
self.assertEqual(draft.submission().xml_version, "3" if 'xml' in formats else None)
self.verify_bibxml_ids_creation(draft)
def test_submit_new_logged_in_txt(self):
self.submit_new_individual_logged_in(["txt"])
@ -1058,6 +1076,7 @@ class SubmitTests(BaseSubmitTestCase):
[str(r) for r in resources],
)
self._assert_extresource_change_event(draft, is_present=True)
self.verify_bibxml_ids_creation(draft)
def test_submit_update_individual(self):
IndividualDraftFactory(name='draft-ietf-random-thing', states=[('draft','rfc')], other_aliases=['rfc9999',], pages=5)
@ -1115,6 +1134,7 @@ class SubmitTests(BaseSubmitTestCase):
self.assertContains(r, draft.name)
self.assertContains(r, draft.title)
self._assert_extresource_change_event(draft, is_present=False)
self.verify_bibxml_ids_creation(draft)
def submit_existing_with_extresources(self, group_type, stream_type='ietf'):
"""Submit a draft with external resources
@ -1390,6 +1410,7 @@ class SubmitTests(BaseSubmitTestCase):
draft = Document.objects.get(docalias__name=name)
self.assertEqual(draft.rev, rev)
self.assertEqual(draft.docextresource_set.count(), 0)
self.verify_bibxml_ids_creation(draft)
def test_search_for_submission_and_edit_as_secretariat(self):
# submit -> edit

15
ietf/submit/utils.py
View file

@ -17,6 +17,7 @@ from django.core.validators import validate_email
from django.db import transaction
from django.http import HttpRequest # pyflakes:ignore
from django.utils.module_loading import import_string
from django.template.loader import render_to_string
import debug # pyflakes:ignore
@ -449,6 +450,20 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc):
submission.save()
create_submission_event(request, submission, approved_subm_desc)
# Create bibxml-ids entry
ref_text = '%s' % render_to_string('doc/bibxml.xml', {'name':draft.name, 'doc': draft, 'doc_bibtype':'I-D'})
# for name in (draft.name, draft.name[6:]):
# ref_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s.xml' % (name, ))
# with io.open(ref_file_name, "w", encoding='utf-8') as f:
# f.write(ref_text)
# ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (name, draft.rev ))
# with io.open(ref_rev_file_name, "w", encoding='utf-8') as f:
# f.write(ref_text)
ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev ))
with io.open(ref_rev_file_name, "w", encoding='utf-8') as f:
f.write(ref_text)
log.log(f"{submission.name}: done")