From a3f59ee9a24cb746136dd1e0e63438a32132d2d2 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Fri, 20 Sep 2019 14:24:44 +0000 Subject: [PATCH] Added conversion of version 3 XML draft submissions to v3 HTML, in addition to text. Added a link to HTML versions (when present) to the document status page, and renamed the link to the htmlized version to 'htmlized'. Converted the submission XML test-file to version 3. Added test code to check that HTML is generated for version 3 XML submissions. - Legacy-Id: 16737 --- ietf/doc/views_doc.py | 10 ++++----- ietf/settings.py | 4 ++++ ietf/submit/forms.py | 6 +++++- ietf/submit/test_submission.xml | 38 +++++++++++++++------------------ ietf/submit/tests.py | 5 ++++- ietf/submit/utils.py | 36 +++++++++++++++++++++++-------- 6 files changed, 62 insertions(+), 37 deletions(-) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 1aa240af2..93e80d663 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -206,7 +206,7 @@ def document_main(request, name, rev=None): # file types base_path = os.path.join(settings.RFC_PATH, name + ".") - possible_types = ["txt", "pdf", "ps"] + possible_types = settings.RFC_FILE_TYPES found_types = [t for t in possible_types if os.path.exists(base_path + t)] base = "https://www.rfc-editor.org/rfc/" @@ -220,7 +220,7 @@ def document_main(request, name, rev=None): file_urls.append(("pdf", base + "pdfrfc/" + name + ".txt.pdf")) if "txt" in found_types: - file_urls.append(("html", settings.TOOLS_ID_HTML_URL + name)) + file_urls.append(("htmlized", settings.TOOLS_ID_HTML_URL + name)) if doc.tags.filter(slug="errata"): file_urls.append(("with errata", settings.RFC_EDITOR_INLINE_ERRATA_URL.format(rfc_number=rfc_number))) @@ -236,8 +236,8 @@ def document_main(request, name, rev=None): # file types base_path = os.path.join(settings.INTERNET_DRAFT_PATH, doc.name + "-" + doc.rev + ".") - possible_types = ["pdf", "xml", "ps"] - found_types = ["txt"] + [t for t in possible_types if os.path.exists(base_path + t)] + possible_types = settings.IDSUBMIT_FILE_TYPES + found_types = [t for t in possible_types if os.path.exists(base_path + t)] if not snapshot and doc.get_state_slug() == "active": base = settings.IETF_ID_URL @@ -251,7 +251,7 @@ def document_main(request, name, rev=None): if "pdf" not in found_types: file_urls.append(("pdf", settings.TOOLS_ID_PDF_URL + doc.name + "-" + doc.rev + ".pdf")) - file_urls.append(("html", settings.TOOLS_ID_HTML_URL + doc.name + "-" + doc.rev)) + file_urls.append(("htmlized", settings.TOOLS_ID_HTML_URL + doc.name + "-" + doc.rev)) # latest revision latest_revision = doc.latest_event(NewRevisionDocEvent, type="new_revision") diff --git a/ietf/settings.py b/ietf/settings.py index 70f241282..dc968168d 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -782,13 +782,17 @@ IDSUBMIT_MANUAL_STAGING_DIR = '/tmp/' IDSUBMIT_FILE_TYPES = ( 'txt', + 'html', 'xml', 'pdf', 'ps', ) +RFC_FILE_TYPES = IDSUBMIT_FILE_TYPES + IDSUBMIT_MAX_DRAFT_SIZE = { 'txt': 2*1024*1024, # Max size of txt draft file in bytes 'xml': 3*1024*1024, # Max size of xml draft file in bytes + 'html': 4*1024*1024, 'pdf': 6*1024*1024, 'ps' : 6*1024*1024, } diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index 8e12ce676..b7829958f 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -352,12 +352,16 @@ class SubmissionBaseUploadForm(forms.Form): class SubmissionManualUploadForm(SubmissionBaseUploadForm): xml = forms.FileField(label='.xml format', required=False) # xml field with required=False instead of True txt = forms.FileField(label='.txt format', required=False) + # We won't permit html upload until we can verify that the content + # reasonably matches the text and/or xml upload. Till then, we generate + # html for version 3 xml submissions. + # html = forms.FileField(label='.html format', required=False) pdf = forms.FileField(label='.pdf format', required=False) ps = forms.FileField(label='.ps format', required=False) def __init__(self, request, *args, **kwargs): super(SubmissionManualUploadForm, self).__init__(request, *args, **kwargs) - self.formats = ['txt', 'pdf', 'xml', 'ps', ] + self.formats = settings.IDSUBMIT_FILE_TYPES self.base_formats = ['txt', 'xml', ] def clean_txt(self): diff --git a/ietf/submit/test_submission.xml b/ietf/submit/test_submission.xml index c147589b6..dc85ac900 100644 --- a/ietf/submit/test_submission.xml +++ b/ietf/submit/test_submission.xml @@ -1,12 +1,12 @@ - + - + %(title)s + Test Centre Inc. -
42 Some Road @@ -16,7 +16,7 @@ %(email)s
- + %(group)s @@ -24,18 +24,16 @@ - -
+
+ Introduction This document describes a protocol for testing tests.
-
-
- - file "ietf-yang-metadata@2016-08-05.yang" +
+ Yang + - ]]> - -
+]]>
- -
+
+ JSON example The JSON object should look like this: @@ -137,17 +132,18 @@ module ietf-yang-metadata { }
-
+
+ Security Considerations There are none.
-
+
+ IANA Considerations No new registrations for IANA.
- - + diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index d87d73885..4a7e55eff 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -64,6 +64,7 @@ def submission_file(name, rev, group, format, templatename, author=None, email=N expiration=(datetime.date.today() + datetime.timedelta(days=100)).strftime("%d %B, %Y"), year=year, month=datetime.date.today().strftime("%B"), + day=datetime.date.today().strftime("%d"), name="%s-%s" % (name, rev), group=group or "", author=author.ascii if ascii else author.name, @@ -151,6 +152,8 @@ class SubmitTests(TestCase): status_url = r["Location"] for format in formats: self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, format)))) + if format == 'xml': + self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, 'html')))) self.assertEqual(Submission.objects.filter(name=name).count(), 1) submission = Submission.objects.get(name=name) if len(submission.authors) != 1: @@ -1684,7 +1687,7 @@ class ApiSubmitTests(TestCase): self.assertContains(r, expected, status_code=400) def test_api_submit_failed_idnits(self): - r, author, name = self.post_submission('00', year="1900") + r, author, name = self.post_submission('00', year="2010") expected = "Document date must be within 3 days of submission date" self.assertContains(r, expected, status_code=400) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 37ddc1f2b..1b92bff09 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -528,16 +528,14 @@ def cancel_submission(submission): remove_submission_files(submission) def rename_submission_files(submission, prev_rev, new_rev): - from ietf.submit.forms import SubmissionManualUploadForm - for ext in list(SubmissionManualUploadForm.base_fields.keys()): + for ext in settings.IDSUBMIT_FILE_TYPES: source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, prev_rev, ext)) dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, new_rev, ext)) if os.path.exists(source): os.rename(source, dest) def move_files_to_repository(submission): - from ietf.submit.forms import SubmissionManualUploadForm - for ext in list(SubmissionManualUploadForm.base_fields.keys()): + for ext in settings.IDSUBMIT_FILE_TYPES: source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext)) dest = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext)) if os.path.exists(source): @@ -622,18 +620,24 @@ def get_draft_meta(form, saved_files): file_name = saved_files abstract = None file_size = None + xml2rfc.log.write_out = open(os.devnull, "w") + xml2rfc.log.write_err = open(os.devnull, "w") if form.cleaned_data['xml']: + try: + xmlroot = form.xmltree.getroot() + xml_version = xmlroot.get('version', '2') + if xml_version == '3': + prep = xml2rfc.PrepToolWriter(form.xmltree, quiet=True) + form.xmltree.tree = prep.prep() + except Exception as e: + raise ValidationError("Error from xml2rfc (prep): %s" % e) if not ('txt' in form.cleaned_data and form.cleaned_data['txt']): file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (form.filename, form.revision)) try: - xmlroot = form.xmltree.getroot() - xml_version = xmlroot.get('version', '2') if xml_version != '3': pagedwriter = xml2rfc.PaginatedTextRfcWriter(form.xmltree, quiet=True) pagedwriter.write(file_name['txt']) else: - prep = xml2rfc.PrepToolWriter(form.xmltree, quiet=True) - form.xmltree.tree = prep.prep() writer = xml2rfc.TextWriter(form.xmltree, quiet=True) writer.write(file_name['txt']) log.log("In %s: xml2rfc %s generated %s from %s (version %s)" % @@ -643,8 +647,22 @@ def get_draft_meta(form, saved_files): os.path.basename(file_name['xml']), xml_version)) except Exception as e: - raise ValidationError("Error from xml2rfc: %s" % e) + raise ValidationError("Error from xml2rfc (text): %s" % e) file_size = os.stat(file_name['txt']).st_size + if xml_version == '3': + try: + file_name['html'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.html' % (form.filename, form.revision)) + writer = xml2rfc.HtmlWriter(form.xmltree, quiet=True) + writer.write(file_name['html']) + form.file_types.append('.html') + log.log("In %s: xml2rfc %s generated %s from %s (version %s)" % + ( os.path.dirname(file_name['xml']), + xml2rfc.__version__, + os.path.basename(file_name['html']), + os.path.basename(file_name['xml']), + xml_version)) + except Exception as e: + raise ValidationError("Error from xml2rfc (html): %s" % e) # Some meta-information, such as the page-count, can only # be retrieved from the generated text file. Provide a # parsed draft object to get at that kind of information.