diff --git a/ietf/settings.py b/ietf/settings.py index 5b138e7f6..b4866275d 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -432,18 +432,23 @@ IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_01 = 13 IDSUBMIT_DEFAULT_CUTOFF_TIME_UTC = datetime.timedelta(hours=23, minutes=59, seconds=59) IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21) -MEETING_MATERIALS_SUBMISSION_START_DAYS = -90 -MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26 -MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 50 - -INTERNET_DRAFT_DAYS_TO_EXPIRE = 185 - IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/' IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/' IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits' -IDSUBMIT_MAX_PLAIN_DRAFT_SIZE = 6291456 # Max size of the txt draft in bytes +IDSUBMIT_FILE_TYPES = ( + 'txt', + 'xml', + 'pdf', + 'ps', +) +IDSUBMIT_MAX_DRAFT_SIZE = { + 'txt': 6*1024*1024, # Max size of txt draft file in bytes + 'xml': 10*1024*1024, # Max size of xml draft file in bytes + 'pdf': 10*1024*1024, + 'ps' : 10*1024*1024, +} IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME = 20 IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE = 50 # in MB @@ -454,6 +459,14 @@ IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE = 450 # in MB IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000 IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB +XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/" + +MEETING_MATERIALS_SUBMISSION_START_DAYS = -90 +MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26 +MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 50 + +INTERNET_DRAFT_DAYS_TO_EXPIRE = 185 + DOT_BINARY = '/usr/bin/dot' UNFLATTEN_BINARY= '/usr/bin/unflatten' PS2PDF_BINARY = '/usr/bin/ps2pdf' diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index e107bdf22..f436fe750 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -1,6 +1,9 @@ import os +import re import datetime import pytz +import xml2rfc +import tempfile from django import forms from django.conf import settings @@ -23,14 +26,14 @@ from ietf.submit.parsers.xml_parser import XMLParser from ietf.utils.draft import Draft -class UploadForm(forms.Form): - txt = forms.FileField(label=u'.txt format', required=True) +class SubmissionUploadForm(forms.Form): + txt = forms.FileField(label=u'.txt format', required=False) xml = forms.FileField(label=u'.xml format', required=False) pdf = forms.FileField(label=u'.pdf format', required=False) - ps = forms.FileField(label=u'.ps format', required=False) + ps = forms.FileField(label=u'.ps format', required=False) def __init__(self, request, *args, **kwargs): - super(UploadForm, self).__init__(*args, **kwargs) + super(SubmissionUploadForm, self).__init__(*args, **kwargs) self.remote_ip = request.META.get('REMOTE_ADDR', None) @@ -41,7 +44,13 @@ class UploadForm(forms.Form): self.set_cutoff_warnings() self.group = None + self.filename = None + self.revision = None + self.title = None + self.abstract = None + self.authors = [] self.parsed_draft = None + self.file_types = [] def set_cutoff_warnings(self): now = datetime.datetime.now(pytz.utc) @@ -93,7 +102,6 @@ class UploadForm(forms.Form): return f - def clean_txt(self): return self.clean_file("txt", PlainParser) @@ -101,7 +109,7 @@ class UploadForm(forms.Form): return self.clean_file("pdf", PDFParser) def clean_ps(self): - return self.clean_file("ps", PSParser) + return self.clean_file("ps", PSParser) def clean_xml(self): return self.clean_file("xml", XMLParser) @@ -116,37 +124,100 @@ class UploadForm(forms.Form): if not os.path.exists(getattr(settings, s)): raise forms.ValidationError('%s defined in settings.py does not exist' % s) + for ext in ['txt', 'pdf', 'xml', 'ps']: + f = self.cleaned_data.get(ext, None) + if not f: + continue + self.file_types.append('.%s' % ext) + if not ('.txt' in self.file_types or '.xml' in self.file_types): + raise forms.ValidationError('You must submit either a .txt or an .xml file; didn\'t find either.') + + #debug.show('self.cleaned_data["xml"]') + if self.cleaned_data.get('xml'): + #if not self.cleaned_data.get('txt'): + xml_file = self.cleaned_data.get('xml') + tfh, tfn = tempfile.mkstemp(suffix='.xml') + try: + # We need to write the xml file to disk in order to hand it + # over to the xml parser. XXX FIXME: investigate updating + # xml2rfc to be able to work with file handles to in-memory + # files. + with open(tfn, 'wb+') as tf: + for chunk in xml_file.chunks(): + tf.write(chunk) + os.environ["XML_LIBRARY"] = settings.XML_LIBRARY + parser = xml2rfc.XmlRfcParser(tfn, quiet=True) + self.xmltree = parser.parse() + ok, errors = self.xmltree.validate() + if not ok: + raise forms.ValidationError(errors) + self.xmlroot = self.xmltree.getroot() + draftname = self.xmlroot.attrib.get('docName') + revmatch = re.search("-[0-9][0-9]$", draftname) + if revmatch: + self.revision = draftname[-2:] + self.filename = draftname[:-3] + else: + self.revision = None + self.filename = draftname + self.title = self.xmlroot.find('front/title').text + self.abstract = self.xmlroot.find('front/abstract').text + self.author_list = [] + author_info = self.xmlroot.findall('front/author') + for author in author_info: + author_dict = dict( + company = author.find('organization').text, + last_name = author.attrib.get('surname'), + full_name = author.attrib.get('fullname'), + email = author.find('address/email').text, + ) + self.author_list.append(author_dict) + line = "%(full_name)s <%(email)s>" % author_dict + self.authors.append(line) + except Exception as e: + raise forms.ValidationError("Exception: %s" % e) + finally: + os.close(tfh) + os.unlink(tfn) + if self.cleaned_data.get('txt'): # try to parse it txt_file = self.cleaned_data['txt'] txt_file.seek(0) self.parsed_draft = Draft(txt_file.read(), txt_file.name) + self.filename = self.parsed_draft.filename + self.revision = self.parsed_draft.revision + self.title = self.parsed_draft.get_title() txt_file.seek(0) - if not self.parsed_draft.filename: - raise forms.ValidationError("Draft parser could not extract a valid draft name from the .txt file") + if not self.filename: + raise forms.ValidationError("Draft parser could not extract a valid draft name from the upload") - if not self.parsed_draft.get_title(): - raise forms.ValidationError("Draft parser could not extract a valid title from the .txt file") + if not self.revision: + raise forms.ValidationError("Draft parser could not extract a valid draft revision from the upload") + if not self.title: + raise forms.ValidationError("Draft parser could not extract a valid title from the upload") + + if self.cleaned_data.get('txt') or self.cleaned_data.get('xml'): # check group self.group = self.deduce_group() # check existing - existing = Submission.objects.filter(name=self.parsed_draft.filename, rev=self.parsed_draft.revision).exclude(state__in=("posted", "cancel")) + existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel")) if existing: raise forms.ValidationError(mark_safe('Submission with same name and revision is currently being processed. Check the status here.' % urlreverse("submit_submission_status", kwargs={ 'submission_id': existing[0].pk }))) # cut-off - if self.parsed_draft.revision == '00' and self.in_first_cut_off: + if self.revision == '00' and self.in_first_cut_off: raise forms.ValidationError(mark_safe(self.cutoff_warning)) # check thresholds today = datetime.date.today() self.check_submissions_tresholds( - "for the draft %s" % self.parsed_draft.filename, - dict(name=self.parsed_draft.filename, rev=self.parsed_draft.revision, submission_date=today), + "for the draft %s" % self.filename, + dict(name=self.filename, rev=self.revision, submission_date=today), settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE, ) self.check_submissions_tresholds( @@ -166,19 +237,19 @@ class UploadForm(forms.Form): settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE, ) - return super(UploadForm, self).clean() + return super(SubmissionUploadForm, self).clean() def check_submissions_tresholds(self, which, filter_kwargs, max_amount, max_size): submissions = Submission.objects.filter(**filter_kwargs) if len(submissions) > max_amount: raise forms.ValidationError("Max submissions %s has been reached for today (maximum is %s submissions)." % (which, max_amount)) - if sum(s.file_size for s in submissions) > max_size * 1024 * 1024: + if sum(s.file_size for s in submissions if s.file_size) > max_size * 1024 * 1024: raise forms.ValidationError("Max uploaded amount %s has been reached for today (maximum is %s MB)." % (which, max_size)) def deduce_group(self): """Figure out group from name or previously submitted draft, returns None if individual.""" - name = self.parsed_draft.filename + name = self.filename existing_draft = Document.objects.filter(name=name, type="draft") if existing_draft: group = existing_draft[0].group diff --git a/ietf/submit/parsers/base.py b/ietf/submit/parsers/base.py index 53c0afa38..0d423d321 100644 --- a/ietf/submit/parsers/base.py +++ b/ietf/submit/parsers/base.py @@ -1,4 +1,11 @@ +import os import re +import magic +import datetime +import debug # pyflakes:ignore + +from django.conf import settings +from django.template.defaultfilters import filesizeformat class MetaData(object): rev = None @@ -42,6 +49,8 @@ class FileParser(object): # no other file parsing is recommended def critical_parse(self): self.parse_invalid_chars_in_filename() + self.parse_max_size(); + self.parsed_info.metadata.submission_date = datetime.date.today() return self.parsed_info def parse_invalid_chars_in_filename(self): @@ -50,3 +59,22 @@ class FileParser(object): chars = regexp.findall(name) if chars: self.parsed_info.add_error('Invalid characters were found in the name of the file which was just submitted: %s' % ', '.join(set(chars))) + + def parse_max_size(self): + __, ext = os.path.splitext(self.fd.name) + ext = ext.lstrip('.') + max_size = settings.IDSUBMIT_MAX_DRAFT_SIZE[ext] + if self.fd.size > max_size: + self.parsed_info.add_error('File size is larger than the permitted maximum of %s' % filesizeformat(max_size)) + self.parsed_info.metadata.file_size = self.fd.size + + def parse_filename_extension(self, ext): + if not self.fd.name.lower().endswith('.'+ext): + self.parsed_info.add_error('Expected the %s file to have extension ".%s", found "%s"' % (ext.upper(), ext, self.fd.name)) + + def parse_file_type(self, ext, expected): + self.fd.file.seek(0) + content = self.fd.file.read(4096) + mimetype = magic.from_buffer(content, mime=True) + if not mimetype == expected: + self.parsed_info.add_error('Expected an %s file of type "%s", found one of type "%s"' % (expected, mimetype)) diff --git a/ietf/submit/parsers/pdf_parser.py b/ietf/submit/parsers/pdf_parser.py index 88a58fc25..9a5213d6e 100644 --- a/ietf/submit/parsers/pdf_parser.py +++ b/ietf/submit/parsers/pdf_parser.py @@ -7,9 +7,6 @@ class PDFParser(FileParser): # no other file parsing is recommended def critical_parse(self): super(PDFParser, self).critical_parse() - self.parse_filename_extension() + self.parse_filename_extension('pdf') + self.parse_file_type('pdf', 'application/pdf') return self.parsed_info - - def parse_filename_extension(self): - if not self.fd.name.endswith('.pdf'): - self.parsed_info.add_error('Format of this document must be PDF') diff --git a/ietf/submit/parsers/plain_parser.py b/ietf/submit/parsers/plain_parser.py index 8ad6c4719..042d38df1 100644 --- a/ietf/submit/parsers/plain_parser.py +++ b/ietf/submit/parsers/plain_parser.py @@ -1,8 +1,5 @@ -import datetime import re -from django.conf import settings -from django.template.defaultfilters import filesizeformat from ietf.submit.parsers.base import FileParser @@ -15,17 +12,12 @@ class PlainParser(FileParser): # no other file parsing is recommended def critical_parse(self): super(PlainParser, self).critical_parse() - self.parse_max_size() + self.parse_filename_extension('txt') + self.parse_file_type('txt', 'text/plain') self.parse_file_charset() self.parse_name() return self.parsed_info - def parse_max_size(self): - if self.fd.size > settings.IDSUBMIT_MAX_PLAIN_DRAFT_SIZE: - self.parsed_info.add_error('File size is larger than %s' % filesizeformat(settings.IDSUBMIT_MAX_PLAIN_DRAFT_SIZE)) - self.parsed_info.metadata.file_size = self.fd.size - self.parsed_info.metadata.submission_date = datetime.date.today() - def parse_file_charset(self): import magic self.fd.file.seek(0) diff --git a/ietf/submit/parsers/ps_parser.py b/ietf/submit/parsers/ps_parser.py index 084a1329a..aff71da2d 100644 --- a/ietf/submit/parsers/ps_parser.py +++ b/ietf/submit/parsers/ps_parser.py @@ -7,9 +7,6 @@ class PSParser(FileParser): # no other file parsing is recommended def critical_parse(self): super(PSParser, self).critical_parse() - self.parse_filename_extension() + self.parse_filename_extension('ps') + self.parse_file_type('ps', 'application/postscript') return self.parsed_info - - def parse_filename_extension(self): - if not self.fd.name.endswith('.ps'): - self.parsed_info.add_error('Format of this document must be PS') diff --git a/ietf/submit/parsers/xml_parser.py b/ietf/submit/parsers/xml_parser.py index 243acb544..b584b1b46 100644 --- a/ietf/submit/parsers/xml_parser.py +++ b/ietf/submit/parsers/xml_parser.py @@ -7,9 +7,7 @@ class XMLParser(FileParser): # no other file parsing is recommended def critical_parse(self): super(XMLParser, self).critical_parse() - self.parse_filename_extension() + self.parse_filename_extension('xml') + self.parse_file_type('xml', 'application/xml') return self.parsed_info - - def parse_filename_extension(self): - if not self.fd.name.endswith('.xml'): - self.parsed_info.add_error('Format of this document must be XML') + diff --git a/ietf/submit/test_submission.pdf b/ietf/submit/test_submission.pdf new file mode 100644 index 000000000..7948a9f47 --- /dev/null +++ b/ietf/submit/test_submission.pdf @@ -0,0 +1,2 @@ +%%PDF-1.5 +This is PDF diff --git a/ietf/submit/test_submission.ps b/ietf/submit/test_submission.ps new file mode 100644 index 000000000..18c3e4192 --- /dev/null +++ b/ietf/submit/test_submission.ps @@ -0,0 +1,2 @@ +%%!PS-Adobe-2.0 +This is PostScript diff --git a/ietf/submit/test_submission.txt b/ietf/submit/test_submission.txt index c5017a4d1..9f16000a5 100644 --- a/ietf/submit/test_submission.txt +++ b/ietf/submit/test_submission.txt @@ -1,18 +1,21 @@ -Informational Author Name -Internet-Draft Test Center Inc. -Intended status: Informational %(date)s -Expires: %(expire)s - - Testing tests - %(name)s + +Network Working Group A. Name +Internet-Draft Test Centre Inc. +Intended status: Informational %(month)s %(year)s +Expires: %(expiration)s + + + Testing Tests + %(name)s + Abstract This document describes how to test tests. -Status of this Memo +Status of This Memo This Internet-Draft is submitted in full conformance with the provisions of BCP 78 and BCP 79. @@ -27,7 +30,7 @@ Status of this Memo time. It is inappropriate to use Internet-Drafts as reference material or to cite them other than as "work in progress." - This Internet-Draft will expire on %(expire)s. + This Internet-Draft will expire on %(expiration)s. Copyright Notice @@ -40,67 +43,49 @@ Copyright Notice publication of this document. Please review these documents carefully, as they describe your rights and restrictions with respect to this document. Code Components extracted from this document must - - -Name Expires %(expire)s [Page 1] - -Internet-Draft Testing tests %(month_year)s - include Simplified BSD License text as described in Section 4.e of the Trust Legal Provisions and are provided without warranty as described in the Simplified BSD License. - This document may contain material from IETF Documents or IETF - Contributions published or made publicly available before November - 10, 2008. The person(s) controlling the copyright in some of this - material may not have granted the IETF Trust the right to allow - modifications of such material outside the IETF Standards Process. - Without obtaining an adequate license from the person(s) controlling - the copyright in such materials, this document may not be modified - outside the IETF Standards Process, and derivative works of it may - not be created outside the IETF Standards Process, except to format - it for publication as an RFC or to translate it into languages other - than English. + + + + + + +Name Expires %(expiration)s [Page 1] + +Internet-Draft Testing Tests %(month)s %(year)s Table of Contents - 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 2. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 4 - 3. Security Considerations . . . . . . . . . . . . . . . . . . . 4 - - -Name Expires %(expire)s [Page 2] - -Internet-Draft Testing tests %(month_year)s + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2 + 2. Security Considerations . . . . . . . . . . . . . . . . . . . 2 + 3. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 2 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . . 2 1. Introduction This document describes a protocol for testing tests. -Name Expires %(expire)s [Page 3] - -Internet-Draft Testing tests %(month_year)s +2. Security Considerations -2. Security Considerations + There are none. - There are none. +3. IANA Considerations + No new registrations for IANA. -3. IANA Considerations +Author's Address - No new registrations for IANA. + Author Name + Test Centre Inc. + 42 Some Road + Some Where 12345 + UK - -Authors' Addresses - - Author Name - Test Center Inc. - 42 Some Road - Some Where 12345 - US - - Email: author@example.com + Email: author@example.com @@ -109,4 +94,19 @@ Authors' Addresses -Name Expires %(expire)s [Page 4] + + + + + + + + + + + + + + + +Name Expires %(expiration)s [Page 2] diff --git a/ietf/submit/test_submission.xml b/ietf/submit/test_submission.xml new file mode 100644 index 000000000..63cbcff1e --- /dev/null +++ b/ietf/submit/test_submission.xml @@ -0,0 +1,47 @@ + + + + + + Testing Tests + + Test Centre Inc. + +
+ + 42 Some Road + Some Where 12345 + UK + + author@example.com +
+ + + %(group)s + + + This document describes how to test tests. + + + + + +
+ + This document describes a protocol for testing tests. + +
+
+ + There are none. + +
+
+ + No new registrations for IANA. + +
+ + + + diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 0fb41a67b..60f35596f 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -9,6 +9,8 @@ from django.core.urlresolvers import reverse as urlreverse from StringIO import StringIO from pyquery import PyQuery +import debug # pyflakes:ignore + from ietf.utils.test_utils import login_testing_unauthorized from ietf.utils.test_data import make_test_data from ietf.utils.mail import outbox @@ -39,25 +41,26 @@ class SubmitTests(TestCase): shutil.rmtree(self.repository_dir) shutil.rmtree(self.archive_dir) - def submission_txt_file(self, name, rev): + def submission_file(self, name, rev, group, format, templatename): # construct appropriate text draft - f = open(os.path.join(settings.BASE_DIR, "submit", "test_submission.txt")) + f = open(os.path.join(settings.BASE_DIR, "submit", templatename)) template = f.read() f.close() submission_text = template % dict( date=datetime.date.today().strftime("%d %B %Y"), - expire=(datetime.date.today() + datetime.timedelta(days=100)).strftime("%Y-%m-%d"), + expiration=(datetime.date.today() + datetime.timedelta(days=100)).strftime("%d %B, %Y"), year=datetime.date.today().strftime("%Y"), - month_year=datetime.date.today().strftime("%B, %Y"), + month=datetime.date.today().strftime("%B"), name="%s-%s" % (name, rev), + group=group or "", ) - txt_file = StringIO(str(submission_text)) - txt_file.name = "somename.txt" - return txt_file + file = StringIO(str(submission_text)) + file.name = "%s-%s.%s" % (name, rev, format) + return file - def do_submission(self, name, rev): + def do_submission(self, name, rev, group=None, formats=["txt",]): # break early in case of missing configuration self.assertTrue(os.path.exists(settings.IDSUBMIT_IDNITS_BINARY)) @@ -67,15 +70,23 @@ class SubmitTests(TestCase): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q('input[type=file][name=txt]')), 1) + self.assertEqual(len(q('input[type=file][name=xml]')), 1) # submit - txt_file = self.submission_txt_file(name, rev) + files = {} + for format in formats: + files[format] = self.submission_file(name, rev, group, format, "test_submission.%s" % format) + + r = self.client.post(url, files) + if r.status_code != 302: + q = PyQuery(r.content) + print(q('div.has-error span.help-block div').text) - r = self.client.post(url, - dict(txt=txt_file)) self.assertEqual(r.status_code, 302) + status_url = r["Location"] - self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev)))) + for format in formats: + self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.%s" % (name, rev, format)))) self.assertEqual(Submission.objects.filter(name=name).count(), 1) submission = Submission.objects.get(name=name) self.assertTrue(re.search('\s+Summary:\s+0\s+errors|No nits found', submission.idnits_message)) @@ -120,7 +131,7 @@ class SubmitTests(TestCase): return confirm_url - def test_submit_new_wg(self): + def submit_new_wg(self, formats): # submit new -> supply submitter info -> approve draft = make_test_data() @@ -147,8 +158,9 @@ class SubmitTests(TestCase): name = "draft-ietf-mars-testing-tests" rev = "00" + group = "mars" - status_url = self.do_submission(name, rev) + status_url = self.do_submission(name, rev, group, formats) # supply submitter info, then draft should be in and ready for approval mailbox_before = len(outbox) @@ -211,7 +223,16 @@ class SubmitTests(TestCase): self.assertTrue("ameschairman" in outbox[-1]["To"].lower()) self.assertTrue("marschairman" in outbox[-1]["To"].lower()) - def test_submit_existing(self): + def test_submit_new_wg_txt(self): + self.submit_new_wg(["txt"]) + + def text_submit_new_wg_xml(self): + self.submit_new_wg(["xml"]) + + def text_submit_new_wg_txt_xml(self): + self.submit_new_wg(["txt", "xml"]) + + def submit_existing(self, formats): # submit new revision of existing -> supply submitter info -> prev authors confirm draft = make_test_data() prev_author = draft.documentauthor_set.all()[0] @@ -241,13 +262,14 @@ class SubmitTests(TestCase): name = draft.name rev = "%02d" % (int(draft.rev) + 1) + group = draft.group # write the old draft in a file so we can check it's moved away old_rev = draft.rev with open(os.path.join(self.repository_dir, "%s-%s.txt" % (name, old_rev)), 'w') as f: f.write("a" * 2000) - status_url = self.do_submission(name, rev) + status_url = self.do_submission(name, rev, group, formats) # supply submitter info, then previous authors get a confirmation email mailbox_before = len(outbox) @@ -310,14 +332,24 @@ class SubmitTests(TestCase): self.assertTrue(name in unicode(outbox[-1])) self.assertTrue("mars" in unicode(outbox[-1])) - def test_submit_new_individual(self): + def test_submit_existing_txt(self): + self.submit_existing(["txt"]) + + def test_submit_existing_xml(self): + self.submit_existing(["xml"]) + + def test_submit_existing_txt_xml(self): + self.submit_existing(["txt", "xml"]) + + def submit_new_individual(self, formats): # submit new -> supply submitter info -> confirm draft = make_test_data() name = "draft-authorname-testing-tests" rev = "00" + group = None - status_url = self.do_submission(name, rev) + status_url = self.do_submission(name, rev, group, formats) # supply submitter info, then draft should be be ready for email auth mailbox_before = len(outbox) @@ -355,6 +387,15 @@ class SubmitTests(TestCase): self.assertEqual(new_revision.type, "new_revision") self.assertEqual(new_revision.by.name, "Submitter Name") + def test_submit_new_individual_txt(self): + self.submit_new_individual(["txt"]) + + def test_submit_new_individual_xml(self): + self.submit_new_individual(["xml"]) + + def test_submit_new_individual_txt_xml(self): + self.submit_new_individual(["txt", "xml"]) + def test_submit_new_wg_with_dash(self): make_test_data() @@ -579,35 +620,17 @@ class SubmitTests(TestCase): name = "draft-ietf-mars-testing-tests" rev = "00" + group = "mars" - txt_file = self.submission_txt_file(name, rev) - - # the checks for other file types are currently embarrassingly - # dumb, so don't bother constructing proper XML/PS/PDF draft - # files - xml_file = StringIO('\nThis is XML') - xml_file.name = "somename.xml" - - pdf_file = StringIO('%PDF-1.5\nThis is PDF') - pdf_file.name = "somename.pdf" - - ps_file = StringIO('%!PS-Adobe-2.0\nThis is PostScript') - ps_file.name = "somename.ps" - - r = self.client.post(urlreverse('submit_upload_submission'), dict( - txt=txt_file, - xml=xml_file, - pdf=pdf_file, - ps=ps_file, - )) - self.assertEqual(r.status_code, 302) + self.do_submission(name, rev, group, ["txt", "xml", "ps", "pdf"]) self.assertEqual(Submission.objects.filter(name=name).count(), 1) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev)))) self.assertTrue(name in open(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))).read()) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev)))) - self.assertTrue('This is XML' in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read()) + self.assertTrue(name in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read()) + self.assertTrue('' in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read()) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.pdf" % (name, rev)))) self.assertTrue('This is PDF' in open(os.path.join(self.staging_dir, u"%s-%s.pdf" % (name, rev))).read()) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.ps" % (name, rev)))) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index e1bbb9219..c15dd6839 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -353,21 +353,24 @@ def cancel_submission(submission): remove_submission_files(submission) def rename_submission_files(submission, prev_rev, new_rev): - for ext in submission.file_types.split(','): - source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, prev_rev, ext)) - dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, new_rev, ext)) - os.rename(source, dest) + from ietf.submit.forms import SubmissionUploadForm + for ext in SubmissionUploadForm.base_fields.keys(): + source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, prev_rev, ext)) + dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, new_rev, ext)) + if os.path.exists(source): + os.rename(source, dest) def move_files_to_repository(submission): - for ext in submission.file_types.split(','): - source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, submission.rev, ext)) - dest = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, '%s-%s%s' % (submission.name, submission.rev, ext)) + from ietf.submit.forms import SubmissionUploadForm + for ext in SubmissionUploadForm.base_fields.keys(): + source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext)) + dest = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext)) if os.path.exists(source): os.rename(source, dest) else: if os.path.exists(dest): log("Intended to move '%s' to '%s', but found source missing while destination exists.") - else: + elif ext in submission.file_types.split(','): raise ValueError("Intended to move '%s' to '%s', but found source and destination missing.") def remove_submission_files(submission): diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 8a6bc5688..a452309cc 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2007, All Rights Reserved import datetime import os +import xml2rfc from django.conf import settings from django.core.urlresolvers import reverse as urlreverse @@ -8,94 +9,127 @@ from django.core.validators import validate_email, ValidationError from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden from django.shortcuts import get_object_or_404, redirect, render +import debug # pyflakes:ignore + from ietf.doc.models import Document, DocAlias from ietf.doc.utils import prettify_std_name from ietf.group.models import Group from ietf.ietfauth.utils import has_role, role_required -from ietf.submit.forms import UploadForm, NameEmailForm, EditSubmissionForm, PreapprovalForm, ReplacesForm +from ietf.submit.forms import SubmissionUploadForm, NameEmailForm, EditSubmissionForm, PreapprovalForm, ReplacesForm from ietf.submit.mail import send_full_url, send_approval_request_to_group, send_submission_confirmation, submission_confirmation_email_list, send_manual_post_request from ietf.submit.models import Submission, Preapproval, DraftSubmissionStateName from ietf.submit.utils import approvable_submissions_for_user, preapprovals_for_user, recently_approved_by_user from ietf.submit.utils import check_idnits, found_idnits, validate_submission, create_submission_event from ietf.submit.utils import post_submission, cancel_submission, rename_submission_files from ietf.utils.accesstoken import generate_random_key, generate_access_token +from ietf.utils.draft import Draft + def upload_submission(request): if request.method == 'POST': try: - form = UploadForm(request, data=request.POST, files=request.FILES) + form = SubmissionUploadForm(request, data=request.POST, files=request.FILES) if form.is_valid(): - # save files - file_types = [] - for ext in ['txt', 'pdf', 'xml', 'ps']: + authors = [] + file_name = {} + abstract = None + file_size = None + for ext in form.fields.keys(): f = form.cleaned_data[ext] if not f: continue - file_types.append('.%s' % ext) - - draft = form.parsed_draft - - name = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (draft.filename, draft.revision, ext)) + + name = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (form.filename, form.revision, ext)) + file_name[ext] = name with open(name, 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) - # check idnits - text_path = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (draft.filename, draft.revision)) - idnits_message = check_idnits(text_path) - - # extract author lines - authors = [] - for author in draft.get_author_list(): - full_name, first_name, middle_initial, last_name, name_suffix, email, company = author - - line = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() - email = (email or "").strip() - - if email: + if form.cleaned_data['xml']: + if not form.cleaned_data['txt']: try: - validate_email(email) - except ValidationError: - email = "" + file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (form.filename, form.revision)) + pagedwriter = xml2rfc.PaginatedTextRfcWriter(form.xmltree, quiet=True) + pagedwriter.write(file_name['txt']) + file_size = os.stat(file_name['txt']).st_size + except Exception as e: + raise ValidationError("Exception: %s" % e) + # Some meta-information, such as the page-count, can only + # be retrieved from the generated text file. Provide a + # parsed draft object to get at that kind of information. + with open(file_name['txt']) as txt_file: + form.parsed_draft = Draft(txt_file.read(), txt_file.name) - if email: - line += u" <%s>" % email + else: + file_size = form.cleaned_data['txt'].size - authors.append(line) + if form.authors: + authors = form.authors + else: + # If we don't have an xml file, try to extract the + # relevant information from the text file + for author in form.parsed_draft.get_author_list(): + full_name, first_name, middle_initial, last_name, name_suffix, email, company = author + + line = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip() + email = (email or "").strip() + + if email: + try: + validate_email(email) + except ValidationError: + email = "" + + if email: + line += u" <%s>" % email + + authors.append(line) + + if form.abstract: + abstract = form.abstract + else: + abstract = form.parsed_draft.get_abstract() + + # check idnits + idnits_message = check_idnits(file_name['txt']) # save submission - submission = Submission.objects.create( - state=DraftSubmissionStateName.objects.get(slug="uploaded"), - remote_ip=form.remote_ip, - name=draft.filename, - group=form.group, - title=draft.get_title(), - abstract=draft.get_abstract(), - rev=draft.revision, - pages=draft.get_pagecount(), - authors="\n".join(authors), - note="", - first_two_pages=''.join(draft.pages[:2]), - file_size=form.cleaned_data['txt'].size, - file_types=','.join(file_types), - submission_date=datetime.date.today(), - document_date=draft.get_creation_date(), - replaces="", - idnits_message=idnits_message, - ) + try: + submission = Submission.objects.create( + state=DraftSubmissionStateName.objects.get(slug="uploaded"), + remote_ip=form.remote_ip, + name=form.filename, + group=form.group, + title=form.title, + abstract=abstract, + rev=form.revision, + pages=form.parsed_draft.get_pagecount(), + authors="\n".join(authors), + note="", + first_two_pages=''.join(form.parsed_draft.pages[:2]), + file_size=file_size, + file_types=','.join(form.file_types), + submission_date=datetime.date.today(), + document_date=form.parsed_draft.get_creation_date(), + replaces="", + idnits_message=idnits_message, + ) + except Exception as e: + import sys + sys.stderr.write("Exception: %s\n" % e) create_submission_event(request, submission, desc="Uploaded submission") return redirect("submit_submission_status_by_hash", submission_id=submission.pk, access_token=submission.access_token()) except IOError as e: if "read error" in str(e): # The server got an IOError when trying to read POST data - form = UploadForm(request=request) + form = SubmissionUploadForm(request=request) form._errors = {} form._errors["__all__"] = form.error_class(["There was a failure receiving the complete form data -- please try again."]) else: raise else: - form = UploadForm(request=request) + form = SubmissionUploadForm(request=request) return render(request, 'submit/upload_submission.html', {'selected': 'index', diff --git a/ietf/templates/registration/edit_profile.html b/ietf/templates/registration/edit_profile.html index c4bb4a9dd..146d0e69f 100644 --- a/ietf/templates/registration/edit_profile.html +++ b/ietf/templates/registration/edit_profile.html @@ -109,11 +109,11 @@ {{person_form.ascii_short|add_class:"form-control"}}
-
+ Example: A. Nonymous. Fill in this with initials and surname only if taking the initials and surname of your ASCII name above produces an incorrect initials-only form. (Blank is ok). -
+
diff --git a/requirements.txt b/requirements.txt index 783be1e62..5681a311a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ coverage>=3.7.1 decorator>=3.4.0 defusedxml>=0.4.1 # for TastyPie when ussing xml; not a declared dependency django-bootstrap3>=5.1.1 +django-tastypie>=0.12.1 django-widget-tweaks>=1.3 docutils>=0.12 html5lib>=0.90 @@ -20,5 +21,5 @@ python-memcached>=1.48 # for django.core.cache.backends.memcached pytz>=2014.7 setuptools>=1.2 six>=1.8.0 -django-tastypie>=0.12.1 wsgiref>=0.1.2 +xml2rfc>=2.5.0