Merged in support of xml-source only drafts submissions from personal/henrik/submitxml/.

- Legacy-Id: 9859

ietf/settings.py

@@ -432,18 +432,23 @@ IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_01 = 13
 IDSUBMIT_DEFAULT_CUTOFF_TIME_UTC = datetime.timedelta(hours=23, minutes=59, seconds=59)
 IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21)
 
-MEETING_MATERIALS_SUBMISSION_START_DAYS = -90
-MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26
-MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 50
-
-INTERNET_DRAFT_DAYS_TO_EXPIRE = 185
-
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
 IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
 IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
 
-IDSUBMIT_MAX_PLAIN_DRAFT_SIZE = 6291456  # Max size of the txt draft in bytes
+IDSUBMIT_FILE_TYPES = (
+    'txt',
+    'xml',
+    'pdf',
+    'ps',
+)
+IDSUBMIT_MAX_DRAFT_SIZE =  {
+    'txt':  6*1024*1024,  # Max size of txt draft file in bytes
+    'xml': 10*1024*1024,  # Max size of xml draft file in bytes
+    'pdf': 10*1024*1024,
+    'ps' : 10*1024*1024,
+}
 
 IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME = 20
 IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE = 50 # in MB
@@ -454,6 +459,14 @@ IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE = 450 # in MB
 IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000
 IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB
 
+XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/"
+
+MEETING_MATERIALS_SUBMISSION_START_DAYS = -90
+MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26
+MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 50
+
+INTERNET_DRAFT_DAYS_TO_EXPIRE = 185
+
 DOT_BINARY = '/usr/bin/dot'
 UNFLATTEN_BINARY= '/usr/bin/unflatten'
 PS2PDF_BINARY = '/usr/bin/ps2pdf'

ietf/submit/forms.py

@@ -1,6 +1,9 @@
 import os
+import re
 import datetime
 import pytz
+import xml2rfc
+import tempfile
 
 from django import forms
 from django.conf import settings
@@ -23,14 +26,14 @@ from ietf.submit.parsers.xml_parser import XMLParser
 from ietf.utils.draft import Draft
 
 
-class UploadForm(forms.Form):
+class SubmissionUploadForm(forms.Form):
-    txt = forms.FileField(label=u'.txt format', required=True)
+    txt = forms.FileField(label=u'.txt format', required=False)
     xml = forms.FileField(label=u'.xml format', required=False)
     pdf = forms.FileField(label=u'.pdf format', required=False)
-    ps = forms.FileField(label=u'.ps format', required=False)
+    ps  = forms.FileField(label=u'.ps format', required=False)
 
     def __init__(self, request, *args, **kwargs):
-        super(UploadForm, self).__init__(*args, **kwargs)
+        super(SubmissionUploadForm, self).__init__(*args, **kwargs)
 
         self.remote_ip = request.META.get('REMOTE_ADDR', None)
 
@@ -41,7 +44,13 @@ class UploadForm(forms.Form):
         self.set_cutoff_warnings()
 
         self.group = None
+        self.filename = None
+        self.revision = None
+        self.title = None
+        self.abstract = None
+        self.authors = []
         self.parsed_draft = None
+        self.file_types = []
 
     def set_cutoff_warnings(self):
         now = datetime.datetime.now(pytz.utc)
@@ -93,7 +102,6 @@ class UploadForm(forms.Form):
 
         return f
 
-
     def clean_txt(self):
         return self.clean_file("txt", PlainParser)
 
@@ -101,7 +109,7 @@ class UploadForm(forms.Form):
         return self.clean_file("pdf", PDFParser)
 
     def clean_ps(self):
-        return self.clean_file("ps", PSParser)
+        return self.clean_file("ps",  PSParser)
 
     def clean_xml(self):
         return self.clean_file("xml", XMLParser)
@@ -116,37 +124,100 @@ class UploadForm(forms.Form):
             if not os.path.exists(getattr(settings, s)):
                 raise forms.ValidationError('%s defined in settings.py does not exist' % s)
 
+        for ext in ['txt', 'pdf', 'xml', 'ps']:
+            f = self.cleaned_data.get(ext, None)
+            if not f:
+                continue
+            self.file_types.append('.%s' % ext)
+        if not ('.txt' in self.file_types or '.xml' in self.file_types):
+            raise forms.ValidationError('You must submit either a .txt or an .xml file; didn\'t find either.')
+
+        #debug.show('self.cleaned_data["xml"]')
+        if self.cleaned_data.get('xml'):
+            #if not self.cleaned_data.get('txt'):
+            xml_file = self.cleaned_data.get('xml')
+            tfh, tfn = tempfile.mkstemp(suffix='.xml')
+            try:
+                # We need to write the xml file to disk in order to hand it
+                # over to the xml parser.  XXX FIXME: investigate updating
+                # xml2rfc to be able to work with file handles to in-memory
+                # files.
+                with open(tfn, 'wb+') as tf:
+                    for chunk in xml_file.chunks():
+                        tf.write(chunk)
+                os.environ["XML_LIBRARY"] = settings.XML_LIBRARY
+                parser = xml2rfc.XmlRfcParser(tfn, quiet=True)
+                self.xmltree = parser.parse()
+                ok, errors = self.xmltree.validate()
+                if not ok:
+                    raise forms.ValidationError(errors)
+                self.xmlroot = self.xmltree.getroot()
+                draftname = self.xmlroot.attrib.get('docName')
+                revmatch = re.search("-[0-9][0-9]$", draftname)
+                if revmatch:
+                    self.revision = draftname[-2:]
+                    self.filename = draftname[:-3]
+                else:
+                    self.revision = None
+                    self.filename = draftname
+                self.title = self.xmlroot.find('front/title').text
+                self.abstract = self.xmlroot.find('front/abstract').text
+                self.author_list = []
+                author_info = self.xmlroot.findall('front/author')
+                for author in author_info:
+                    author_dict = dict(
+                        company = author.find('organization').text,
+                        last_name = author.attrib.get('surname'),
+                        full_name = author.attrib.get('fullname'),
+                        email = author.find('address/email').text,
+                    )
+                    self.author_list.append(author_dict)
+                    line = "%(full_name)s <%(email)s>" % author_dict
+                    self.authors.append(line)
+            except Exception as e:
+                raise forms.ValidationError("Exception: %s" % e)
+            finally:
+                os.close(tfh)
+                os.unlink(tfn)
+
         if self.cleaned_data.get('txt'):
             # try to parse it
             txt_file = self.cleaned_data['txt']
             txt_file.seek(0)
             self.parsed_draft = Draft(txt_file.read(), txt_file.name)
+            self.filename = self.parsed_draft.filename
+            self.revision = self.parsed_draft.revision
+            self.title    = self.parsed_draft.get_title()
             txt_file.seek(0)
 
-            if not self.parsed_draft.filename:
+            if not self.filename:
-                raise forms.ValidationError("Draft parser could not extract a valid draft name from the .txt file")
+                raise forms.ValidationError("Draft parser could not extract a valid draft name from the upload")
 
-            if not self.parsed_draft.get_title():
+            if not self.revision:
-                raise forms.ValidationError("Draft parser could not extract a valid title from the .txt file")
+                raise forms.ValidationError("Draft parser could not extract a valid draft revision from the upload")
 
+            if not self.title:
+                raise forms.ValidationError("Draft parser could not extract a valid title from the upload")
+
+        if self.cleaned_data.get('txt') or self.cleaned_data.get('xml'):
             # check group
             self.group = self.deduce_group()
 
             # check existing
-            existing = Submission.objects.filter(name=self.parsed_draft.filename, rev=self.parsed_draft.revision).exclude(state__in=("posted", "cancel"))
+            existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel"))
             if existing:
                 raise forms.ValidationError(mark_safe('Submission with same name and revision is currently being processed. <a href="%s">Check the status here.</a>' % urlreverse("submit_submission_status", kwargs={ 'submission_id': existing[0].pk })))
 
             # cut-off
-            if self.parsed_draft.revision == '00' and self.in_first_cut_off:
+            if self.revision == '00' and self.in_first_cut_off:
                 raise forms.ValidationError(mark_safe(self.cutoff_warning))
 
             # check thresholds
             today = datetime.date.today()
 
             self.check_submissions_tresholds(
-                "for the draft %s" % self.parsed_draft.filename,
+                "for the draft %s" % self.filename,
-                dict(name=self.parsed_draft.filename, rev=self.parsed_draft.revision, submission_date=today),
+                dict(name=self.filename, rev=self.revision, submission_date=today),
                 settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
             )
             self.check_submissions_tresholds(
@@ -166,19 +237,19 @@ class UploadForm(forms.Form):
                 settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
             )
 
-        return super(UploadForm, self).clean()
+        return super(SubmissionUploadForm, self).clean()
 
     def check_submissions_tresholds(self, which, filter_kwargs, max_amount, max_size):
         submissions = Submission.objects.filter(**filter_kwargs)
 
         if len(submissions) > max_amount:
             raise forms.ValidationError("Max submissions %s has been reached for today (maximum is %s submissions)." % (which, max_amount))
-        if sum(s.file_size for s in submissions) > max_size * 1024 * 1024:
+        if sum(s.file_size for s in submissions if s.file_size) > max_size * 1024 * 1024:
             raise forms.ValidationError("Max uploaded amount %s has been reached for today (maximum is %s MB)." % (which, max_size))
 
     def deduce_group(self):
         """Figure out group from name or previously submitted draft, returns None if individual."""
-        name = self.parsed_draft.filename
+        name = self.filename
         existing_draft = Document.objects.filter(name=name, type="draft")
         if existing_draft:
             group = existing_draft[0].group

ietf/submit/parsers/base.py

@@ -1,4 +1,11 @@
+import os
 import re
+import magic
+import datetime
+import debug                            # pyflakes:ignore
+
+from django.conf import settings
+from django.template.defaultfilters import filesizeformat
 
 class MetaData(object):
     rev = None
@@ -42,6 +49,8 @@ class FileParser(object):
     # no other file parsing is recommended
     def critical_parse(self):
         self.parse_invalid_chars_in_filename()
+        self.parse_max_size();
+        self.parsed_info.metadata.submission_date = datetime.date.today()
         return self.parsed_info
 
     def parse_invalid_chars_in_filename(self):
@@ -50,3 +59,22 @@ class FileParser(object):
         chars = regexp.findall(name)
         if chars:
             self.parsed_info.add_error('Invalid characters were found in the name of the file which was just submitted: %s' % ', '.join(set(chars)))
+
+    def parse_max_size(self):
+        __, ext = os.path.splitext(self.fd.name)
+        ext = ext.lstrip('.')
+        max_size = settings.IDSUBMIT_MAX_DRAFT_SIZE[ext]
+        if self.fd.size > max_size:
+            self.parsed_info.add_error('File size is larger than the permitted maximum of %s' % filesizeformat(max_size))
+        self.parsed_info.metadata.file_size = self.fd.size
+
+    def parse_filename_extension(self, ext):
+        if not self.fd.name.lower().endswith('.'+ext):
+            self.parsed_info.add_error('Expected the %s file to have extension ".%s", found "%s"' % (ext.upper(), ext, self.fd.name))
+
+    def parse_file_type(self, ext, expected):
+        self.fd.file.seek(0)
+        content = self.fd.file.read(4096)
+        mimetype = magic.from_buffer(content, mime=True)
+        if not mimetype == expected:
+            self.parsed_info.add_error('Expected an %s file of type "%s", found one of type "%s"' % (expected, mimetype))

ietf/submit/parsers/pdf_parser.py

@@ -7,9 +7,6 @@ class PDFParser(FileParser):
     # no other file parsing is recommended
     def critical_parse(self):
         super(PDFParser, self).critical_parse()
-        self.parse_filename_extension()
+        self.parse_filename_extension('pdf')
+        self.parse_file_type('pdf', 'application/pdf')
         return self.parsed_info
-
-    def parse_filename_extension(self):
-        if not self.fd.name.endswith('.pdf'):
-            self.parsed_info.add_error('Format of this document must be PDF')

ietf/submit/parsers/plain_parser.py

@@ -1,8 +1,5 @@
-import datetime
 import re
 
-from django.conf import settings
-from django.template.defaultfilters import filesizeformat
 from ietf.submit.parsers.base import FileParser
 
 
@@ -15,17 +12,12 @@ class PlainParser(FileParser):
     # no other file parsing is recommended
     def critical_parse(self):
         super(PlainParser, self).critical_parse()
-        self.parse_max_size()
+        self.parse_filename_extension('txt')
+        self.parse_file_type('txt', 'text/plain')
         self.parse_file_charset()
         self.parse_name()
         return self.parsed_info
 
-    def parse_max_size(self):
-        if self.fd.size > settings.IDSUBMIT_MAX_PLAIN_DRAFT_SIZE:
-            self.parsed_info.add_error('File size is larger than %s' % filesizeformat(settings.IDSUBMIT_MAX_PLAIN_DRAFT_SIZE))
-        self.parsed_info.metadata.file_size = self.fd.size
-        self.parsed_info.metadata.submission_date = datetime.date.today()
-
     def parse_file_charset(self):
         import magic
         self.fd.file.seek(0)

ietf/submit/parsers/ps_parser.py

@@ -7,9 +7,6 @@ class PSParser(FileParser):
     # no other file parsing is recommended
     def critical_parse(self):
         super(PSParser, self).critical_parse()
-        self.parse_filename_extension()
+        self.parse_filename_extension('ps')
+        self.parse_file_type('ps', 'application/postscript')
         return self.parsed_info
-
-    def parse_filename_extension(self):
-        if not self.fd.name.endswith('.ps'):
-            self.parsed_info.add_error('Format of this document must be PS')

ietf/submit/parsers/xml_parser.py

@@ -7,9 +7,7 @@ class XMLParser(FileParser):
     # no other file parsing is recommended
     def critical_parse(self):
         super(XMLParser, self).critical_parse()
-        self.parse_filename_extension()
+        self.parse_filename_extension('xml')
+        self.parse_file_type('xml', 'application/xml')
         return self.parsed_info
-
+    
-    def parse_filename_extension(self):
-        if not self.fd.name.endswith('.xml'):
-            self.parsed_info.add_error('Format of this document must be XML')

ietf/submit/test_submission.pdf

@@ -0,0 +1,2 @@
+%%PDF-1.5
+This is PDF

ietf/submit/test_submission.ps

@@ -0,0 +1,2 @@
+%%!PS-Adobe-2.0
+This is PostScript

ietf/submit/test_submission.txt

@@ -1,18 +1,21 @@
-Informational                                                Author Name
-Internet-Draft                                          Test Center Inc.
-Intended status: Informational                           %(date)s
-Expires: %(expire)s
 
-                          
-                        Testing tests
-                  %(name)s
 
 
+
+Network Working Group                                            A. Name
+Internet-Draft                                          Test Centre Inc.
+Intended status: Informational                                %(month)s %(year)s
+Expires: %(expiration)s
+
+
+                             Testing Tests
+                         %(name)s
+
 Abstract
 
    This document describes how to test tests.
 
-Status of this Memo
+Status of This Memo
 
    This Internet-Draft is submitted in full conformance with the
    provisions of BCP 78 and BCP 79.
@@ -27,7 +30,7 @@ Status of this Memo
    time.  It is inappropriate to use Internet-Drafts as reference
    material or to cite them other than as "work in progress."
 
-   This Internet-Draft will expire on %(expire)s.
+   This Internet-Draft will expire on %(expiration)s.
 
 Copyright Notice
 
@@ -40,67 +43,49 @@ Copyright Notice
    publication of this document.  Please review these documents
    carefully, as they describe your rights and restrictions with respect
    to this document.  Code Components extracted from this document must
-
-
-Name                     Expires %(expire)s                [Page 1]
-
-Internet-Draft             Testing tests                %(month_year)s
-
    include Simplified BSD License text as described in Section 4.e of
    the Trust Legal Provisions and are provided without warranty as
    described in the Simplified BSD License.
 
-   This document may contain material from IETF Documents or IETF
+
-   Contributions published or made publicly available before November
+
-   10, 2008.  The person(s) controlling the copyright in some of this
+
-   material may not have granted the IETF Trust the right to allow
+
-   modifications of such material outside the IETF Standards Process.
+
-   Without obtaining an adequate license from the person(s) controlling
+
-   the copyright in such materials, this document may not be modified
+Name                    Expires %(expiration)s               [Page 1]
-   outside the IETF Standards Process, and derivative works of it may
+
-   not be created outside the IETF Standards Process, except to format
+Internet-Draft                Testing Tests                   %(month)s %(year)s
-   it for publication as an RFC or to translate it into languages other
-   than English.
 
 
 Table of Contents
 
-   1.  Introduction . . . . . . . . . . . . . . . . . . . . . . . . .  3
+   1.  Introduction  . . . . . . . . . . . . . . . . . . . . . . . .   2
-   2.  IANA Considerations  . . . . . . . . . . . . . . . . . . . . .  4
+   2.  Security Considerations . . . . . . . . . . . . . . . . . . .   2
-   3.  Security Considerations  . . . . . . . . . . . . . . . . . . .  4
+   3.  IANA Considerations . . . . . . . . . . . . . . . . . . . . .   2
-
+   Author's Address  . . . . . . . . . . . . . . . . . . . . . . . .   2
-
-Name                     Expires %(expire)s                [Page 2]
-
-Internet-Draft             Testing tests                %(month_year)s
 
 1.  Introduction
 
    This document describes a protocol for testing tests.
 
-Name                     Expires %(expire)s                [Page 3]
+2.  Security Considerations
-
-Internet-Draft             Testing tests                %(month_year)s
 
-2. Security Considerations
+   There are none.
 
-  There are none.
+3.  IANA Considerations
 
+   No new registrations for IANA.
 
-3. IANA Considerations
+Author's Address
 
-  No new registrations for IANA.
+   Author Name
+   Test Centre Inc.
+   42 Some Road
+   Some Where 12345
+   UK
 
-
+   Email: author@example.com
-Authors' Addresses
-
-  Author Name
-  Test Center Inc.
-  42 Some Road
-  Some Where 12345
-  US
-
-  Email: author@example.com
 
 
 
@@ -109,4 +94,19 @@ Authors' Addresses
 
 
 
-Name                     Expires %(expire)s                [Page 4]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Name                    Expires %(expiration)s               [Page 2]

ietf/submit/test_submission.xml

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="US-ASCII"?>
+<!DOCTYPE rfc SYSTEM "rfc2629.dtd" []>
+<?rfc toc="yes"?>
+<rfc category="info" docName="%(name)s" ipr="trust200902">
+  <front>
+    <title>Testing Tests</title>
+    <author fullname="Author Name" initials="A." surname="Name">
+      <organization>Test Centre Inc.</organization>
+
+      <address>
+        <postal>
+          <street>42 Some Road</street>
+          <city>Some Where 12345</city>
+          <country>UK</country>
+        </postal>
+        <email>author@example.com</email>
+      </address>
+    </author>
+    <date month="%(month)s" year="%(year)s" />
+    <workgroup>%(group)s</workgroup>
+    <abstract>
+      <t>
+	 This document describes how to test tests.
+      </t>
+    </abstract>
+  </front>
+
+  <middle>
+    <section title="Introduction">
+      <t>
+	 This document describes a protocol for testing tests.
+      </t>
+    </section>
+    <section anchor="Security" title="Security Considerations">
+      <t>
+	 There are none.
+      </t>
+    </section>
+    <section anchor="IANA" title="IANA Considerations">
+      <t>
+	 No new registrations for IANA.
+      </t>
+    </section>
+  </middle>
+  <back>
+  </back>
+</rfc>

ietf/submit/tests.py

@@ -9,6 +9,8 @@ from django.core.urlresolvers import reverse as urlreverse
 from StringIO import StringIO
 from pyquery import PyQuery
 
+import debug                            # pyflakes:ignore
+
 from ietf.utils.test_utils import login_testing_unauthorized
 from ietf.utils.test_data import make_test_data
 from ietf.utils.mail import outbox
@@ -39,25 +41,26 @@ class SubmitTests(TestCase):
         shutil.rmtree(self.repository_dir)
         shutil.rmtree(self.archive_dir)
 
-    def submission_txt_file(self, name, rev):
+    def submission_file(self, name, rev, group, format, templatename):
         # construct appropriate text draft
-        f = open(os.path.join(settings.BASE_DIR, "submit", "test_submission.txt"))
+        f = open(os.path.join(settings.BASE_DIR, "submit", templatename))
         template = f.read()
         f.close()
 
         submission_text = template % dict(
             date=datetime.date.today().strftime("%d %B %Y"),
-            expire=(datetime.date.today() + datetime.timedelta(days=100)).strftime("%Y-%m-%d"),
+            expiration=(datetime.date.today() + datetime.timedelta(days=100)).strftime("%d %B, %Y"),
             year=datetime.date.today().strftime("%Y"),
-            month_year=datetime.date.today().strftime("%B, %Y"),
+            month=datetime.date.today().strftime("%B"),
             name="%s-%s" % (name, rev),
+            group=group or "",
             )
 
-        txt_file = StringIO(str(submission_text))
+        file = StringIO(str(submission_text))
-        txt_file.name = "somename.txt"
+        file.name = "%s-%s.%s" % (name, rev, format)
-        return txt_file
+        return file
 
-    def do_submission(self, name, rev):
+    def do_submission(self, name, rev, group=None, formats=["txt",]):
         # break early in case of missing configuration
         self.assertTrue(os.path.exists(settings.IDSUBMIT_IDNITS_BINARY))
 
@@ -67,15 +70,23 @@ class SubmitTests(TestCase):
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertEqual(len(q('input[type=file][name=txt]')), 1)
+        self.assertEqual(len(q('input[type=file][name=xml]')), 1)
 
         # submit
-        txt_file = self.submission_txt_file(name, rev)
+        files = {}
+        for format in formats:
+            files[format] = self.submission_file(name, rev, group, format, "test_submission.%s" % format)
+
+        r = self.client.post(url, files)
+        if r.status_code != 302:
+            q = PyQuery(r.content)
+            print(q('div.has-error span.help-block div').text)
 
-        r = self.client.post(url,
-                             dict(txt=txt_file))
         self.assertEqual(r.status_code, 302)
+
         status_url = r["Location"]
-        self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))))
+        for format in formats:
+            self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.%s" % (name, rev, format))))
         self.assertEqual(Submission.objects.filter(name=name).count(), 1)
         submission = Submission.objects.get(name=name)
         self.assertTrue(re.search('\s+Summary:\s+0\s+errors|No nits found', submission.idnits_message))
@@ -120,7 +131,7 @@ class SubmitTests(TestCase):
 
         return confirm_url
 
-    def test_submit_new_wg(self):
+    def submit_new_wg(self, formats):
         # submit new -> supply submitter info -> approve
         draft = make_test_data()
 
@@ -147,8 +158,9 @@ class SubmitTests(TestCase):
 
         name = "draft-ietf-mars-testing-tests"
         rev = "00"
+        group = "mars"
 
-        status_url = self.do_submission(name, rev)
+        status_url = self.do_submission(name, rev, group, formats)
 
         # supply submitter info, then draft should be in and ready for approval
         mailbox_before = len(outbox)
@@ -211,7 +223,16 @@ class SubmitTests(TestCase):
         self.assertTrue("ameschairman" in outbox[-1]["To"].lower())
         self.assertTrue("marschairman" in outbox[-1]["To"].lower())
 
-    def test_submit_existing(self):
+    def test_submit_new_wg_txt(self):
+        self.submit_new_wg(["txt"])
+
+    def text_submit_new_wg_xml(self):
+        self.submit_new_wg(["xml"])
+
+    def text_submit_new_wg_txt_xml(self):
+        self.submit_new_wg(["txt", "xml"])
+
+    def submit_existing(self, formats):
         # submit new revision of existing -> supply submitter info -> prev authors confirm
         draft = make_test_data()
         prev_author = draft.documentauthor_set.all()[0]
@@ -241,13 +262,14 @@ class SubmitTests(TestCase):
 
         name = draft.name
         rev = "%02d" % (int(draft.rev) + 1)
+        group = draft.group
 
         # write the old draft in a file so we can check it's moved away
         old_rev = draft.rev
         with open(os.path.join(self.repository_dir, "%s-%s.txt" % (name, old_rev)), 'w') as f:
             f.write("a" * 2000)
 
-        status_url = self.do_submission(name, rev)
+        status_url = self.do_submission(name, rev, group, formats)
 
         # supply submitter info, then previous authors get a confirmation email
         mailbox_before = len(outbox)
@@ -310,14 +332,24 @@ class SubmitTests(TestCase):
         self.assertTrue(name in unicode(outbox[-1]))
         self.assertTrue("mars" in unicode(outbox[-1]))
 
-    def test_submit_new_individual(self):
+    def test_submit_existing_txt(self):
+        self.submit_existing(["txt"])
+
+    def test_submit_existing_xml(self):
+        self.submit_existing(["xml"])
+
+    def test_submit_existing_txt_xml(self):
+        self.submit_existing(["txt", "xml"])
+
+    def submit_new_individual(self, formats):
         # submit new -> supply submitter info -> confirm
         draft = make_test_data()
 
         name = "draft-authorname-testing-tests"
         rev = "00"
+        group = None
 
-        status_url = self.do_submission(name, rev)
+        status_url = self.do_submission(name, rev, group, formats)
 
         # supply submitter info, then draft should be be ready for email auth
         mailbox_before = len(outbox)
@@ -355,6 +387,15 @@ class SubmitTests(TestCase):
         self.assertEqual(new_revision.type, "new_revision")
         self.assertEqual(new_revision.by.name, "Submitter Name")
 
+    def test_submit_new_individual_txt(self):
+        self.submit_new_individual(["txt"])
+
+    def test_submit_new_individual_xml(self):
+        self.submit_new_individual(["xml"])
+
+    def test_submit_new_individual_txt_xml(self):
+        self.submit_new_individual(["txt", "xml"])
+
     def test_submit_new_wg_with_dash(self):
         make_test_data()
 
@@ -579,35 +620,17 @@ class SubmitTests(TestCase):
 
         name = "draft-ietf-mars-testing-tests"
         rev = "00"
+        group = "mars"
 
-        txt_file = self.submission_txt_file(name, rev)
+        self.do_submission(name, rev, group, ["txt", "xml", "ps", "pdf"])
-
-        # the checks for other file types are currently embarrassingly
-        # dumb, so don't bother constructing proper XML/PS/PDF draft
-        # files
-        xml_file = StringIO('<?xml version="1.0" encoding="utf-8"?>\n<draft>This is XML</draft>')
-        xml_file.name = "somename.xml"
-
-        pdf_file = StringIO('%PDF-1.5\nThis is PDF')
-        pdf_file.name = "somename.pdf"
-
-        ps_file = StringIO('%!PS-Adobe-2.0\nThis is PostScript')
-        ps_file.name = "somename.ps"
-        
-        r = self.client.post(urlreverse('submit_upload_submission'), dict(
-            txt=txt_file,
-            xml=xml_file,
-            pdf=pdf_file,
-            ps=ps_file,
-        ))
-        self.assertEqual(r.status_code, 302)
 
         self.assertEqual(Submission.objects.filter(name=name).count(), 1)
 
         self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))))
         self.assertTrue(name in open(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))).read())
         self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))))
-        self.assertTrue('This is XML' in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read())
+        self.assertTrue(name in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read())
+        self.assertTrue('<?xml version="1.0" encoding="US-ASCII"?>' in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read())
         self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.pdf" % (name, rev))))
         self.assertTrue('This is PDF' in open(os.path.join(self.staging_dir, u"%s-%s.pdf" % (name, rev))).read())
         self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.ps" % (name, rev))))

ietf/submit/utils.py

@@ -353,21 +353,24 @@ def cancel_submission(submission):
     remove_submission_files(submission)
 
 def rename_submission_files(submission, prev_rev, new_rev):
-    for ext in submission.file_types.split(','):
+    from ietf.submit.forms import SubmissionUploadForm
-        source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, prev_rev, ext))
+    for ext in SubmissionUploadForm.base_fields.keys():
-        dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, new_rev, ext))
+        source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, prev_rev, ext))
-        os.rename(source, dest)
+        dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, new_rev, ext))
+        if os.path.exists(source):
+            os.rename(source, dest)
 
 def move_files_to_repository(submission):
-    for ext in submission.file_types.split(','):
+    from ietf.submit.forms import SubmissionUploadForm
-        source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, submission.rev, ext))
+    for ext in SubmissionUploadForm.base_fields.keys():
-        dest = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, '%s-%s%s' % (submission.name, submission.rev, ext))
+        source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext))
+        dest = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext))
         if os.path.exists(source):
             os.rename(source, dest)
         else:
             if os.path.exists(dest):
                 log("Intended to move '%s' to '%s', but found source missing while destination exists.")
-            else:
+            elif ext in submission.file_types.split(','):
                 raise ValueError("Intended to move '%s' to '%s', but found source and destination missing.")
 
 def remove_submission_files(submission):

ietf/submit/views.py

@@ -1,6 +1,7 @@
 # Copyright The IETF Trust 2007, All Rights Reserved
 import datetime
 import os
+import xml2rfc
 
 from django.conf import settings
 from django.core.urlresolvers import reverse as urlreverse
@@ -8,94 +9,127 @@ from django.core.validators import validate_email, ValidationError
 from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden
 from django.shortcuts import get_object_or_404, redirect, render
 
+import debug                            # pyflakes:ignore
+
 from ietf.doc.models import Document, DocAlias
 from ietf.doc.utils import prettify_std_name
 from ietf.group.models import Group
 from ietf.ietfauth.utils import has_role, role_required
-from ietf.submit.forms import UploadForm, NameEmailForm, EditSubmissionForm, PreapprovalForm, ReplacesForm
+from ietf.submit.forms import SubmissionUploadForm, NameEmailForm, EditSubmissionForm, PreapprovalForm, ReplacesForm
 from ietf.submit.mail import send_full_url, send_approval_request_to_group, send_submission_confirmation, submission_confirmation_email_list, send_manual_post_request
 from ietf.submit.models import Submission, Preapproval, DraftSubmissionStateName
 from ietf.submit.utils import approvable_submissions_for_user, preapprovals_for_user, recently_approved_by_user
 from ietf.submit.utils import check_idnits, found_idnits, validate_submission, create_submission_event
 from ietf.submit.utils import post_submission, cancel_submission, rename_submission_files
 from ietf.utils.accesstoken import generate_random_key, generate_access_token
+from ietf.utils.draft import Draft
+
 
 def upload_submission(request):
     if request.method == 'POST':
         try:
-            form = UploadForm(request, data=request.POST, files=request.FILES)
+            form = SubmissionUploadForm(request, data=request.POST, files=request.FILES)
             if form.is_valid():
-                # save files
+                authors = []
-                file_types = []
+                file_name = {}
-                for ext in ['txt', 'pdf', 'xml', 'ps']:
+                abstract = None
+                file_size = None
+                for ext in form.fields.keys():
                     f = form.cleaned_data[ext]
                     if not f:
                         continue
-                    file_types.append('.%s' % ext)
+                    
-
+                    name = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (form.filename, form.revision, ext))
-                    draft = form.parsed_draft
+                    file_name[ext] = name
-
-                    name = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (draft.filename, draft.revision, ext))
                     with open(name, 'wb+') as destination:
                         for chunk in f.chunks():
                             destination.write(chunk)
 
-                # check idnits
+                if form.cleaned_data['xml']:
-                text_path = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (draft.filename, draft.revision))
+                    if not form.cleaned_data['txt']:
-                idnits_message = check_idnits(text_path)
-
-                # extract author lines
-                authors = []
-                for author in draft.get_author_list():
-                    full_name, first_name, middle_initial, last_name, name_suffix, email, company = author
-
-                    line = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip()
-                    email = (email or "").strip()
-
-                    if email:
                         try:
-                            validate_email(email)
+                            file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (form.filename, form.revision))
-                        except ValidationError:
+                            pagedwriter = xml2rfc.PaginatedTextRfcWriter(form.xmltree, quiet=True)
-                            email = ""
+                            pagedwriter.write(file_name['txt'])
+                            file_size = os.stat(file_name['txt']).st_size
+                        except Exception as e:
+                            raise ValidationError("Exception: %s" % e)
+                    # Some meta-information, such as the page-count, can only
+                    # be retrieved from the generated text file.  Provide a
+                    # parsed draft object to get at that kind of information.
+                    with open(file_name['txt']) as txt_file:
+                        form.parsed_draft = Draft(txt_file.read(), txt_file.name)
 
-                    if email:
+                else:
-                        line += u" <%s>" % email
+                    file_size = form.cleaned_data['txt'].size
 
-                    authors.append(line)
+                if form.authors:
+                    authors = form.authors
+                else:
+                    # If we don't have an xml file, try to extract the
+                    # relevant information from the text file
+                    for author in form.parsed_draft.get_author_list():
+                        full_name, first_name, middle_initial, last_name, name_suffix, email, company = author
+
+                        line = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip()
+                        email = (email or "").strip()
+
+                        if email:
+                            try:
+                                validate_email(email)
+                            except ValidationError:
+                                email = ""
+
+                        if email:
+                            line += u" <%s>" % email
+
+                        authors.append(line)
+
+                if form.abstract:
+                    abstract = form.abstract
+                else:
+                    abstract = form.parsed_draft.get_abstract()
+
+                # check idnits
+                idnits_message = check_idnits(file_name['txt'])
 
                 # save submission
-                submission = Submission.objects.create(
+                try:
-                    state=DraftSubmissionStateName.objects.get(slug="uploaded"),
+                    submission = Submission.objects.create(
-                    remote_ip=form.remote_ip,
+                        state=DraftSubmissionStateName.objects.get(slug="uploaded"),
-                    name=draft.filename,
+                        remote_ip=form.remote_ip,
-                    group=form.group,
+                        name=form.filename,
-                    title=draft.get_title(),
+                        group=form.group,
-                    abstract=draft.get_abstract(),
+                        title=form.title,
-                    rev=draft.revision,
+                        abstract=abstract,
-                    pages=draft.get_pagecount(),
+                        rev=form.revision,
-                    authors="\n".join(authors),
+                        pages=form.parsed_draft.get_pagecount(),
-                    note="",
+                        authors="\n".join(authors),
-                    first_two_pages=''.join(draft.pages[:2]),
+                        note="",
-                    file_size=form.cleaned_data['txt'].size,
+                        first_two_pages=''.join(form.parsed_draft.pages[:2]),
-                    file_types=','.join(file_types),
+                        file_size=file_size,
-                    submission_date=datetime.date.today(),
+                        file_types=','.join(form.file_types),
-                    document_date=draft.get_creation_date(),
+                        submission_date=datetime.date.today(),
-                    replaces="",
+                        document_date=form.parsed_draft.get_creation_date(),
-                    idnits_message=idnits_message,
+                        replaces="",
-                    )
+                        idnits_message=idnits_message,
+                        )
+                except Exception as e:
+                    import sys
+                    sys.stderr.write("Exception: %s\n" % e)
 
                 create_submission_event(request, submission, desc="Uploaded submission")
 
                 return redirect("submit_submission_status_by_hash", submission_id=submission.pk, access_token=submission.access_token())
         except IOError as e:
             if "read error" in str(e): # The server got an IOError when trying to read POST data
-                form = UploadForm(request=request)
+                form = SubmissionUploadForm(request=request)
                 form._errors = {}
                 form._errors["__all__"] = form.error_class(["There was a failure receiving the complete form data -- please try again."])
             else:
                 raise
     else:
-        form = UploadForm(request=request)
+        form = SubmissionUploadForm(request=request)
 
     return render(request, 'submit/upload_submission.html',
                               {'selected': 'index',

ietf/templates/registration/edit_profile.html

@@ -109,11 +109,11 @@
 	    {{person_form.ascii_short|add_class:"form-control"}}
 	  </div>
 	  <div class="col-sm-7">
-	    <div class="help-block">
+	    <span class="help-block">
 	       Example: A. Nonymous.  Fill in this with initials and surname only if 
 	       taking the initials and surname of your ASCII name above produces an incorrect
 	       initials-only form. (Blank is ok).
-	    </div>
+	    </span>
 	  </div>
         </div>
       </div>

requirements.txt

@@ -4,6 +4,7 @@ coverage>=3.7.1
 decorator>=3.4.0
 defusedxml>=0.4.1		# for TastyPie when ussing xml; not a declared dependency
 django-bootstrap3>=5.1.1
+django-tastypie>=0.12.1
 django-widget-tweaks>=1.3
 docutils>=0.12                  
 html5lib>=0.90
@@ -20,5 +21,5 @@ python-memcached>=1.48		# for django.core.cache.backends.memcached
 pytz>=2014.7
 setuptools>=1.2
 six>=1.8.0
-django-tastypie>=0.12.1
 wsgiref>=0.1.2
+xml2rfc>=2.5.0