altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Merged in support of xml-source only drafts submissions from personal/henrik/submitxml/.

Browse source 
- Legacy-Id: 9859
This commit is contained in:
Henrik Levkowetz 2015-07-22 21:53:23 +00:00
parent c8ac65ab1d d5f36911cf
commit da893dad1c
16 changed files with 410 additions and 202 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
ietf/settings.py
View file

@ -432,18 +432,23 @@ IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_01 = 13
IDSUBMIT_DEFAULT_CUTOFF_TIME_UTC = datetime.timedelta(hours=23, minutes=59, seconds=59)
IDSUBMIT_DEFAULT_CUTOFF_WARNING_DAYS = datetime.timedelta(days=21)
MEETING_MATERIALS_SUBMISSION_START_DAYS = -90
MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26
MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 50
INTERNET_DRAFT_DAYS_TO_EXPIRE = 185
IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
IDSUBMIT_STAGING_PATH = '/a/www/www6s/staging/'
IDSUBMIT_STAGING_URL = '//www.ietf.org/staging/'
IDSUBMIT_IDNITS_BINARY = '/a/www/ietf-datatracker/scripts/idnits'
IDSUBMIT_MAX_PLAIN_DRAFT_SIZE = 6291456 # Max size of the txt draft in bytes
IDSUBMIT_FILE_TYPES = (
'txt',
'xml',
'pdf',
'ps',
)
IDSUBMIT_MAX_DRAFT_SIZE = {
'txt': 6*1024*1024, # Max size of txt draft file in bytes
'xml': 10*1024*1024, # Max size of xml draft file in bytes
'pdf': 10*1024*1024,
'ps' : 10*1024*1024,
}
IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME = 20
IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE = 50 # in MB
@ -454,6 +459,14 @@ IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE = 450 # in MB
IDSUBMIT_MAX_DAILY_SUBMISSIONS = 1000
IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE = 2000 # in MB
XML_LIBRARY = "/www/tools.ietf.org/tools/xml2rfc/web/public/rfc/"
MEETING_MATERIALS_SUBMISSION_START_DAYS = -90
MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26
MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 50
INTERNET_DRAFT_DAYS_TO_EXPIRE = 185
DOT_BINARY = '/usr/bin/dot'
UNFLATTEN_BINARY= '/usr/bin/unflatten'
PS2PDF_BINARY = '/usr/bin/ps2pdf'

101
ietf/submit/forms.py
View file

@ -1,6 +1,9 @@
import os
import re
import datetime
import pytz
import xml2rfc
import tempfile
from django import forms
from django.conf import settings
@ -23,14 +26,14 @@ from ietf.submit.parsers.xml_parser import XMLParser
from ietf.utils.draft import Draft
class UploadForm(forms.Form):
txt = forms.FileField(label=u'.txt format', required=True)
class SubmissionUploadForm(forms.Form):
txt = forms.FileField(label=u'.txt format', required=False)
xml = forms.FileField(label=u'.xml format', required=False)
pdf = forms.FileField(label=u'.pdf format', required=False)
ps = forms.FileField(label=u'.ps format', required=False)
def __init__(self, request, *args, **kwargs):
super(UploadForm, self).__init__(*args, **kwargs)
super(SubmissionUploadForm, self).__init__(*args, **kwargs)
self.remote_ip = request.META.get('REMOTE_ADDR', None)
@ -41,7 +44,13 @@ class UploadForm(forms.Form):
self.set_cutoff_warnings()
self.group = None
self.filename = None
self.revision = None
self.title = None
self.abstract = None
self.authors = []
self.parsed_draft = None
self.file_types = []
def set_cutoff_warnings(self):
now = datetime.datetime.now(pytz.utc)
@ -93,7 +102,6 @@ class UploadForm(forms.Form):
return f
def clean_txt(self):
return self.clean_file("txt", PlainParser)
@ -116,37 +124,100 @@ class UploadForm(forms.Form):
if not os.path.exists(getattr(settings, s)):
raise forms.ValidationError('%s defined in settings.py does not exist' % s)
for ext in ['txt', 'pdf', 'xml', 'ps']:
f = self.cleaned_data.get(ext, None)
if not f:
continue
self.file_types.append('.%s' % ext)
if not ('.txt' in self.file_types or '.xml' in self.file_types):
raise forms.ValidationError('You must submit either a .txt or an .xml file; didn\'t find either.')
#debug.show('self.cleaned_data["xml"]')
if self.cleaned_data.get('xml'):
#if not self.cleaned_data.get('txt'):
xml_file = self.cleaned_data.get('xml')
tfh, tfn = tempfile.mkstemp(suffix='.xml')
try:
# We need to write the xml file to disk in order to hand it
# over to the xml parser. XXX FIXME: investigate updating
# xml2rfc to be able to work with file handles to in-memory
# files.
with open(tfn, 'wb+') as tf:
for chunk in xml_file.chunks():
tf.write(chunk)
os.environ["XML_LIBRARY"] = settings.XML_LIBRARY
parser = xml2rfc.XmlRfcParser(tfn, quiet=True)
self.xmltree = parser.parse()
ok, errors = self.xmltree.validate()
if not ok:
raise forms.ValidationError(errors)
self.xmlroot = self.xmltree.getroot()
draftname = self.xmlroot.attrib.get('docName')
revmatch = re.search("-[0-9][0-9]$", draftname)
if revmatch:
self.revision = draftname[-2:]
self.filename = draftname[:-3]
else:
self.revision = None
self.filename = draftname
self.title = self.xmlroot.find('front/title').text
self.abstract = self.xmlroot.find('front/abstract').text
self.author_list = []
author_info = self.xmlroot.findall('front/author')
for author in author_info:
author_dict = dict(
company = author.find('organization').text,
last_name = author.attrib.get('surname'),
full_name = author.attrib.get('fullname'),
email = author.find('address/email').text,
)
self.author_list.append(author_dict)
line = "%(full_name)s <%(email)s>" % author_dict
self.authors.append(line)
except Exception as e:
raise forms.ValidationError("Exception: %s" % e)
finally:
os.close(tfh)
os.unlink(tfn)
if self.cleaned_data.get('txt'):
# try to parse it
txt_file = self.cleaned_data['txt']
txt_file.seek(0)
self.parsed_draft = Draft(txt_file.read(), txt_file.name)
self.filename = self.parsed_draft.filename
self.revision = self.parsed_draft.revision
self.title = self.parsed_draft.get_title()
txt_file.seek(0)
if not self.parsed_draft.filename:
raise forms.ValidationError("Draft parser could not extract a valid draft name from the .txt file")
if not self.filename:
raise forms.ValidationError("Draft parser could not extract a valid draft name from the upload")
if not self.parsed_draft.get_title():
raise forms.ValidationError("Draft parser could not extract a valid title from the .txt file")
if not self.revision:
raise forms.ValidationError("Draft parser could not extract a valid draft revision from the upload")
if not self.title:
raise forms.ValidationError("Draft parser could not extract a valid title from the upload")
if self.cleaned_data.get('txt') or self.cleaned_data.get('xml'):
# check group
self.group = self.deduce_group()
# check existing
existing = Submission.objects.filter(name=self.parsed_draft.filename, rev=self.parsed_draft.revision).exclude(state__in=("posted", "cancel"))
existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel"))
if existing:
raise forms.ValidationError(mark_safe('Submission with same name and revision is currently being processed. <a href="%s">Check the status here.</a>' % urlreverse("submit_submission_status", kwargs={ 'submission_id': existing[0].pk })))
# cut-off
if self.parsed_draft.revision == '00' and self.in_first_cut_off:
if self.revision == '00' and self.in_first_cut_off:
raise forms.ValidationError(mark_safe(self.cutoff_warning))
# check thresholds
today = datetime.date.today()
self.check_submissions_tresholds(
"for the draft %s" % self.parsed_draft.filename,
dict(name=self.parsed_draft.filename, rev=self.parsed_draft.revision, submission_date=today),
"for the draft %s" % self.filename,
dict(name=self.filename, rev=self.revision, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
)
self.check_submissions_tresholds(
@ -166,19 +237,19 @@ class UploadForm(forms.Form):
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
)
return super(UploadForm, self).clean()
return super(SubmissionUploadForm, self).clean()
def check_submissions_tresholds(self, which, filter_kwargs, max_amount, max_size):
submissions = Submission.objects.filter(**filter_kwargs)
if len(submissions) > max_amount:
raise forms.ValidationError("Max submissions %s has been reached for today (maximum is %s submissions)." % (which, max_amount))
if sum(s.file_size for s in submissions) > max_size * 1024 * 1024:
if sum(s.file_size for s in submissions if s.file_size) > max_size * 1024 * 1024:
raise forms.ValidationError("Max uploaded amount %s has been reached for today (maximum is %s MB)." % (which, max_size))
def deduce_group(self):
"""Figure out group from name or previously submitted draft, returns None if individual."""
name = self.parsed_draft.filename
name = self.filename
existing_draft = Document.objects.filter(name=name, type="draft")
if existing_draft:
group = existing_draft[0].group

28
ietf/submit/parsers/base.py
View file

@ -1,4 +1,11 @@
import os
import re
import magic
import datetime
import debug # pyflakes:ignore
from django.conf import settings
from django.template.defaultfilters import filesizeformat
class MetaData(object):
rev = None
@ -42,6 +49,8 @@ class FileParser(object):
# no other file parsing is recommended
def critical_parse(self):
self.parse_invalid_chars_in_filename()
self.parse_max_size();
self.parsed_info.metadata.submission_date = datetime.date.today()
return self.parsed_info
def parse_invalid_chars_in_filename(self):
@ -50,3 +59,22 @@ class FileParser(object):
chars = regexp.findall(name)
if chars:
self.parsed_info.add_error('Invalid characters were found in the name of the file which was just submitted: %s' % ', '.join(set(chars)))
def parse_max_size(self):
__, ext = os.path.splitext(self.fd.name)
ext = ext.lstrip('.')
max_size = settings.IDSUBMIT_MAX_DRAFT_SIZE[ext]
if self.fd.size > max_size:
self.parsed_info.add_error('File size is larger than the permitted maximum of %s' % filesizeformat(max_size))
self.parsed_info.metadata.file_size = self.fd.size
def parse_filename_extension(self, ext):
if not self.fd.name.lower().endswith('.'+ext):
self.parsed_info.add_error('Expected the %s file to have extension ".%s", found "%s"' % (ext.upper(), ext, self.fd.name))
def parse_file_type(self, ext, expected):
self.fd.file.seek(0)
content = self.fd.file.read(4096)
mimetype = magic.from_buffer(content, mime=True)
if not mimetype == expected:
self.parsed_info.add_error('Expected an %s file of type "%s", found one of type "%s"' % (expected, mimetype))

7
ietf/submit/parsers/pdf_parser.py
View file

@ -7,9 +7,6 @@ class PDFParser(FileParser):
# no other file parsing is recommended
def critical_parse(self):
super(PDFParser, self).critical_parse()
self.parse_filename_extension()
self.parse_filename_extension('pdf')
self.parse_file_type('pdf', 'application/pdf')
return self.parsed_info
def parse_filename_extension(self):
if not self.fd.name.endswith('.pdf'):
self.parsed_info.add_error('Format of this document must be PDF')

12
ietf/submit/parsers/plain_parser.py
View file

@ -1,8 +1,5 @@
import datetime
import re
from django.conf import settings
from django.template.defaultfilters import filesizeformat
from ietf.submit.parsers.base import FileParser
@ -15,17 +12,12 @@ class PlainParser(FileParser):
# no other file parsing is recommended
def critical_parse(self):
super(PlainParser, self).critical_parse()
self.parse_max_size()
self.parse_filename_extension('txt')
self.parse_file_type('txt', 'text/plain')
self.parse_file_charset()
self.parse_name()
return self.parsed_info
def parse_max_size(self):
if self.fd.size > settings.IDSUBMIT_MAX_PLAIN_DRAFT_SIZE:
self.parsed_info.add_error('File size is larger than %s' % filesizeformat(settings.IDSUBMIT_MAX_PLAIN_DRAFT_SIZE))
self.parsed_info.metadata.file_size = self.fd.size
self.parsed_info.metadata.submission_date = datetime.date.today()
def parse_file_charset(self):
import magic
self.fd.file.seek(0)

7
ietf/submit/parsers/ps_parser.py
View file

@ -7,9 +7,6 @@ class PSParser(FileParser):
# no other file parsing is recommended
def critical_parse(self):
super(PSParser, self).critical_parse()
self.parse_filename_extension()
self.parse_filename_extension('ps')
self.parse_file_type('ps', 'application/postscript')
return self.parsed_info
def parse_filename_extension(self):
if not self.fd.name.endswith('.ps'):
self.parsed_info.add_error('Format of this document must be PS')

6
ietf/submit/parsers/xml_parser.py
View file

@ -7,9 +7,7 @@ class XMLParser(FileParser):
# no other file parsing is recommended
def critical_parse(self):
super(XMLParser, self).critical_parse()
self.parse_filename_extension()
self.parse_filename_extension('xml')
self.parse_file_type('xml', 'application/xml')
return self.parsed_info
def parse_filename_extension(self):
if not self.fd.name.endswith('.xml'):
self.parsed_info.add_error('Format of this document must be XML')

2
ietf/submit/test_submission.pdf Normal file
View file

@ -0,0 +1,2 @@
%%PDF-1.5
This is PDF

2
ietf/submit/test_submission.ps Normal file
View file

@ -0,0 +1,2 @@
%%!PS-Adobe-2.0
This is PostScript

86
ietf/submit/test_submission.txt
View file

@ -1,18 +1,21 @@
Informational Author Name
Internet-Draft Test Center Inc.
Intended status: Informational %(date)s
Expires: %(expire)s
Testing tests
Network Working Group A. Name
Internet-Draft Test Centre Inc.
Intended status: Informational %(month)s %(year)s
Expires: %(expiration)s
Testing Tests
%(name)s
Abstract
This document describes how to test tests.
Status of this Memo
Status of This Memo
This Internet-Draft is submitted in full conformance with the
provisions of BCP 78 and BCP 79.
@ -27,7 +30,7 @@ Status of this Memo
time. It is inappropriate to use Internet-Drafts as reference
material or to cite them other than as "work in progress."
This Internet-Draft will expire on %(expire)s.
This Internet-Draft will expire on %(expiration)s.
Copyright Notice
@ -40,65 +43,47 @@ Copyright Notice
publication of this document. Please review these documents
carefully, as they describe your rights and restrictions with respect
to this document. Code Components extracted from this document must
Name Expires %(expire)s [Page 1]
Internet-Draft Testing tests %(month_year)s
include Simplified BSD License text as described in Section 4.e of
the Trust Legal Provisions and are provided without warranty as
described in the Simplified BSD License.
This document may contain material from IETF Documents or IETF
Contributions published or made publicly available before November
10, 2008. The person(s) controlling the copyright in some of this
material may not have granted the IETF Trust the right to allow
modifications of such material outside the IETF Standards Process.
Without obtaining an adequate license from the person(s) controlling
the copyright in such materials, this document may not be modified
outside the IETF Standards Process, and derivative works of it may
not be created outside the IETF Standards Process, except to format
it for publication as an RFC or to translate it into languages other
than English.
Name Expires %(expiration)s [Page 1]
Internet-Draft Testing Tests %(month)s %(year)s
Table of Contents
1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3
2. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 4
3. Security Considerations . . . . . . . . . . . . . . . . . . . 4
Name Expires %(expire)s [Page 2]
Internet-Draft Testing tests %(month_year)s
1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . 2
2. Security Considerations . . . . . . . . . . . . . . . . . . . 2
3. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 2
Author's Address . . . . . . . . . . . . . . . . . . . . . . . . 2
1. Introduction
This document describes a protocol for testing tests.
Name Expires %(expire)s [Page 3]
Internet-Draft Testing tests %(month_year)s
2. Security Considerations
There are none.
3. IANA Considerations
No new registrations for IANA.
Authors' Addresses
Author's Address
Author Name
Test Center Inc.
Test Centre Inc.
42 Some Road
Some Where 12345
US
UK
Email: author@example.com
@ -109,4 +94,19 @@ Authors' Addresses
Name Expires %(expire)s [Page 4]
Name Expires %(expiration)s [Page 2]

47
ietf/submit/test_submission.xml Normal file
View file

@ -0,0 +1,47 @@
<?xml version="1.0" encoding="US-ASCII"?>
<!DOCTYPE rfc SYSTEM "rfc2629.dtd" []>
<?rfc toc="yes"?>
<rfc category="info" docName="%(name)s" ipr="trust200902">
<front>
<title>Testing Tests</title>
<author fullname="Author Name" initials="A." surname="Name">
<organization>Test Centre Inc.</organization>
<address>
<postal>
<street>42 Some Road</street>
<city>Some Where 12345</city>
<country>UK</country>
</postal>
<email>author@example.com</email>
</address>
</author>
<date month="%(month)s" year="%(year)s" />
<workgroup>%(group)s</workgroup>
<abstract>
<t>
This document describes how to test tests.
</t>
</abstract>
</front>
<middle>
<section title="Introduction">
<t>
This document describes a protocol for testing tests.
</t>
</section>
<section anchor="Security" title="Security Considerations">
<t>
There are none.
</t>
</section>
<section anchor="IANA" title="IANA Considerations">
<t>
No new registrations for IANA.
</t>
</section>
</middle>
<back>
</back>
</rfc>

103
ietf/submit/tests.py
View file

@ -9,6 +9,8 @@ from django.core.urlresolvers import reverse as urlreverse
from StringIO import StringIO
from pyquery import PyQuery
import debug # pyflakes:ignore
from ietf.utils.test_utils import login_testing_unauthorized
from ietf.utils.test_data import make_test_data
from ietf.utils.mail import outbox
@ -39,25 +41,26 @@ class SubmitTests(TestCase):
shutil.rmtree(self.repository_dir)
shutil.rmtree(self.archive_dir)
def submission_txt_file(self, name, rev):
def submission_file(self, name, rev, group, format, templatename):
# construct appropriate text draft
f = open(os.path.join(settings.BASE_DIR, "submit", "test_submission.txt"))
f = open(os.path.join(settings.BASE_DIR, "submit", templatename))
template = f.read()
f.close()
submission_text = template % dict(
date=datetime.date.today().strftime("%d %B %Y"),
expire=(datetime.date.today() + datetime.timedelta(days=100)).strftime("%Y-%m-%d"),
expiration=(datetime.date.today() + datetime.timedelta(days=100)).strftime("%d %B, %Y"),
year=datetime.date.today().strftime("%Y"),
month_year=datetime.date.today().strftime("%B, %Y"),
month=datetime.date.today().strftime("%B"),
name="%s-%s" % (name, rev),
group=group or "",
)
txt_file = StringIO(str(submission_text))
txt_file.name = "somename.txt"
return txt_file
file = StringIO(str(submission_text))
file.name = "%s-%s.%s" % (name, rev, format)
return file
def do_submission(self, name, rev):
def do_submission(self, name, rev, group=None, formats=["txt",]):
# break early in case of missing configuration
self.assertTrue(os.path.exists(settings.IDSUBMIT_IDNITS_BINARY))
@ -67,15 +70,23 @@ class SubmitTests(TestCase):
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertEqual(len(q('input[type=file][name=txt]')), 1)
self.assertEqual(len(q('input[type=file][name=xml]')), 1)
# submit
txt_file = self.submission_txt_file(name, rev)
files = {}
for format in formats:
files[format] = self.submission_file(name, rev, group, format, "test_submission.%s" % format)
r = self.client.post(url, files)
if r.status_code != 302:
q = PyQuery(r.content)
print(q('div.has-error span.help-block div').text)
r = self.client.post(url,
dict(txt=txt_file))
self.assertEqual(r.status_code, 302)
status_url = r["Location"]
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))))
for format in formats:
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.%s" % (name, rev, format))))
self.assertEqual(Submission.objects.filter(name=name).count(), 1)
submission = Submission.objects.get(name=name)
self.assertTrue(re.search('\s+Summary:\s+0\s+errors|No nits found', submission.idnits_message))
@ -120,7 +131,7 @@ class SubmitTests(TestCase):
return confirm_url
def test_submit_new_wg(self):
def submit_new_wg(self, formats):
# submit new -> supply submitter info -> approve
draft = make_test_data()
@ -147,8 +158,9 @@ class SubmitTests(TestCase):
name = "draft-ietf-mars-testing-tests"
rev = "00"
group = "mars"
status_url = self.do_submission(name, rev)
status_url = self.do_submission(name, rev, group, formats)
# supply submitter info, then draft should be in and ready for approval
mailbox_before = len(outbox)
@ -211,7 +223,16 @@ class SubmitTests(TestCase):
self.assertTrue("ameschairman" in outbox[-1]["To"].lower())
self.assertTrue("marschairman" in outbox[-1]["To"].lower())
def test_submit_existing(self):
def test_submit_new_wg_txt(self):
self.submit_new_wg(["txt"])
def text_submit_new_wg_xml(self):
self.submit_new_wg(["xml"])
def text_submit_new_wg_txt_xml(self):
self.submit_new_wg(["txt", "xml"])
def submit_existing(self, formats):
# submit new revision of existing -> supply submitter info -> prev authors confirm
draft = make_test_data()
prev_author = draft.documentauthor_set.all()[0]
@ -241,13 +262,14 @@ class SubmitTests(TestCase):
name = draft.name
rev = "%02d" % (int(draft.rev) + 1)
group = draft.group
# write the old draft in a file so we can check it's moved away
old_rev = draft.rev
with open(os.path.join(self.repository_dir, "%s-%s.txt" % (name, old_rev)), 'w') as f:
f.write("a" * 2000)
status_url = self.do_submission(name, rev)
status_url = self.do_submission(name, rev, group, formats)
# supply submitter info, then previous authors get a confirmation email
mailbox_before = len(outbox)
@ -310,14 +332,24 @@ class SubmitTests(TestCase):
self.assertTrue(name in unicode(outbox[-1]))
self.assertTrue("mars" in unicode(outbox[-1]))
def test_submit_new_individual(self):
def test_submit_existing_txt(self):
self.submit_existing(["txt"])
def test_submit_existing_xml(self):
self.submit_existing(["xml"])
def test_submit_existing_txt_xml(self):
self.submit_existing(["txt", "xml"])
def submit_new_individual(self, formats):
# submit new -> supply submitter info -> confirm
draft = make_test_data()
name = "draft-authorname-testing-tests"
rev = "00"
group = None
status_url = self.do_submission(name, rev)
status_url = self.do_submission(name, rev, group, formats)
# supply submitter info, then draft should be be ready for email auth
mailbox_before = len(outbox)
@ -355,6 +387,15 @@ class SubmitTests(TestCase):
self.assertEqual(new_revision.type, "new_revision")
self.assertEqual(new_revision.by.name, "Submitter Name")
def test_submit_new_individual_txt(self):
self.submit_new_individual(["txt"])
def test_submit_new_individual_xml(self):
self.submit_new_individual(["xml"])
def test_submit_new_individual_txt_xml(self):
self.submit_new_individual(["txt", "xml"])
def test_submit_new_wg_with_dash(self):
make_test_data()
@ -579,35 +620,17 @@ class SubmitTests(TestCase):
name = "draft-ietf-mars-testing-tests"
rev = "00"
group = "mars"
txt_file = self.submission_txt_file(name, rev)
# the checks for other file types are currently embarrassingly
# dumb, so don't bother constructing proper XML/PS/PDF draft
# files
xml_file = StringIO('<?xml version="1.0" encoding="utf-8"?>\n<draft>This is XML</draft>')
xml_file.name = "somename.xml"
pdf_file = StringIO('%PDF-1.5\nThis is PDF')
pdf_file.name = "somename.pdf"
ps_file = StringIO('%!PS-Adobe-2.0\nThis is PostScript')
ps_file.name = "somename.ps"
r = self.client.post(urlreverse('submit_upload_submission'), dict(
txt=txt_file,
xml=xml_file,
pdf=pdf_file,
ps=ps_file,
))
self.assertEqual(r.status_code, 302)
self.do_submission(name, rev, group, ["txt", "xml", "ps", "pdf"])
self.assertEqual(Submission.objects.filter(name=name).count(), 1)
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))))
self.assertTrue(name in open(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))).read())
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))))
self.assertTrue('This is XML' in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read())
self.assertTrue(name in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read())
self.assertTrue('<?xml version="1.0" encoding="US-ASCII"?>' in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read())
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.pdf" % (name, rev))))
self.assertTrue('This is PDF' in open(os.path.join(self.staging_dir, u"%s-%s.pdf" % (name, rev))).read())
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.ps" % (name, rev))))

17
ietf/submit/utils.py
View file

@ -353,21 +353,24 @@ def cancel_submission(submission):
remove_submission_files(submission)
def rename_submission_files(submission, prev_rev, new_rev):
for ext in submission.file_types.split(','):
source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, prev_rev, ext))
dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, new_rev, ext))
from ietf.submit.forms import SubmissionUploadForm
for ext in SubmissionUploadForm.base_fields.keys():
source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, prev_rev, ext))
dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, new_rev, ext))
if os.path.exists(source):
os.rename(source, dest)
def move_files_to_repository(submission):
for ext in submission.file_types.split(','):
source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, submission.rev, ext))
dest = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, '%s-%s%s' % (submission.name, submission.rev, ext))
from ietf.submit.forms import SubmissionUploadForm
for ext in SubmissionUploadForm.base_fields.keys():
source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext))
dest = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext))
if os.path.exists(source):
os.rename(source, dest)
else:
if os.path.exists(dest):
log("Intended to move '%s' to '%s', but found source missing while destination exists.")
else:
elif ext in submission.file_types.split(','):
raise ValueError("Intended to move '%s' to '%s', but found source and destination missing.")
def remove_submission_files(submission):

86
ietf/submit/views.py
View file

@ -1,6 +1,7 @@
# Copyright The IETF Trust 2007, All Rights Reserved
import datetime
import os
import xml2rfc
from django.conf import settings
from django.core.urlresolvers import reverse as urlreverse
@ -8,45 +9,66 @@ from django.core.validators import validate_email, ValidationError
from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden
from django.shortcuts import get_object_or_404, redirect, render
import debug # pyflakes:ignore
from ietf.doc.models import Document, DocAlias
from ietf.doc.utils import prettify_std_name
from ietf.group.models import Group
from ietf.ietfauth.utils import has_role, role_required
from ietf.submit.forms import UploadForm, NameEmailForm, EditSubmissionForm, PreapprovalForm, ReplacesForm
from ietf.submit.forms import SubmissionUploadForm, NameEmailForm, EditSubmissionForm, PreapprovalForm, ReplacesForm
from ietf.submit.mail import send_full_url, send_approval_request_to_group, send_submission_confirmation, submission_confirmation_email_list, send_manual_post_request
from ietf.submit.models import Submission, Preapproval, DraftSubmissionStateName
from ietf.submit.utils import approvable_submissions_for_user, preapprovals_for_user, recently_approved_by_user
from ietf.submit.utils import check_idnits, found_idnits, validate_submission, create_submission_event
from ietf.submit.utils import post_submission, cancel_submission, rename_submission_files
from ietf.utils.accesstoken import generate_random_key, generate_access_token
from ietf.utils.draft import Draft
def upload_submission(request):
if request.method == 'POST':
try:
form = UploadForm(request, data=request.POST, files=request.FILES)
form = SubmissionUploadForm(request, data=request.POST, files=request.FILES)
if form.is_valid():
# save files
file_types = []
for ext in ['txt', 'pdf', 'xml', 'ps']:
authors = []
file_name = {}
abstract = None
file_size = None
for ext in form.fields.keys():
f = form.cleaned_data[ext]
if not f:
continue
file_types.append('.%s' % ext)
draft = form.parsed_draft
name = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (draft.filename, draft.revision, ext))
name = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (form.filename, form.revision, ext))
file_name[ext] = name
with open(name, 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
# check idnits
text_path = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (draft.filename, draft.revision))
idnits_message = check_idnits(text_path)
if form.cleaned_data['xml']:
if not form.cleaned_data['txt']:
try:
file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (form.filename, form.revision))
pagedwriter = xml2rfc.PaginatedTextRfcWriter(form.xmltree, quiet=True)
pagedwriter.write(file_name['txt'])
file_size = os.stat(file_name['txt']).st_size
except Exception as e:
raise ValidationError("Exception: %s" % e)
# Some meta-information, such as the page-count, can only
# be retrieved from the generated text file. Provide a
# parsed draft object to get at that kind of information.
with open(file_name['txt']) as txt_file:
form.parsed_draft = Draft(txt_file.read(), txt_file.name)
# extract author lines
authors = []
for author in draft.get_author_list():
else:
file_size = form.cleaned_data['txt'].size
if form.authors:
authors = form.authors
else:
# If we don't have an xml file, try to extract the
# relevant information from the text file
for author in form.parsed_draft.get_author_list():
full_name, first_name, middle_initial, last_name, name_suffix, email, company = author
line = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip()
@ -63,39 +85,51 @@ def upload_submission(request):
authors.append(line)
if form.abstract:
abstract = form.abstract
else:
abstract = form.parsed_draft.get_abstract()
# check idnits
idnits_message = check_idnits(file_name['txt'])
# save submission
try:
submission = Submission.objects.create(
state=DraftSubmissionStateName.objects.get(slug="uploaded"),
remote_ip=form.remote_ip,
name=draft.filename,
name=form.filename,
group=form.group,
title=draft.get_title(),
abstract=draft.get_abstract(),
rev=draft.revision,
pages=draft.get_pagecount(),
title=form.title,
abstract=abstract,
rev=form.revision,
pages=form.parsed_draft.get_pagecount(),
authors="\n".join(authors),
note="",
first_two_pages=''.join(draft.pages[:2]),
file_size=form.cleaned_data['txt'].size,
file_types=','.join(file_types),
first_two_pages=''.join(form.parsed_draft.pages[:2]),
file_size=file_size,
file_types=','.join(form.file_types),
submission_date=datetime.date.today(),
document_date=draft.get_creation_date(),
document_date=form.parsed_draft.get_creation_date(),
replaces="",
idnits_message=idnits_message,
)
except Exception as e:
import sys
sys.stderr.write("Exception: %s\n" % e)
create_submission_event(request, submission, desc="Uploaded submission")
return redirect("submit_submission_status_by_hash", submission_id=submission.pk, access_token=submission.access_token())
except IOError as e:
if "read error" in str(e): # The server got an IOError when trying to read POST data
form = UploadForm(request=request)
form = SubmissionUploadForm(request=request)
form._errors = {}
form._errors["__all__"] = form.error_class(["There was a failure receiving the complete form data -- please try again."])
else:
raise
else:
form = UploadForm(request=request)
form = SubmissionUploadForm(request=request)
return render(request, 'submit/upload_submission.html',
{'selected': 'index',

4
ietf/templates/registration/edit_profile.html
View file

@ -109,11 +109,11 @@
{{person_form.ascii_short|add_class:"form-control"}}
</div>
<div class="col-sm-7">
<div class="help-block">
<span class="help-block">
Example: A. Nonymous. Fill in this with initials and surname only if
taking the initials and surname of your ASCII name above produces an incorrect
initials-only form. (Blank is ok).
</div>
</span>
</div>
</div>
</div>

3
requirements.txt
View file

@ -4,6 +4,7 @@ coverage>=3.7.1
decorator>=3.4.0
defusedxml>=0.4.1 # for TastyPie when ussing xml; not a declared dependency
django-bootstrap3>=5.1.1
django-tastypie>=0.12.1
django-widget-tweaks>=1.3
docutils>=0.12
html5lib>=0.90
@ -20,5 +21,5 @@ python-memcached>=1.48 # for django.core.cache.backends.memcached
pytz>=2014.7
setuptools>=1.2
six>=1.8.0
django-tastypie>=0.12.1
wsgiref>=0.1.2
xml2rfc>=2.5.0