altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

feat: Process uploaded submissions asynchronously (#5580)

Browse source 
* fix: Use relative URL for submission status link

* refactor: Refactor/rename process_uploaded_submission async task

* feat: Add async task to process but not accept a submission

* feat: Replace upload_submission() with an async implementation (WIP)

* fix: Do not put Submission in "uploaded" state if an error occured

* refactor: Improve text/XML draft processing flow

* feat: Extract authors from text in async processing

* fix: Fix call signatures and abort submission on failed validation

* feat: Validate submission name format

* fix: Correctly validate emails from text submission

* fix: Clean up submission validation

* fix: Better display errors on upload_submission page

* feat: Reload submission status page when awaiting validation

* test: Fix call signatures; remove unused imports

* chore: Add type hint

* test: Update tests to match renamed task

* fix: Fix typo in error message

* test: Fix failing Api- and AsyncSubmissionTests

* Rename process_uploaded_submission to process_and_accept_...
* Remove outdated tests

Does not yet test new behavior.

* refactor: Break up submission_file() helper

* test: Refactor tests to run the async processing (wip)

* test: Drop test of bad PDF submission

The PDF submission field was removed, so no need to test it.

* test: Update more tests

* test: Bring back create_and_post_submission() and fix more tests

* fix: Drop to manual, don't cancel, on revision inconsistency

Fixes remaining failing SubmitTest tests

* style: Restyle upload_submission() with black

* test: Verify that async submission processing is invoked on upload

* test: Bring back old do_submission and fix tests

Properly separating the upload and async processing stages of submission
is a bigger refactoring than will fit right now. This better exercises
the submission pipeline.

* fix: Accept only XML for API submissions

* test: Test submission processing utilities

* feat: Improve status display for "validating" submissions

* chore: Remove obsolete code

* test: Update test to match amended text

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
This commit is contained in:
Jennifer Richards 2023-05-09 16:21:46 -04:00 committed by GitHub
parent b1c60efbb3
commit a0f6cdb661
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 858 additions and 551 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

144
ietf/static/js/draft-submit.js
View file

@ -1,69 +1,93 @@
$(document)
.ready(function () {
// fill in submitter info when an author button is clicked
$("form.idsubmit button.author")
.on("click", function () {
var name = $(this)
.data("name");
var email = $(this)
.data("email");
$(function () {
// fill in submitter info when an author button is clicked
$("form.idsubmit button.author")
.on("click", function () {
var name = $(this)
.data("name");
var email = $(this)
.data("email");
$(this)
.parents("form")
.find("input[name=submitter-name]")
.val(name || "");
$(this)
.parents("form")
.find("input[name=submitter-email]")
.val(email || "");
});
$(this)
.parents("form")
.find("input[name=submitter-name]")
.val(name || "");
$(this)
.parents("form")
.find("input[name=submitter-email]")
.val(email || "");
});
$("form.idsubmit")
.on("submit", function () {
if (this.submittedAlready)
return false;
else {
this.submittedAlready = true;
return true;
}
});
$("form.idsubmit")
.on("submit", function () {
if (this.submittedAlready)
return false;
else {
this.submittedAlready = true;
return true;
}
});
$("form.idsubmit #add-author")
.on("click", function () {
// clone the last author block and make it empty
var cloner = $("#cloner");
var next = cloner.clone();
next.find('input:not([type=hidden])')
.val('');
$("form.idsubmit #add-author")
.on("click", function () {
// clone the last author block and make it empty
var cloner = $("#cloner");
var next = cloner.clone();
next.find('input:not([type=hidden])')
.val('');
// find the author number
var t = next.children('h3')
.text();
var n = parseInt(t.replace(/\D/g, ''));
// find the author number
var t = next.children('h3')
.text();
var n = parseInt(t.replace(/\D/g, ''));
// change the number in attributes and text
next.find('*')
.each(function () {
var e = this;
$.each(['id', 'for', 'name', 'value'], function (i, v) {
if ($(e)
.attr(v)) {
$(e)
.attr(v, $(e)
.attr(v)
.replace(n - 1, n));
}
});
// change the number in attributes and text
next.find('*')
.each(function () {
var e = this;
$.each(['id', 'for', 'name', 'value'], function (i, v) {
if ($(e)
.attr(v)) {
$(e)
.attr(v, $(e)
.attr(v)
.replace(n - 1, n));
}
});
});
t = t.replace(n, n + 1);
next.children('h3')
.text(t);
t = t.replace(n, n + 1);
next.children('h3')
.text(t);
// move the cloner id to next and insert next into the DOM
cloner.removeAttr('id');
next.attr('id', 'cloner');
next.insertAfter(cloner);
// move the cloner id to next and insert next into the DOM
cloner.removeAttr('id');
next.attr('id', 'cloner');
next.insertAfter(cloner);
});
});
});
// Reload page periodically if the enableAutoReload checkbox is present and checked
const autoReloadSwitch = document.getElementById("enableAutoReload");
const timeSinceDisplay = document.getElementById("time-since-uploaded");
if (autoReloadSwitch) {
const autoReloadTime = 30000; // ms
let autoReloadTimeoutId;
autoReloadSwitch.parentElement.classList.remove("d-none");
timeSinceDisplay.classList.remove("d-none");
autoReloadTimeoutId = setTimeout(() => location.reload(), autoReloadTime);
autoReloadSwitch.addEventListener("change", (e) => {
if (e.currentTarget.checked) {
if (!autoReloadTimeoutId) {
autoReloadTimeoutId = setTimeout(() => location.reload(), autoReloadTime);
timeSinceDisplay.classList.remove("d-none");
}
} else {
if (autoReloadTimeoutId) {
clearTimeout(autoReloadTimeoutId);
autoReloadTimeoutId = null;
timeSinceDisplay.classList.add("d-none");
}
}
});
}
});

297
ietf/submit/forms.py
View file

@ -13,8 +13,8 @@ import xml2rfc
from contextlib import ExitStack
from email.utils import formataddr
from typing import Tuple
from unidecode import unidecode
from urllib.parse import urljoin
from django import forms
from django.conf import settings
@ -36,7 +36,6 @@ from ietf.message.models import Message
from ietf.name.models import FormalLanguageName, GroupTypeName
from ietf.submit.models import Submission, Preapproval
from ietf.submit.utils import validate_submission_name, validate_submission_rev, validate_submission_document_date, remote_ip
from ietf.submit.parsers.pdf_parser import PDFParser
from ietf.submit.parsers.plain_parser import PlainParser
from ietf.submit.parsers.xml_parser import XMLParser
from ietf.utils import log
@ -49,6 +48,9 @@ from ietf.utils.xmldraft import XMLDraft, XMLParseError
class SubmissionBaseUploadForm(forms.Form):
xml = forms.FileField(label='.xml format', required=True)
formats: Tuple[str, ...] = ('xml',) # allowed formats
base_formats: Tuple[str, ...] = ('xml',) # at least one of these is required
def __init__(self, request, *args, **kwargs):
super(SubmissionBaseUploadForm, self).__init__(*args, **kwargs)
@ -66,18 +68,11 @@ class SubmissionBaseUploadForm(forms.Form):
self.title = None
self.abstract = None
self.authors = []
self.parsed_draft = None
self.file_types = []
self.file_info = {} # indexed by file field name, e.g., 'txt', 'xml', ...
self.xml_version = None
# No code currently (14 Sep 2017) uses this class directly; it is
# only used through its subclasses. The two assignments below are
# set to trigger an exception if it is used directly only to make
# sure that adequate consideration is made if it is decided to use it
# directly in the future. Feel free to set these appropriately to
# avoid the exceptions in that case:
self.formats = None # None will raise an exception in clean() if this isn't changed in a subclass
self.base_formats = None # None will raise an exception in clean() if this isn't changed in a subclass
self._extracted_filenames_and_revisions = {}
def set_cutoff_warnings(self):
now = timezone.now()
@ -126,20 +121,17 @@ class SubmissionBaseUploadForm(forms.Form):
'The I-D submission tool will be reopened after %s (IETF-meeting local time).' % (cutoff_01_str, reopen_str))
self.shutdown = True
def clean_file(self, field_name, parser_class):
def _clean_file(self, field_name, parser_class):
f = self.cleaned_data[field_name]
if not f:
return f
self.file_info[field_name] = parser_class(f).critical_parse()
if self.file_info[field_name].errors:
raise forms.ValidationError(self.file_info[field_name].errors)
raise forms.ValidationError(self.file_info[field_name].errors, code="critical_error")
return f
def clean_xml(self):
return self.clean_file("xml", XMLParser)
def clean(self):
def format_messages(where, e, log_msgs):
m = str(e)
if m:
@ -148,38 +140,12 @@ class SubmissionBaseUploadForm(forms.Form):
import traceback
typ, val, tb = sys.exc_info()
m = traceback.format_exception(typ, val, tb)
m = [ l.replace('\n ', ':\n ') for l in m ]
msgs = [s for s in (["Error from xml2rfc (%s):" % (where,)] + m + log_msgs) if s]
m = [l.replace('\n ', ':\n ') for l in m]
msgs = [s for s in ([f"Error from xml2rfc ({where}):"] + m + log_msgs) if s]
return msgs
if self.shutdown and not has_role(self.request.user, "Secretariat"):
raise forms.ValidationError('The submission tool is currently shut down')
# check general submission rate thresholds before doing any more work
today = date_today()
self.check_submissions_thresholds(
"for the same submitter",
dict(remote_ip=self.remote_ip, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER, settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER_SIZE,
)
self.check_submissions_thresholds(
"across all submitters",
dict(submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
)
for ext in self.formats:
f = self.cleaned_data.get(ext, None)
if not f:
continue
self.file_types.append('.%s' % ext)
if not ('.txt' in self.file_types or '.xml' in self.file_types):
if not self.errors:
raise forms.ValidationError('Unexpected submission file types; found %s, but %s is required' % (', '.join(self.file_types), ' or '.join(self.base_formats)))
# Determine the draft name and revision. Try XML first.
if self.cleaned_data.get('xml'):
xml_file = self.cleaned_data.get('xml')
xml_file = self._clean_file("xml", XMLParser)
if xml_file:
tfn = None
with ExitStack() as stack:
@stack.callback
@ -204,86 +170,131 @@ class SubmissionBaseUploadForm(forms.Form):
xml_draft = XMLDraft(tfn)
except XMLParseError as e:
msgs = format_messages('xml', e, e.parser_msgs())
self.add_error('xml', msgs)
return
raise forms.ValidationError(msgs, code="xml_parse_error")
except Exception as e:
self.add_error('xml', f'Error parsing XML Internet-Draft: {e}')
return
raise forms.ValidationError(f"Error parsing XML Internet-Draft: {e}", code="parse_exception")
if not xml_draft.filename:
raise forms.ValidationError(
"Could not extract a valid Internet-Draft name from the XML. "
"Please make sure that the top-level <rfc/> "
"element has a docName attribute which provides the full Internet-Draft name including "
"revision number.",
code="parse_error_filename",
)
if not xml_draft.revision:
raise forms.ValidationError(
"Could not extract a valid Internet-Draft revision from the XML. "
"Please make sure that the top-level <rfc/> "
"element has a docName attribute which provides the full Internet-Draft name including "
"revision number.",
code="parse_error_revision",
)
self._extracted_filenames_and_revisions['xml'] = (xml_draft.filename, xml_draft.revision)
return xml_file
self.filename = xml_draft.filename
self.revision = xml_draft.revision
elif self.cleaned_data.get('txt'):
# no XML available, extract from the text if we have it
# n.b., this code path is unused until a subclass with a 'txt' field is created.
txt_file = self.cleaned_data['txt']
txt_file.seek(0)
bytes = txt_file.read()
try:
text = bytes.decode(self.file_info['txt'].charset)
self.parsed_draft = PlaintextDraft(text, txt_file.name)
self.filename = self.parsed_draft.filename
self.revision = self.parsed_draft.revision
except (UnicodeDecodeError, LookupError) as e:
self.add_error('txt', 'Failed decoding the uploaded file: "%s"' % str(e))
def clean(self):
if self.shutdown and not has_role(self.request.user, "Secretariat"):
raise forms.ValidationError('The submission tool is currently shut down')
rev_error = validate_submission_rev(self.filename, self.revision)
if rev_error:
raise forms.ValidationError(rev_error)
# check general submission rate thresholds before doing any more work
today = date_today()
self.check_submissions_thresholds(
"for the same submitter",
dict(remote_ip=self.remote_ip, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER, settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER_SIZE,
)
self.check_submissions_thresholds(
"across all submitters",
dict(submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
)
for ext in self.formats:
f = self.cleaned_data.get(ext, None)
if not f:
continue
self.file_types.append('.%s' % ext)
if not any(f".{bt}" in self.file_types for bt in self.base_formats):
if not self.errors:
raise forms.ValidationError(
"Unexpected submission file types; found {}, but {} is required".format(
", ".join(ft.lstrip(".") for ft in self.file_types),
" or ".join(self.base_formats),
)
)
# The following errors are likely noise if we have previous field
# errors:
if self.errors:
raise forms.ValidationError('')
# Check that all formats agree on draft name/rev
filename_from = None
for fmt, (extracted_name, extracted_rev) in self._extracted_filenames_and_revisions.items():
if self.filename is None:
filename_from = fmt
self.filename = extracted_name
self.revision = extracted_rev
elif self.filename != extracted_name:
raise forms.ValidationError(
{fmt: f"Extracted filename '{extracted_name}' does not match filename '{self.filename}' from {filename_from} format"},
code="filename_mismatch",
)
elif self.revision != extracted_rev:
raise forms.ValidationError(
{fmt: f"Extracted revision ({extracted_rev}) does not match revision from {filename_from} format ({self.revision})"},
code="revision_mismatch",
)
# Not expected to encounter missing filename/revision here because
# the individual fields should fail validation, but just in case
if not self.filename:
raise forms.ValidationError("Could not extract a valid Internet-Draft name from the upload. "
"To fix this in a text upload, please make sure that the full Internet-Draft name including "
"revision number appears centered on its own line below the document title on the "
"first page. In an xml upload, please make sure that the top-level <rfc/> "
"element has a docName attribute which provides the full Internet-Draft name including "
"revision number.")
raise forms.ValidationError(
"Unable to extract a filename from any uploaded format.",
code="no_filename",
)
if not self.revision:
raise forms.ValidationError("Could not extract a valid Internet-Draft revision from the upload. "
"To fix this in a text upload, please make sure that the full Internet-Draft name including "
"revision number appears centered on its own line below the document title on the "
"first page. In an xml upload, please make sure that the top-level <rfc/> "
"element has a docName attribute which provides the full Internet-Draft name including "
"revision number.")
raise forms.ValidationError(
"Unable to extract a revision from any uploaded format.",
code="no_revision",
)
name_error = validate_submission_name(self.filename)
if name_error:
raise forms.ValidationError(name_error)
rev_error = validate_submission_rev(self.filename, self.revision)
if rev_error:
raise forms.ValidationError(rev_error)
self.check_for_old_uppercase_collisions(self.filename)
if self.cleaned_data.get('txt') or self.cleaned_data.get('xml'):
# check group
self.group = self.deduce_group(self.filename)
# check existing
existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel", "waiting-for-draft"))
if existing:
raise forms.ValidationError(
format_html(
'A submission with same name and revision is currently being processed. <a href="{}">Check the status here.</a>',
urljoin(
settings.IDTRACKER_BASE_URL,
urlreverse("ietf.submit.views.submission_status", kwargs={'submission_id': existing[0].pk}),
)
)
# check group
self.group = self.deduce_group(self.filename)
# check existing
existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel", "waiting-for-draft"))
if existing:
raise forms.ValidationError(
format_html(
'A submission with same name and revision is currently being processed. <a href="{}">Check the status here.</a>',
urlreverse("ietf.submit.views.submission_status", kwargs={'submission_id': existing[0].pk}),
)
# cut-off
if self.revision == '00' and self.in_first_cut_off:
raise forms.ValidationError(mark_safe(self.cutoff_warning))
# check thresholds that depend on the draft / group
self.check_submissions_thresholds(
"for the Internet-Draft %s" % self.filename,
dict(name=self.filename, rev=self.revision, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
)
if self.group:
self.check_submissions_thresholds(
"for the group \"%s\"" % (self.group.acronym),
dict(group=self.group, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_GROUP, settings.IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE,
)
# cut-off
if self.revision == '00' and self.in_first_cut_off:
raise forms.ValidationError(mark_safe(self.cutoff_warning))
# check thresholds that depend on the draft / group
self.check_submissions_thresholds(
"for the Internet-Draft %s" % self.filename,
dict(name=self.filename, rev=self.revision, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
)
if self.group:
self.check_submissions_thresholds(
"for the group \"%s\"" % (self.group.acronym),
dict(group=self.group, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_GROUP, settings.IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE,
)
return super().clean()
@staticmethod
@ -613,26 +624,6 @@ class DeprecatedSubmissionBaseUploadForm(SubmissionBaseUploadForm):
return super().clean()
class SubmissionManualUploadForm(DeprecatedSubmissionBaseUploadForm):
xml = forms.FileField(label='.xml format', required=False) # xml field with required=False instead of True
txt = forms.FileField(label='.txt format', required=False)
# We won't permit html upload until we can verify that the content
# reasonably matches the text and/or xml upload. Till then, we generate
# html for version 3 xml submissions.
# html = forms.FileField(label='.html format', required=False)
pdf = forms.FileField(label='.pdf format', required=False)
def __init__(self, request, *args, **kwargs):
super(SubmissionManualUploadForm, self).__init__(request, *args, **kwargs)
self.formats = settings.IDSUBMIT_FILE_TYPES
self.base_formats = ['txt', 'xml', ]
def clean_txt(self):
return self.clean_file("txt", PlainParser)
def clean_pdf(self):
return self.clean_file("pdf", PDFParser)
class DeprecatedSubmissionAutoUploadForm(DeprecatedSubmissionBaseUploadForm):
"""Full-service upload form, replaced by the asynchronous version"""
user = forms.EmailField(required=True)
@ -642,17 +633,50 @@ class DeprecatedSubmissionAutoUploadForm(DeprecatedSubmissionBaseUploadForm):
self.formats = ['xml', ]
self.base_formats = ['xml', ]
class SubmissionManualUploadForm(SubmissionBaseUploadForm):
txt = forms.FileField(label='.txt format', required=False)
formats = SubmissionBaseUploadForm.formats + ('txt',)
base_formats = SubmissionBaseUploadForm.base_formats + ('txt',)
def __init__(self, request, *args, **kwargs):
super().__init__(request, *args, **kwargs)
self.fields['xml'].required = False
def clean_txt(self):
txt_file = self._clean_file("txt", PlainParser)
if txt_file is not None:
bytes = txt_file.read()
try:
text = bytes.decode(self.file_info["txt"].charset)
parsed_draft = PlaintextDraft(text, txt_file.name)
self._extracted_filenames_and_revisions["txt"] = (parsed_draft.filename, parsed_draft.revision)
except (UnicodeDecodeError, LookupError) as e:
raise forms.ValidationError(f'Failed decoding the uploaded file: "{str(e)}"', code="decode_failed")
if not parsed_draft.filename:
raise forms.ValidationError(
"Could not extract a valid Internet-Draft name from the plaintext. "
"Please make sure that the full Internet-Draft name including "
"revision number appears centered on its own line below the document title on the "
"first page.",
code="parse_error_filename",
)
if not parsed_draft.revision:
raise forms.ValidationError(
"Could not extract a valid Internet-Draft revision from the plaintext. "
"Please make sure that the full Internet-Draft name including "
"revision number appears centered on its own line below the document title on the "
"first page.",
code="parse_error_revision",
)
return txt_file
class SubmissionAutoUploadForm(SubmissionBaseUploadForm):
user = forms.EmailField(required=True)
replaces = forms.CharField(required=False, max_length=1000, strip=True)
def __init__(self, request, *args, **kwargs):
super().__init__(request, *args, **kwargs)
self.formats = ['xml', ]
self.base_formats = ['xml', ]
def clean(self):
super().clean()
cleaned_data = super().clean()
# Clean the replaces field after the rest of the cleaning so we know the name of the
# uploaded draft via self.filename
@ -692,6 +716,7 @@ class SubmissionAutoUploadForm(SubmissionBaseUploadForm):
alias.name + " is approved by the IESG and cannot be replaced"
),
)
return cleaned_data
class NameEmailForm(forms.Form):

13
ietf/submit/tasks.py
View file

@ -9,7 +9,8 @@ from django.conf import settings
from django.utils import timezone
from ietf.submit.models import Submission
from ietf.submit.utils import cancel_submission, create_submission_event, process_uploaded_submission
from ietf.submit.utils import (cancel_submission, create_submission_event, process_uploaded_submission,
process_and_accept_uploaded_submission)
from ietf.utils import log
@ -23,6 +24,16 @@ def process_uploaded_submission_task(submission_id):
process_uploaded_submission(submission)
@shared_task
def process_and_accept_uploaded_submission_task(submission_id):
try:
submission = Submission.objects.get(pk=submission_id)
except Submission.DoesNotExist:
log.log(f'process_uploaded_submission_task called for missing submission_id={submission_id}')
else:
process_and_accept_uploaded_submission(submission)
@shared_task
def cancel_stale_submissions():
now = timezone.now()

0
ietf/submit/test_submission.nonascii → ietf/submit/test_submission.nonascii.txt
View file

2
ietf/submit/test_submission_invalid_yang.txt
View file

@ -2,7 +2,7 @@
Network Working Group A. Name
Network Working Group %(firstpagename)37s
Internet-Draft Test Centre Inc.
Intended status: Informational %(month)s %(year)s
Expires: %(expiration)s

422
ietf/submit/tests.py
View file

@ -5,13 +5,14 @@
import datetime
import email
import io
import mock
import os
import re
import sys
import mock
from io import StringIO
from pyquery import PyQuery
from typing import Tuple
from pathlib import Path
@ -28,7 +29,9 @@ import debug # pyflakes:ignore
from ietf.submit.utils import (expirable_submissions, expire_submission, find_submission_filenames,
post_submission, validate_submission_name, validate_submission_rev,
process_uploaded_submission, SubmissionError, process_submission_text)
process_and_accept_uploaded_submission, SubmissionError, process_submission_text,
process_submission_xml, process_uploaded_submission,
process_and_validate_submission)
from ietf.doc.factories import (DocumentFactory, WgDraftFactory, IndividualDraftFactory, IndividualRfcFactory,
ReviewFactory, WgRfcFactory)
from ietf.doc.models import ( Document, DocAlias, DocEvent, State,
@ -47,7 +50,7 @@ from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactor
from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm
from ietf.submit.models import Submission, Preapproval, SubmissionExtResource
from ietf.submit.mail import add_submission_email, process_response_email
from ietf.submit.tasks import cancel_stale_submissions, process_uploaded_submission_task
from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task
from ietf.utils.accesstoken import generate_access_token
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.models import VersionInfo
@ -91,7 +94,28 @@ class BaseSubmitTestCase(TestCase):
def archive_dir(self):
return settings.INTERNET_DRAFT_ARCHIVE_DIR
def submission_file(name_in_doc, name_in_post, group, templatename, author=None, email=None, title=None, year=None, ascii=True):
def post_to_upload_submission(self, *args, **kwargs):
"""POST to the upload_submission endpoint
Use this instead of directly POSTing to be sure that the appropriate celery
tasks would be queued (but are not actually queued during testing)
"""
# Mock task so we can check that it's called without actually submitting a celery task.
# Also mock on_commit() because otherwise the test transaction prevents the call from
# ever being made.
with mock.patch("ietf.submit.views.process_uploaded_submission_task") as mocked_task:
with mock.patch("ietf.submit.views.transaction.on_commit", side_effect=lambda x: x()):
response = self.client.post(*args, **kwargs)
if response.status_code == 302:
# A 302 indicates we're being redirected to the status page, meaning the upload
# was accepted. Check that the task would have been queued.
self.assertTrue(mocked_task.delay.called)
else:
self.assertFalse(mocked_task.delay.called)
return response
def submission_file_contents(name_in_doc, group, templatename, author=None, email=None, title=None, year=None, ascii=True):
_today = date_today()
# construct appropriate text draft
f = io.open(os.path.join(settings.BASE_DIR, "submit", templatename))
@ -128,10 +152,18 @@ def submission_file(name_in_doc, name_in_post, group, templatename, author=None,
email=email,
title=title,
)
return submission_text, author
def submission_file(name_in_doc, name_in_post, group, templatename, author=None, email=None, title=None, year=None, ascii=True):
submission_text, author = submission_file_contents(
name_in_doc, group, templatename, author, email, title, year, ascii
)
file = StringIO(submission_text)
file.name = name_in_post
return file, author
def create_draft_submission_with_rev_mismatch(rev='01'):
"""Create a draft and submission with mismatched version
@ -172,31 +204,44 @@ class SubmitTests(BaseSubmitTestCase):
# Submit views assume there is a "next" IETF to look for cutoff dates against
MeetingFactory(type_id='ietf', date=date_today()+datetime.timedelta(days=180))
def create_and_post_submission(self, name, rev, author, group=None, formats=("txt",), base_filename=None):
def create_and_post_submission(self, name, rev, author, group=None, formats=("txt",), base_filename=None, ascii=True):
"""Helper to create and post a submission
If base_filename is None, defaults to 'test_submission'
If base_filename is None, defaults to 'test_submission'.
"""
url = urlreverse('ietf.submit.views.upload_submission')
files = dict()
for format in formats:
fn = '.'.join((base_filename or 'test_submission', format))
files[format], __ = submission_file(f'{name}-{rev}', f'{name}-{rev}.{format}', group, fn, author=author)
files[format], __ = submission_file(f'{name}-{rev}', f'{name}-{rev}.{format}', group, fn, author=author, ascii=ascii)
r = self.client.post(url, files)
if r.status_code != 302:
r = self.post_to_upload_submission(url, files)
if r.status_code == 302:
# A redirect means the upload was accepted and queued for processing
process_submission = True
last_submission = Submission.objects.order_by("-pk").first()
self.assertEqual(last_submission.state_id, "validating")
else:
process_submission = False
q = PyQuery(r.content)
print(q('div.invalid-feedback').text())
self.assertNoFormPostErrors(r, ".invalid-feedback,.alert-danger")
for format in formats:
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, format))))
if format == 'xml':
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, 'html'))))
# Now process the submission like the task would do
if process_submission:
process_uploaded_submission(Submission.objects.order_by('-pk').first())
for format in formats:
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, format))))
if format == 'xml':
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, 'html'))))
return r
def do_submission(self, name, rev, group=None, formats=["txt",], author=None):
def do_submission(self, name, rev, group=None, formats: Tuple[str, ...]=("txt",), author=None, base_filename=None, ascii=True):
"""Simulate uploading a draft and waiting for validation results
Returns the "full access" status URL and the author associated with the submitted draft.
"""
# break early in case of missing configuration
self.assertTrue(os.path.exists(settings.IDSUBMIT_IDNITS_BINARY))
@ -211,7 +256,9 @@ class SubmitTests(BaseSubmitTestCase):
# submit
if author is None:
author = PersonFactory()
r = self.create_and_post_submission(name, rev, author, group, formats)
r = self.create_and_post_submission(
name=name, rev=rev, author=author, group=group, formats=formats, base_filename=base_filename, ascii=ascii
)
status_url = r["Location"]
self.assertEqual(Submission.objects.filter(name=name).count(), 1)
@ -223,8 +270,10 @@ class SubmitTests(BaseSubmitTestCase):
sys.stderr.write("Author initials: %s\n" % author.initials())
self.assertEqual(len(submission.authors), 1)
a = submission.authors[0]
self.assertEqual(a["name"], author.ascii_name())
self.assertEqual(a["email"], author.email().address.lower())
if ascii:
self.assertEqual(a["name"], author.ascii_name())
if author.email():
self.assertEqual(a["email"], author.email().address.lower())
self.assertEqual(a["affiliation"], "Test Centre Inc.")
self.assertEqual(a["country"], "UK")
@ -1262,11 +1311,8 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_new_wg_with_dash(self):
group = Group.objects.create(acronym="mars-special", name="Mars Special", type_id="wg", state_id="active")
name = "draft-ietf-%s-testing-tests" % group.acronym
self.do_submission(name, "00")
self.create_and_post_submission(name=name, rev="00", author=PersonFactory())
self.assertEqual(Submission.objects.get(name=name).group.acronym, group.acronym)
def test_submit_new_wg_v2_country_only(self):
@ -1292,19 +1338,15 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_new_irtf(self):
group = Group.objects.create(acronym="saturnrg", name="Saturn", type_id="rg", state_id="active")
name = "draft-irtf-%s-testing-tests" % group.acronym
self.do_submission(name, "00")
self.assertEqual(Submission.objects.get(name=name).group.acronym, group.acronym)
self.assertEqual(Submission.objects.get(name=name).group.type_id, group.type_id)
self.create_and_post_submission(name=name, rev="00", author=PersonFactory())
submission = Submission.objects.get(name=name)
self.assertEqual(submission.group.acronym, group.acronym)
self.assertEqual(submission.group.type_id, group.type_id)
def test_submit_new_iab(self):
name = "draft-iab-testing-tests"
self.do_submission(name, "00")
self.create_and_post_submission(name=name, rev="00", author=PersonFactory())
self.assertEqual(Submission.objects.get(name=name).group.acronym, "iab")
def test_cancel_submission(self):
@ -1514,7 +1556,7 @@ class SubmitTests(BaseSubmitTestCase):
rev = "00"
group = "mars"
self.do_submission(name, rev, group, ["txt", "xml", "pdf"])
self.do_submission(name, rev, group, ["txt", "xml"])
self.assertEqual(Submission.objects.filter(name=name).count(), 1)
@ -1523,8 +1565,6 @@ class SubmitTests(BaseSubmitTestCase):
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))))
self.assertTrue(name in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read())
self.assertTrue('<?xml version="1.0" encoding="UTF-8"?>' in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read())
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))))
self.assertTrue('This is PDF' in io.open(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))).read())
def test_expire_submissions(self):
s = Submission.objects.create(name="draft-ietf-mars-foo",
@ -1630,7 +1670,7 @@ class SubmitTests(BaseSubmitTestCase):
for format in formats:
files[format], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.bad', group, "test_submission.bad")
r = self.client.post(url, files)
r = self.post_to_upload_submission(url, files)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
@ -1649,14 +1689,9 @@ class SubmitTests(BaseSubmitTestCase):
files[format], author = submission_file(name_in_doc, name_in_post, group, "test_submission.%s" % format)
files[format].name = name_in_post
r = self.client.post(url, files)
r = self.post_to_upload_submission(url, files)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertTrue(len(q("form .invalid-feedback")) > 0)
m = q('div.invalid-feedback').text()
return r, q, m
return r
def test_submit_bad_file_txt(self):
r, q, m = self.submit_bad_file("some name", ["txt"])
@ -1666,15 +1701,15 @@ class SubmitTests(BaseSubmitTestCase):
self.assertIn('document does not contain a legitimate name', m)
def test_submit_bad_doc_name(self):
r, q, m = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo.dot-bar", name_in_post="draft-foo.dot-bar", formats=["txt"])
self.assertIn('contains a disallowed character with byte code: 46', m)
r = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo.dot-bar", name_in_post="draft-foo.dot-bar", formats=["txt"])
self.assertContains(r, "contains a disallowed character with byte code: 46")
# This actually is allowed by the existing code. A significant rework of the validation mechanics is needed.
# r, q, m = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo-bar-00.txt", name_in_post="draft-foo-bar-00.txt", formats=["txt"])
# self.assertIn('Did you include a filename extension in the name by mistake?', m)
r, q, m = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo-bar-00.xml", name_in_post="draft-foo-bar-00.xml", formats=["xml"])
self.assertIn('Did you include a filename extension in the name by mistake?', m)
r, q, m = self.submit_bad_doc_name_with_ext(name_in_doc="../malicious-name-in-content-00", name_in_post="../malicious-name-in-post-00.xml", formats=["xml"])
self.assertIn('Did you include a filename extension in the name by mistake?', m)
r = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo-bar-00.xml", name_in_post="draft-foo-bar-00.xml", formats=["xml"])
self.assertContains(r, "Could not extract a valid Internet-Draft revision from the XML")
r = self.submit_bad_doc_name_with_ext(name_in_doc="../malicious-name-in-content-00", name_in_post="../malicious-name-in-post-00.xml", formats=["xml"])
self.assertContains(r, "Did you include a filename extension in the name by mistake?")
def test_submit_bad_file_xml(self):
r, q, m = self.submit_bad_file("some name", ["xml"])
@ -1682,12 +1717,6 @@ class SubmitTests(BaseSubmitTestCase):
self.assertIn('Expected the XML file to have extension ".xml"', m)
self.assertIn('Expected an XML file of type "application/xml"', m)
def test_submit_bad_file_pdf(self):
r, q, m = self.submit_bad_file("some name", ["pdf"])
self.assertIn('Invalid characters were found in the name', m)
self.assertIn('Expected the PDF file to have extension ".pdf"', m)
self.assertIn('Expected an PDF file of type "application/pdf"', m)
def test_submit_file_in_archive(self):
name = "draft-authorname-testing-file-exists"
rev = '00'
@ -1711,7 +1740,7 @@ class SubmitTests(BaseSubmitTestCase):
with io.open(fn, 'w') as f:
f.write("a" * 2000)
files[format], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.{format}', group, "test_submission.%s" % format)
r = self.client.post(url, files)
r = self.post_to_upload_submission(url, files)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
@ -1722,25 +1751,11 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_nonascii_name(self):
name = "draft-authorname-testing-nonascii"
rev = "00"
group = None
# get
url = urlreverse('ietf.submit.views.upload_submission')
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
# submit
#author = PersonFactory(name=u"Jörgen Nilsson".encode('latin1'))
user = UserFactory(first_name="Jörgen", last_name="Nilsson")
author = PersonFactory(user=user)
file, __ = submission_file(f'{name}-{rev}', f'{name}-{rev}.txt', group, "test_submission.nonascii", author=author, ascii=False)
files = {"txt": file }
r = self.client.post(url, files)
self.assertEqual(r.status_code, 302)
status_url = r["Location"]
status_url, _ = self.do_submission(name=name, rev=rev, author=author, base_filename="test_submission.nonascii", ascii=False)
r = self.client.get(status_url)
q = PyQuery(r.content)
m = q('p.alert-warning').text()
@ -1750,19 +1765,12 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_missing_author_email(self):
name = "draft-authorname-testing-noemail"
rev = "00"
group = None
author = PersonFactory()
for e in author.email_set.all():
e.delete()
files = {"txt": submission_file(f'{name}-{rev}', f'{name}-{rev}.txt', group, "test_submission.txt", author=author, ascii=True)[0] }
# submit
url = urlreverse('ietf.submit.views.upload_submission')
r = self.client.post(url, files)
self.assertEqual(r.status_code, 302)
status_url = r["Location"]
status_url, _ = self.do_submission(name=name, rev=rev, author=author)
r = self.client.get(status_url)
q = PyQuery(r.content)
m = q('p.text-danger').text()
@ -1773,20 +1781,13 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_bad_author_email(self):
name = "draft-authorname-testing-bademail"
rev = "00"
group = None
author = PersonFactory()
email = author.email_set.first()
email.address = '@bad.email'
email.save()
files = {"xml": submission_file(f'{name}-{rev}',f'{name}-{rev}.xml', group, "test_submission.xml", author=author, ascii=False)[0] }
# submit
url = urlreverse('ietf.submit.views.upload_submission')
r = self.client.post(url, files)
self.assertEqual(r.status_code, 302)
status_url = r["Location"]
status_url, _ = self.do_submission(name=name, rev=rev, author=author, formats=('xml',))
r = self.client.get(status_url)
q = PyQuery(r.content)
m = q('p.text-danger').text()
@ -1797,15 +1798,8 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_invalid_yang(self):
name = "draft-yang-testing-invalid"
rev = "00"
group = None
# submit
files = {"txt": submission_file(f'{name}-{rev}', f'{name}-{rev}.txt', group, "test_submission_invalid_yang.txt")[0] }
url = urlreverse('ietf.submit.views.upload_submission')
r = self.client.post(url, files)
self.assertEqual(r.status_code, 302)
status_url = r["Location"]
status_url, _ = self.do_submission(name=name, rev=rev, base_filename="test_submission_invalid_yang")
r = self.client.get(status_url)
q = PyQuery(r.content)
#
@ -2693,7 +2687,7 @@ Subject: test
for format in formats:
files[format], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.{format}', group, "test_submission.%s" % format)
r = self.client.post(url, files)
r = self.post_to_upload_submission(url, files)
if r.status_code != 302:
q = PyQuery(r.content)
print(q('div.invalid-feedback span.form-text div').text())
@ -2742,6 +2736,8 @@ Subject: test
@mock.patch.object(transaction, 'on_commit', lambda x: x())
@override_settings(IDTRACKER_BASE_URL='https://datatracker.example.com')
class ApiSubmissionTests(BaseSubmitTestCase):
TASK_TO_MOCK = "ietf.submit.views.process_and_accept_uploaded_submission_task"
def test_upload_draft(self):
"""api_submission accepts a submission and queues it for processing"""
url = urlreverse('ietf.submit.views.api_submission')
@ -2750,7 +2746,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'xml': xml,
'user': author.user.username,
}
with mock.patch('ietf.submit.views.process_uploaded_submission_task') as mock_task:
with mock.patch(self.TASK_TO_MOCK) as mock_task:
r = self.client.post(url, data)
self.assertEqual(r.status_code, 200)
response = r.json()
@ -2788,7 +2784,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'replaces': existing_draft.name,
}
# mock out the task so we don't call to celery during testing!
with mock.patch('ietf.submit.views.process_uploaded_submission_task'):
with mock.patch(self.TASK_TO_MOCK):
r = self.client.post(url, data)
self.assertEqual(r.status_code, 200)
submission = Submission.objects.last()
@ -2806,7 +2802,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'xml': xml,
'user': 'i.dont.exist@nowhere.example.com',
}
with mock.patch('ietf.submit.views.process_uploaded_submission_task') as mock_task:
with mock.patch(self.TASK_TO_MOCK) as mock_task:
r = self.client.post(url, data)
self.assertEqual(r.status_code, 400)
response = r.json()
@ -2820,7 +2816,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'xml': xml,
'user': author.user.username,
}
with mock.patch('ietf.submit.views.process_uploaded_submission_task') as mock_task:
with mock.patch(self.TASK_TO_MOCK) as mock_task:
r = self.client.post(url, data)
self.assertEqual(r.status_code, 400)
response = r.json()
@ -2834,7 +2830,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'xml': xml,
'user': author.user.username,
}
with mock.patch('ietf.submit.views.process_uploaded_submission_task') as mock_task:
with mock.patch(self.TASK_TO_MOCK) as mock_task:
r = self.client.post(url, data)
self.assertEqual(r.status_code, 400)
response = r.json()
@ -2850,7 +2846,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'xml': xml,
'user': author.user.username,
}
with mock.patch('ietf.submit.views.process_uploaded_submission_task') as mock_task:
with mock.patch(self.TASK_TO_MOCK) as mock_task:
r = self.client.post(url, data)
self.assertEqual(r.status_code, 400)
response = r.json()
@ -3105,8 +3101,8 @@ class SubmissionUploadFormTests(BaseSubmitTestCase):
class AsyncSubmissionTests(BaseSubmitTestCase):
"""Tests of async submission-related tasks"""
def test_process_uploaded_submission(self):
"""process_uploaded_submission should properly process a submission"""
def test_process_and_accept_uploaded_submission(self):
"""process_and_accept_uploaded_submission should properly process a submission"""
_today = date_today()
xml, author = submission_file('draft-somebody-test-00', 'draft-somebody-test-00.xml', None, 'test_submission.xml')
xml_data = xml.read()
@ -3126,7 +3122,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
self.assertFalse(txt_path.exists())
html_path = xml_path.with_suffix('.html')
self.assertFalse(html_path.exists())
process_uploaded_submission(submission)
process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'auth', 'accepted submission should be in auth state')
@ -3144,8 +3140,8 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
self.assertEqual(submission.file_size, os.stat(txt_path).st_size)
self.assertIn('Completed submission validation checks', submission.submissionevent_set.last().desc)
def test_process_uploaded_submission_invalid(self):
"""process_uploaded_submission should properly process an invalid submission"""
def test_process_and_accept_uploaded_submission_invalid(self):
"""process_and_accept_uploaded_submission should properly process an invalid submission"""
xml, author = submission_file('draft-somebody-test-00', 'draft-somebody-test-00.xml', None, 'test_submission.xml')
xml_data = xml.read()
xml.close()
@ -3166,7 +3162,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
with xml_path.open('w') as f:
f.write(xml_data)
process_uploaded_submission(submission)
process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel')
self.assertIn('not one of the document authors', submission.submissionevent_set.last().desc)
@ -3182,10 +3178,10 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
with xml_path.open('w') as f:
f.write(re.sub(r'<email>.*</email>', '', xml_data))
process_uploaded_submission(submission)
process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel')
self.assertIn('Missing email address', submission.submissionevent_set.last().desc)
self.assertIn('Email address not found for all authors', submission.submissionevent_set.last().desc)
# no title
submission = SubmissionFactory(
@ -3198,7 +3194,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
with xml_path.open('w') as f:
f.write(re.sub(r'<title>.*</title>', '<title></title>', xml_data))
process_uploaded_submission(submission)
process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel')
self.assertIn('Could not extract a valid title', submission.submissionevent_set.last().desc)
@ -3214,10 +3210,10 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-different-name-00.xml'
with xml_path.open('w') as f:
f.write(xml_data)
process_uploaded_submission(submission)
process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel')
self.assertIn('Internet-Draft filename disagrees', submission.submissionevent_set.last().desc)
self.assertIn('Submission rejected: XML Internet-Draft filename', submission.submissionevent_set.last().desc)
# rev mismatch
submission = SubmissionFactory(
@ -3230,10 +3226,10 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-01.xml'
with xml_path.open('w') as f:
f.write(xml_data)
process_uploaded_submission(submission)
process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel')
self.assertIn('revision disagrees', submission.submissionevent_set.last().desc)
self.assertIn('Submission rejected: XML Internet-Draft revision', submission.submissionevent_set.last().desc)
# not xml
submission = SubmissionFactory(
@ -3246,7 +3242,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.txt'
with txt_path.open('w') as f:
f.write(txt_data)
process_uploaded_submission(submission)
process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel')
self.assertIn('Only XML Internet-Draft submissions', submission.submissionevent_set.last().desc)
@ -3263,7 +3259,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
with xml_path.open('w') as f:
f.write(xml_data)
with mock.patch('ietf.submit.utils.process_submission_xml') as mock_proc_xml:
process_uploaded_submission(submission)
process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertFalse(mock_proc_xml.called, 'Should not process submission not in "validating" state')
self.assertEqual(submission.state_id, 'uploaded', 'State should not be changed')
@ -3291,80 +3287,192 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
symbol='x',
)
):
process_uploaded_submission(submission)
process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel')
self.assertIn('fake failure', submission.submissionevent_set.last().desc)
@mock.patch('ietf.submit.tasks.process_uploaded_submission')
def test_process_uploaded_submission_task(self, mock_method):
"""process_uploaded_submission_task task should properly call its method"""
@mock.patch('ietf.submit.tasks.process_and_accept_uploaded_submission')
def test_process_and_accept_uploaded_submission_task(self, mock_method):
"""process_and_accept_uploaded_submission_task task should properly call its method"""
s = SubmissionFactory()
process_uploaded_submission_task(s.pk)
process_and_accept_uploaded_submission_task(s.pk)
self.assertEqual(mock_method.call_count, 1)
self.assertEqual(mock_method.call_args.args, (s,))
@mock.patch('ietf.submit.tasks.process_uploaded_submission')
def test_process_uploaded_submission_task_ignores_invalid_id(self, mock_method):
"""process_uploaded_submission_task should ignore an invalid submission_id"""
@mock.patch('ietf.submit.tasks.process_and_accept_uploaded_submission')
def test_process_and_accept_uploaded_submission_task_ignores_invalid_id(self, mock_method):
"""process_and_accept_uploaded_submission_task should ignore an invalid submission_id"""
SubmissionFactory() # be sure there is a Submission
bad_pk = 9876
self.assertEqual(Submission.objects.filter(pk=bad_pk).count(), 0)
process_uploaded_submission_task(bad_pk)
process_and_accept_uploaded_submission_task(bad_pk)
self.assertEqual(mock_method.call_count, 0)
def test_process_submission_text_consistency_checks(self):
"""process_submission_text should check draft metadata against submission"""
submission = SubmissionFactory(
name='draft-somebody-test',
rev='00',
title='Correct Draft Title',
def test_process_submission_xml(self):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / "draft-somebody-test-00.xml"
xml, _ = submission_file(
"draft-somebody-test-00",
"draft-somebody-test-00.xml",
None,
"test_submission.xml",
title="Correct Draft Title",
)
txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.txt'
xml_path.write_text(xml.read())
output = process_submission_xml("draft-somebody-test", "00")
self.assertEqual(output["filename"], "draft-somebody-test")
self.assertEqual(output["rev"], "00")
self.assertEqual(output["title"], "Correct Draft Title")
self.assertIsNone(output["abstract"])
self.assertEqual(len(output["authors"]), 1) # not checking in detail, parsing is unreliable
self.assertIsNone(output["document_date"])
self.assertIsNone(output["pages"])
self.assertIsNone(output["words"])
self.assertIsNone(output["first_two_pages"])
self.assertIsNone(output["file_size"])
self.assertIsNone(output["formal_languages"])
self.assertEqual(output["xml_version"], "3")
# name mismatch
xml, _ = submission_file(
"draft-somebody-wrong-name-00", # name that appears in the file
"draft-somebody-test-00.xml",
None,
"test_submission.xml",
title="Correct Draft Title",
)
xml_path.write_text(xml.read())
with self.assertRaisesMessage(SubmissionError, "disagrees with submission filename"):
process_submission_xml("draft-somebody-test", "00")
# rev mismatch
xml, _ = submission_file(
"draft-somebody-test-01", # name that appears in the file
"draft-somebody-test-00.xml",
None,
"test_submission.xml",
title="Correct Draft Title",
)
xml_path.write_text(xml.read())
with self.assertRaisesMessage(SubmissionError, "disagrees with submission revision"):
process_submission_xml("draft-somebody-test", "00")
# missing title
xml, _ = submission_file(
"draft-somebody-test-00", # name that appears in the file
"draft-somebody-test-00.xml",
None,
"test_submission.xml",
title="",
)
xml_path.write_text(xml.read())
with self.assertRaisesMessage(SubmissionError, "Could not extract a valid title"):
process_submission_xml("draft-somebody-test", "00")
def test_process_submission_text(self):
txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / "draft-somebody-test-00.txt"
txt, _ = submission_file(
"draft-somebody-test-00",
"draft-somebody-test-00.txt",
None,
"test_submission.txt",
title="Correct Draft Title",
)
txt_path.write_text(txt.read())
output = process_submission_text("draft-somebody-test", "00")
self.assertEqual(output["filename"], "draft-somebody-test")
self.assertEqual(output["rev"], "00")
self.assertEqual(output["title"], "Correct Draft Title")
self.assertEqual(output["abstract"].strip(), "This document describes how to test tests.")
self.assertEqual(len(output["authors"]), 1) # not checking in detail, parsing is unreliable
self.assertLessEqual(output["document_date"] - date_today(), datetime.timedelta(days=1))
self.assertEqual(output["pages"], 2)
self.assertGreater(output["words"], 0) # make sure it got something
self.assertGreater(len(output["first_two_pages"]), 0) # make sure it got something
self.assertGreater(output["file_size"], 0) # make sure it got something
self.assertEqual(output["formal_languages"].count(), 1)
self.assertIsNone(output["xml_version"])
# name mismatch
txt, _ = submission_file(
'draft-somebody-wrong-name-00', # name that appears in the file
'draft-somebody-test-00.xml',
"draft-somebody-wrong-name-00", # name that appears in the file
"draft-somebody-test-00.txt",
None,
'test_submission.txt',
title='Correct Draft Title',
"test_submission.txt",
title="Correct Draft Title",
)
txt_path.open('w').write(txt.read())
with self.assertRaisesMessage(SubmissionError, 'disagrees with submission filename'):
process_submission_text(submission)
txt_path.write_text(txt.read())
with self.assertRaisesMessage(SubmissionError, "disagrees with submission filename"):
process_submission_text("draft-somebody-test", "00")
# rev mismatch
txt, _ = submission_file(
'draft-somebody-test-01', # name that appears in the file
'draft-somebody-test-00.xml',
"draft-somebody-test-01", # name that appears in the file
"draft-somebody-test-00.txt",
None,
'test_submission.txt',
title='Correct Draft Title',
"test_submission.txt",
title="Correct Draft Title",
)
txt_path.open('w').write(txt.read())
with self.assertRaisesMessage(SubmissionError, 'disagrees with submission revision'):
process_submission_text(submission)
txt_path.write_text(txt.read())
with self.assertRaisesMessage(SubmissionError, "disagrees with submission revision"):
process_submission_text("draft-somebody-test", "00")
# title mismatch
txt, _ = submission_file(
'draft-somebody-test-00', # name that appears in the file
'draft-somebody-test-00.xml',
None,
'test_submission.txt',
title='Not Correct Draft Title',
def test_process_and_validate_submission(self):
xml_data = {
"title": "The Title",
"authors": [{
"name": "Jane Doe",
"email": "jdoe@example.com",
"affiliation": "Test Centre",
"country": "UK",
}],
"xml_version": "3",
}
text_data = {
"title": "The Title",
"abstract": "This is an abstract.",
"authors": [{
"name": "John Doh",
"email": "ignored@example.com",
"affiliation": "Ignored",
"country": "CA",
}],
"document_date": date_today(),
"pages": 25,
"words": 1234,
"first_two_pages": "Pages One and Two",
"file_size": 4321,
"formal_languages": FormalLanguageName.objects.none(),
}
submission = SubmissionFactory(
state_id="validating",
file_types=".xml,.txt",
)
txt_path.open('w').write(txt.read())
with self.assertRaisesMessage(SubmissionError, 'disagrees with submission title'):
process_submission_text(submission)
with mock.patch("ietf.submit.utils.process_submission_xml", return_value=xml_data):
with mock.patch("ietf.submit.utils.process_submission_text", return_value=text_data):
with mock.patch("ietf.submit.utils.render_missing_formats") as mock_render:
with mock.patch("ietf.submit.utils.apply_checkers") as mock_checkers:
process_and_validate_submission(submission)
self.assertTrue(mock_render.called)
self.assertTrue(mock_checkers.called)
submission = Submission.objects.get(pk=submission.pk)
self.assertEqual(submission.title, text_data["title"])
self.assertEqual(submission.abstract, text_data["abstract"])
self.assertEqual(submission.authors, xml_data["authors"])
self.assertEqual(submission.document_date, text_data["document_date"])
self.assertEqual(submission.pages, text_data["pages"])
self.assertEqual(submission.words, text_data["words"])
self.assertEqual(submission.first_two_pages, text_data["first_two_pages"])
self.assertEqual(submission.file_size, text_data["file_size"])
self.assertEqual(submission.xml_version, xml_data["xml_version"])
def test_status_of_validating_submission(self):
s = SubmissionFactory(state_id='validating')
url = urlreverse('ietf.submit.views.submission_status', kwargs={'submission_id': s.pk})
r = self.client.get(url)
self.assertContains(r, s.name)
self.assertContains(r, 'still being processed and validated', status_code=200)
self.assertContains(r, 'This submission is being processed and validated.', status_code=200)
@override_settings(IDSUBMIT_MAX_VALIDATION_TIME=datetime.timedelta(minutes=30))
def test_cancel_stale_submissions(self):
@ -3648,5 +3756,5 @@ class TestOldNamesAreProtected(BaseSubmitTestCase):
url = urlreverse("ietf.submit.views.upload_submission")
files = {}
files["xml"], _ = submission_file("draft-something-hascapitalletters-00", "draft-something-hascapitalletters-00.xml", None, "test_submission.xml")
r = self.client.post(url, files)
r = self.post_to_upload_submission(url, files)
self.assertContains(r,"Case-conflicting draft name found",status_code=200)

325
ietf/submit/utils.py
View file

@ -11,7 +11,7 @@ import time
import traceback
import xml2rfc
from typing import Optional # pyflakes:ignore
from typing import Optional, Union # pyflakes:ignore
from unidecode import unidecode
from django.conf import settings
@ -40,7 +40,7 @@ from ietf.name.models import StreamName, FormalLanguageName
from ietf.person.models import Person, Email
from ietf.community.utils import update_name_contains_indexes_with_new_doc
from ietf.submit.mail import ( announce_to_lists, announce_new_version, announce_to_authors,
send_approval_request, send_submission_confirmation, announce_new_wg_00 )
send_approval_request, send_submission_confirmation, announce_new_wg_00, send_manual_post_request )
from ietf.submit.models import ( Submission, SubmissionEvent, Preapproval, DraftSubmissionStateName,
SubmissionCheck, SubmissionExtResource )
from ietf.utils import log
@ -911,6 +911,9 @@ class SubmissionError(Exception):
"""Exception for errors during submission processing"""
pass
class InconsistentRevisionError(SubmissionError):
"""SubmissionError caused by an inconsistent revision"""
def staging_path(filename, revision, ext):
if len(ext) > 0 and ext[0] != '.':
@ -1128,102 +1131,169 @@ def _normalize_title(title):
return normalize_text(title) # normalize whitespace
def process_submission_xml(submission):
def process_submission_xml(filename, revision):
"""Validate and extract info from an uploaded submission"""
xml_path = staging_path(submission.name, submission.rev, '.xml')
xml_path = staging_path(filename, revision, '.xml')
xml_draft = XMLDraft(xml_path)
if submission.name != xml_draft.filename:
raise SubmissionError('XML Internet-Draft filename disagrees with submission filename')
if submission.rev != xml_draft.revision:
raise SubmissionError('XML Internet-Draft revision disagrees with submission revision')
if filename != xml_draft.filename:
raise SubmissionError(
f"XML Internet-Draft filename ({xml_draft.filename}) "
f"disagrees with submission filename ({filename})"
)
if revision != xml_draft.revision:
raise SubmissionError(
f"XML Internet-Draft revision ({xml_draft.revision}) "
f"disagrees with submission revision ({revision})"
)
title = _normalize_title(xml_draft.get_title())
if not title:
raise SubmissionError("Could not extract a valid title from the XML")
authors = xml_draft.get_author_list()
for a in authors:
if not a['email']:
raise SubmissionError(f'Missing email address for author {a}')
author_emails = [a['email'].lower() for a in authors]
submitter = get_person_from_name_email(**submission.submitter_parsed()) # the ** expands dict into kwargs
if not any(
email.address.lower() in author_emails
for email in submitter.email_set.filter(active=True)
):
raise SubmissionError(f'Submitter ({submitter}) is not one of the document authors')
# Fill in the submission data
submission.title = _normalize_title(xml_draft.get_title())
if not submission.title:
raise SubmissionError('Could not extract a valid title from the XML')
submission.authors = [
{key: auth[key] for key in ('name', 'email', 'affiliation', 'country')}
for auth in authors
]
submission.xml_version = xml_draft.xml_version
submission.save()
return {
"filename": xml_draft.filename,
"rev": xml_draft.revision,
"title": title,
"authors": [
{key: auth[key] for key in ('name', 'email', 'affiliation', 'country')}
for auth in xml_draft.get_author_list()
],
"abstract": None, # not supported from XML
"document_date": None, # not supported from XML
"pages": None, # not supported from XML
"words": None, # not supported from XML
"first_two_pages": None, # not supported from XML
"file_size": None, # not supported from XML
"formal_languages": None, # not supported from XML
"xml_version": xml_draft.xml_version,
}
def process_submission_text(submission):
"""Validate/extract data from the text version of a submitted draft
def _turn_into_unicode(s: Optional[Union[str, bytes]]):
"""Decode a possibly null string-like item as a string
This assumes the draft was uploaded as XML and extracts data that is not
currently available directly from the XML. Additional processing, e.g. from
get_draft_meta(), would need to be added in order to support direct text
draft uploads.
Copied from ietf.submit.utils.get_draft_meta(), would be nice to
ditch this.
"""
text_path = staging_path(submission.name, submission.rev, '.txt')
if s is None:
return ""
if isinstance(s, str):
return s
else:
try:
return s.decode("utf-8")
except UnicodeDecodeError:
try:
return s.decode("latin-1")
except UnicodeDecodeError:
return ""
def _is_valid_email(addr):
try:
validate_email(addr)
except ValidationError:
return False
return True
def process_submission_text(filename, revision):
"""Validate/extract data from the text version of a submitted draft"""
text_path = staging_path(filename, revision, '.txt')
text_draft = PlaintextDraft.from_file(text_path)
if submission.name != text_draft.filename:
if filename != text_draft.filename:
raise SubmissionError(
f'Text Internet-Draft filename ({text_draft.filename}) disagrees with submission filename ({submission.name})'
f"Text Internet-Draft filename ({text_draft.filename}) "
f"disagrees with submission filename ({filename})"
)
if submission.rev != text_draft.revision:
if revision != text_draft.revision:
raise SubmissionError(
f'Text Internet-Draft revision ({text_draft.revision}) disagrees with submission revision ({submission.rev})')
text_title = _normalize_title(text_draft.get_title())
if not text_title:
raise SubmissionError('Could not extract a valid title from the text')
if text_title != submission.title:
raise SubmissionError(
f'Text Internet-Draft title ({text_title}) disagrees with submission title ({submission.title})')
f"Text Internet-Draft revision ({text_draft.revision}) "
f"disagrees with submission revision ({revision})"
)
title = _normalize_title(text_draft.get_title())
if not title:
# This test doesn't work well - the text_draft parser tends to grab "Abstract" as
# the title if there's an empty title.
raise SubmissionError("Could not extract a title from the text")
submission.abstract = text_draft.get_abstract()
submission.document_date = text_draft.get_creation_date()
submission.pages = text_draft.get_pagecount()
submission.words = text_draft.get_wordcount()
submission.first_two_pages = ''.join(text_draft.pages[:2])
submission.file_size = os.stat(text_path).st_size
submission.save()
submission.formal_languages.set(
FormalLanguageName.objects.filter(
# Drops \r, \n, <, >. Based on get_draft_meta() behavior
trans_table = str.maketrans("", "", "\r\n<>")
authors = [
{
"name": fullname.translate(trans_table).strip(),
"email": _turn_into_unicode(email if _is_valid_email(email) else ""),
"affiliation": _turn_into_unicode(company),
"country": _turn_into_unicode(country),
}
for (fullname, _, _, _, _, email, country, company) in text_draft.get_author_list()
]
return {
"filename": text_draft.filename,
"rev": text_draft.revision,
"title": _normalize_title(text_draft.get_title()),
"authors": authors,
"abstract": text_draft.get_abstract(),
"document_date": text_draft.get_creation_date(),
"pages": text_draft.get_pagecount(),
"words": text_draft.get_wordcount(),
"first_two_pages": ''.join(text_draft.pages[:2]),
"file_size": os.stat(text_path).st_size,
"formal_languages": FormalLanguageName.objects.filter(
slug__in=text_draft.get_formal_languages()
)
)
),
"xml_version": None, # not supported from text
}
def process_uploaded_submission(submission):
def abort_submission(error):
cancel_submission(submission)
create_submission_event(None, submission, f'Submission rejected: {error}')
def process_and_validate_submission(submission):
"""Process and validate a submission
if submission.state_id != 'validating':
log.log(f'Submission {submission.pk} is not in "validating" state, skipping.')
return # do nothing
if submission.file_types != '.xml':
abort_submission('Only XML Internet-Draft submissions can be processed.')
Raises SubmissionError if an error is encountered.
"""
if len(set(submission.file_types.split(",")).intersection({".xml", ".txt"})) == 0:
raise SubmissionError("Require XML and/or text format to process an Internet-Draft submission.")
try:
process_submission_xml(submission)
if check_submission_revision_consistency(submission):
xml_metadata = None
# Parse XML first, if we have it
if ".xml" in submission.file_types:
xml_metadata = process_submission_xml(submission.name, submission.rev)
render_missing_formats(submission) # makes HTML and text, unless text was uploaded
# Parse text, whether uploaded or generated from XML
text_metadata = process_submission_text(submission.name, submission.rev)
if xml_metadata and xml_metadata["title"] != text_metadata["title"]:
raise SubmissionError(
'Document revision inconsistency error in the database. '
'Please contact the secretariat for assistance.'
f"Text Internet-Draft title ({text_metadata['title']}) "
f"disagrees with XML Internet-Draft title ({xml_metadata['title']})"
)
render_missing_formats(submission)
process_submission_text(submission)
# Fill in the submission from the parsed XML/text metadata
if xml_metadata is not None:
# Items preferred / only available from XML
submission.xml_version = xml_metadata["xml_version"]
submission.authors = xml_metadata["authors"]
else:
# Items to get from text only if XML not available
submission.authors = text_metadata["authors"]
# Items always to get from text, even when XML is available
submission.title = text_metadata["title"] # verified above this agrees with XML, if present
submission.abstract = text_metadata["abstract"]
submission.document_date = text_metadata["document_date"]
submission.pages = text_metadata["pages"]
submission.words = text_metadata["words"]
submission.first_two_pages = text_metadata["first_two_pages"]
submission.file_size = text_metadata["file_size"]
submission.save()
submission.formal_languages.set(text_metadata["formal_languages"])
consistency_error = check_submission_revision_consistency(submission)
if consistency_error:
raise InconsistentRevisionError(consistency_error)
set_extresources_from_existing_draft(submission)
apply_checkers(
submission,
@ -1235,16 +1305,105 @@ def process_uploaded_submission(submission):
errors = [c.message for c in submission.checks.filter(passed__isnull=False) if not c.passed]
if len(errors) > 0:
raise SubmissionError('Checks failed: ' + ' / '.join(errors))
except SubmissionError as err:
abort_submission(err)
except SubmissionError:
raise # pass SubmissionErrors up the stack
except Exception:
# convert other exceptions into SubmissionErrors
log.log(f'Unexpected exception while processing submission {submission.pk}.')
log.log(traceback.format_exc())
abort_submission('A system error occurred while processing the submission.')
raise SubmissionError('A system error occurred while processing the submission.')
# if we get here and are still "validating", accept the draft
if submission.state_id == 'validating':
submission.state_id = 'uploaded'
def submitter_is_author(submission):
submitter = get_person_from_name_email(**submission.submitter_parsed())
if submitter:
author_emails = [
author["email"].strip().lower()
for author in submission.authors
if "email" in author
]
return any(
email.address.lower() in author_emails
for email in submitter.email_set.filter(active=True)
)
return False
def all_authors_have_emails(submission):
return all(a["email"] for a in submission.authors)
def process_and_accept_uploaded_submission(submission):
"""Process, validate, and, if valid, accept an uploaded submission
Requires that the submitter already be set and is an author of the submitted draft.
The submission must be in the "validating" state. On success, it will be in the
"posted" state. On error, it wil be in the "cancel" state.
"""
if submission.state_id != "validating":
log.log(f'Submission {submission.pk} is not in "validating" state, skipping.')
return # do nothing
if submission.file_types != '.xml':
# permit only XML uploads for automatic acceptance
cancel_submission(submission)
create_submission_event(
None,
submission,
"Only XML Internet-Draft submissions can be processed.",
)
return
try:
process_and_validate_submission(submission)
except SubmissionError as err:
cancel_submission(submission) # changes Submission.state
create_submission_event(None, submission, f"Submission rejected: {err}")
return
if not all_authors_have_emails(submission):
cancel_submission(submission) # changes Submission.state
create_submission_event(
None,
submission,
"Submission rejected: Email address not found for all authors"
)
return
if not submitter_is_author(submission):
cancel_submission(submission) # changes Submission.state
create_submission_event(
None,
submission,
f"Submission rejected: Submitter ({submission.submitter}) is not one of the document authors",
)
return
create_submission_event(None, submission, desc="Completed submission validation checks")
accept_submission(submission)
def process_uploaded_submission(submission):
"""Process and validate an uploaded submission
The submission must be in the "validating" state. On success, it will be in the "uploaded"
state. On error, it will be in the "cancel" state.
"""
if submission.state_id != "validating":
log.log(f'Submission {submission.pk} is not in "validating" state, skipping.')
return # do nothing
try:
process_and_validate_submission(submission)
except InconsistentRevisionError as consistency_error:
submission.state_id = "manual"
submission.save()
create_submission_event(None, submission, desc="Uploaded submission (diverted to manual process)")
send_manual_post_request(None, submission, errors=dict(consistency=str(consistency_error)))
except SubmissionError as err:
cancel_submission(submission) # changes Submission.state
create_submission_event(None, submission, f"Submission rejected: {err}")
else:
submission.state_id = "uploaded"
submission.save()
create_submission_event(None, submission, desc="Completed submission validation checks")
accept_submission(submission)

93
ietf/submit/views.py
View file

@ -12,7 +12,7 @@ from urllib.parse import urljoin
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.models import User
from django.db import DataError, transaction
from django.db import transaction
from django.urls import reverse as urlreverse
from django.core.exceptions import ValidationError
from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden, HttpResponse, JsonResponse
@ -30,13 +30,13 @@ from ietf.ietfauth.utils import has_role, role_required
from ietf.mailtrigger.utils import gather_address_lists
from ietf.message.models import Message, MessageAttachment
from ietf.person.models import Email
from ietf.submit.forms import ( SubmissionManualUploadForm, SubmissionAutoUploadForm, AuthorForm,
SubmitterForm, EditSubmissionForm, PreapprovalForm, ReplacesForm, SubmissionEmailForm, MessageModelForm,
DeprecatedSubmissionAutoUploadForm )
from ietf.submit.forms import (SubmissionAutoUploadForm, AuthorForm, SubmitterForm, EditSubmissionForm,
PreapprovalForm, ReplacesForm, SubmissionEmailForm, MessageModelForm,
DeprecatedSubmissionAutoUploadForm, SubmissionManualUploadForm)
from ietf.submit.mail import send_full_url, send_manual_post_request, add_submission_email, get_reply_to
from ietf.submit.models import (Submission, Preapproval, SubmissionExtResource,
DraftSubmissionStateName, SubmissionEmailEvent )
from ietf.submit.tasks import process_uploaded_submission_task, poke
from ietf.submit.tasks import process_uploaded_submission_task, process_and_accept_uploaded_submission_task, poke
from ietf.submit.utils import ( approvable_submissions_for_user, preapprovals_for_user,
recently_approved_by_user, validate_submission, create_submission_event, docevent_from_submission,
post_submission, cancel_submission, rename_submission_files, remove_submission_files, get_draft_meta,
@ -52,64 +52,35 @@ from ietf.utils.timezone import date_today
def upload_submission(request):
if request.method == 'POST':
try:
form = SubmissionManualUploadForm(request, data=request.POST, files=request.FILES)
if form.is_valid():
log('got valid submission form for %s' % form.filename)
saved_files = save_files(form)
authors, abstract, file_name, file_size = get_draft_meta(form, saved_files)
submission = get_submission(form)
try:
fill_in_submission(form, submission, authors, abstract, file_size)
except Exception as e:
log("Exception: %s\n" % e)
if submission and submission.id:
submission.delete()
raise
apply_checkers(submission, file_name)
consistency_error = check_submission_revision_consistency(submission)
if consistency_error:
# A data consistency problem diverted this to manual processing - send notification
submission.state = DraftSubmissionStateName.objects.get(slug="manual")
submission.save()
create_submission_event(request, submission, desc="Uploaded submission (diverted to manual process)")
send_manual_post_request(request, submission, errors=dict(consistency=consistency_error))
else:
# This is the usual case
create_submission_event(request, submission, desc="Uploaded submission")
# Don't add an "Uploaded new revision doevent yet, in case of cancellation
return redirect("ietf.submit.views.submission_status", submission_id=submission.pk, access_token=submission.access_token())
except IOError as e:
if "read error" in str(e): # The server got an IOError when trying to read POST data
form = SubmissionManualUploadForm(request=request)
form._errors = {}
form._errors["__all__"] = form.error_class(["There was a failure receiving the complete form data -- please try again."])
else:
raise
except ValidationError as e:
form = SubmissionManualUploadForm(request=request)
form._errors = {}
form._errors["__all__"] = form.error_class(["There was a failure converting the xml file to text -- please verify that your xml file is valid. (%s)" % e.message])
if debug.debug:
raise
except DataError as e:
form = SubmissionManualUploadForm(request=request)
form._errors = {}
form._errors["__all__"] = form.error_class(["There was a failure processing your upload -- please verify that your Internet-Draft passes idnits. (%s)" % e.message])
if debug.debug:
raise
if request.method == "POST":
form = SubmissionManualUploadForm(
request, data=request.POST, files=request.FILES
)
if form.is_valid():
submission = get_submission(form)
submission.state = DraftSubmissionStateName.objects.get(slug="validating")
submission.remote_ip = form.remote_ip
submission.file_types = ",".join(form.file_types)
submission.submission_date = date_today()
submission.save()
clear_existing_files(form)
save_files(form)
create_submission_event(request, submission, desc="Uploaded submission")
# Wrap in on_commit so the delayed task cannot start until the view is done with the DB
transaction.on_commit(
lambda: process_uploaded_submission_task.delay(submission.pk)
)
return redirect(
"ietf.submit.views.submission_status",
submission_id=submission.pk,
access_token=submission.access_token(),
)
else:
form = SubmissionManualUploadForm(request=request)
return render(request, 'submit/upload_submission.html',
{'selected': 'index',
'form': form})
return render(
request, "submit/upload_submission.html", {"selected": "index", "form": form}
)
@csrf_exempt
def api_submission(request):
@ -173,7 +144,7 @@ def api_submission(request):
# Wrap in on_commit so the delayed task cannot start until the view is done with the DB
transaction.on_commit(
lambda: process_uploaded_submission_task.delay(submission.pk)
lambda: process_and_accept_uploaded_submission_task.delay(submission.pk)
)
return JsonResponse(
{

28
ietf/templates/submit/submission_status.html
View file

@ -133,17 +133,31 @@
This submission is awaiting the first Internet-Draft upload.
</p>
{% elif submission.state_id == 'validating' %}
<p class="alert alert-warning my-3">
This submission is still being processed and validated. This normally takes a few minutes after
<div class="alert alert-danger my-3">
Notice: The Internet-Draft submission process has changed as of Datatracker version 10.3.0.
Your Internet-Draft is currently being processed and validated asynchronously. Results will be
displayed at this URL when they are available. If JavaScript is enabled in your
browser, this page will refreshed automatically. If JavaScript is not enabled, or if you
disable the automatic refresh with the toggle below, please reload this page in a few
minutes to see the results.
</div>
<div class="alert alert-warning my-3">
This submission is being processed and validated. This normally takes a few minutes after
submission.
{% with earliest_event=submission.submissionevent_set.last %}
{% if earliest_event %}
It has been {{ earliest_event.time|timesince }} since submission.
{% endif %}
{% if earliest_event %}
Your draft was uploaded at {{ earliest_event.time }}<span id="time-since-uploaded" class="d-none">
({{ earliest_event.time|timesince }} ago)</span>.
{% endif %}
{% endwith %}
Please contact the secretariat for assistance if it has been more than an hour.
</p>
{% else %}
<div class="form-check form-switch mt-3 d-none">{# hide with d-none unless javascript makes it visible #}
<input class="form-check-input" type="checkbox" id="enableAutoReload" checked>
<label class="form-check-label" for="enableAutoReload"> Refresh automatically </label>
</div>
</div>
{% else %}
<h2 class="mt-5">Meta-data from the submission</h2>
{% if errors %}
<div class="alert alert-danger my-3">

11
ietf/templates/submit/upload_submission.html
View file

@ -47,7 +47,8 @@
aria-controls="other-formats">
<input class="form-check-input"
id="checkbox"
type="checkbox">
type="checkbox"
{% if form.errors.txt %}checked {% endif %}>
Submit other formats
</label>
</div>
@ -60,14 +61,8 @@
However, if you cannot for some reason submit XML, you must
submit a plaintext rendering of your I-D.
</p>
{% bootstrap_label '<i class="bi bi-file-pdf"></i> PDF rendering of the I-D' label_class="form-label fw-bold" %}
{% bootstrap_field form.pdf show_label=False %}
<p class="form-text">
Optional to submit, will be auto-generated based
on the submitted XML.
</p>
</div>
{% bootstrap_form_errors form %}
{% bootstrap_form_errors form type="non_fields" %}
{% bootstrap_button button_type="submit" name="upload" content="Upload" %}
</form>
{% include "submit/problem-reports-footer.html" %}