altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

feat: Process uploaded submissions asynchronously (#5580)

Browse source 
* fix: Use relative URL for submission status link

* refactor: Refactor/rename process_uploaded_submission async task

* feat: Add async task to process but not accept a submission

* feat: Replace upload_submission() with an async implementation (WIP)

* fix: Do not put Submission in "uploaded" state if an error occured

* refactor: Improve text/XML draft processing flow

* feat: Extract authors from text in async processing

* fix: Fix call signatures and abort submission on failed validation

* feat: Validate submission name format

* fix: Correctly validate emails from text submission

* fix: Clean up submission validation

* fix: Better display errors on upload_submission page

* feat: Reload submission status page when awaiting validation

* test: Fix call signatures; remove unused imports

* chore: Add type hint

* test: Update tests to match renamed task

* fix: Fix typo in error message

* test: Fix failing Api- and AsyncSubmissionTests

* Rename process_uploaded_submission to process_and_accept_...
* Remove outdated tests

Does not yet test new behavior.

* refactor: Break up submission_file() helper

* test: Refactor tests to run the async processing (wip)

* test: Drop test of bad PDF submission

The PDF submission field was removed, so no need to test it.

* test: Update more tests

* test: Bring back create_and_post_submission() and fix more tests

* fix: Drop to manual, don't cancel, on revision inconsistency

Fixes remaining failing SubmitTest tests

* style: Restyle upload_submission() with black

* test: Verify that async submission processing is invoked on upload

* test: Bring back old do_submission and fix tests

Properly separating the upload and async processing stages of submission
is a bigger refactoring than will fit right now. This better exercises
the submission pipeline.

* fix: Accept only XML for API submissions

* test: Test submission processing utilities

* feat: Improve status display for "validating" submissions

* chore: Remove obsolete code

* test: Update test to match amended text

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
This commit is contained in:
Jennifer Richards 2023-05-09 16:21:46 -04:00 committed by GitHub
parent b1c60efbb3
commit a0f6cdb661
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 858 additions and 551 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

144
ietf/static/js/draft-submit.js
View file

@ -1,69 +1,93 @@
$(document) $(function () {
.ready(function () { // fill in submitter info when an author button is clicked
// fill in submitter info when an author button is clicked $("form.idsubmit button.author")
$("form.idsubmit button.author") .on("click", function () {
.on("click", function () { var name = $(this)
var name = $(this) .data("name");
.data("name"); var email = $(this)
var email = $(this) .data("email");
.data("email");
$(this) $(this)
.parents("form") .parents("form")
.find("input[name=submitter-name]") .find("input[name=submitter-name]")
.val(name || ""); .val(name || "");
$(this) $(this)
.parents("form") .parents("form")
.find("input[name=submitter-email]") .find("input[name=submitter-email]")
.val(email || ""); .val(email || "");
}); });
$("form.idsubmit") $("form.idsubmit")
.on("submit", function () { .on("submit", function () {
if (this.submittedAlready) if (this.submittedAlready)
return false; return false;
else { else {
this.submittedAlready = true; this.submittedAlready = true;
return true; return true;
} }
}); });
$("form.idsubmit #add-author") $("form.idsubmit #add-author")
.on("click", function () { .on("click", function () {
// clone the last author block and make it empty // clone the last author block and make it empty
var cloner = $("#cloner"); var cloner = $("#cloner");
var next = cloner.clone(); var next = cloner.clone();
next.find('input:not([type=hidden])') next.find('input:not([type=hidden])')
.val(''); .val('');
// find the author number // find the author number
var t = next.children('h3') var t = next.children('h3')
.text(); .text();
var n = parseInt(t.replace(/\D/g, '')); var n = parseInt(t.replace(/\D/g, ''));
// change the number in attributes and text // change the number in attributes and text
next.find('*') next.find('*')
.each(function () { .each(function () {
var e = this; var e = this;
$.each(['id', 'for', 'name', 'value'], function (i, v) { $.each(['id', 'for', 'name', 'value'], function (i, v) {
if ($(e) if ($(e)
.attr(v)) { .attr(v)) {
$(e) $(e)
.attr(v, $(e) .attr(v, $(e)
.attr(v) .attr(v)
.replace(n - 1, n)); .replace(n - 1, n));
} }
});
}); });
});
t = t.replace(n, n + 1); t = t.replace(n, n + 1);
next.children('h3') next.children('h3')
.text(t); .text(t);
// move the cloner id to next and insert next into the DOM // move the cloner id to next and insert next into the DOM
cloner.removeAttr('id'); cloner.removeAttr('id');
next.attr('id', 'cloner'); next.attr('id', 'cloner');
next.insertAfter(cloner); next.insertAfter(cloner);
}); });
});
// Reload page periodically if the enableAutoReload checkbox is present and checked
const autoReloadSwitch = document.getElementById("enableAutoReload");
const timeSinceDisplay = document.getElementById("time-since-uploaded");
if (autoReloadSwitch) {
const autoReloadTime = 30000; // ms
let autoReloadTimeoutId;
autoReloadSwitch.parentElement.classList.remove("d-none");
timeSinceDisplay.classList.remove("d-none");
autoReloadTimeoutId = setTimeout(() => location.reload(), autoReloadTime);
autoReloadSwitch.addEventListener("change", (e) => {
if (e.currentTarget.checked) {
if (!autoReloadTimeoutId) {
autoReloadTimeoutId = setTimeout(() => location.reload(), autoReloadTime);
timeSinceDisplay.classList.remove("d-none");
}
} else {
if (autoReloadTimeoutId) {
clearTimeout(autoReloadTimeoutId);
autoReloadTimeoutId = null;
timeSinceDisplay.classList.add("d-none");
}
}
});
}
});

297
ietf/submit/forms.py
View file

@ -13,8 +13,8 @@ import xml2rfc
from contextlib import ExitStack from contextlib import ExitStack
from email.utils import formataddr from email.utils import formataddr
from typing import Tuple
from unidecode import unidecode from unidecode import unidecode
from urllib.parse import urljoin
from django import forms from django import forms
from django.conf import settings from django.conf import settings
@ -36,7 +36,6 @@ from ietf.message.models import Message
from ietf.name.models import FormalLanguageName, GroupTypeName from ietf.name.models import FormalLanguageName, GroupTypeName
from ietf.submit.models import Submission, Preapproval from ietf.submit.models import Submission, Preapproval
from ietf.submit.utils import validate_submission_name, validate_submission_rev, validate_submission_document_date, remote_ip from ietf.submit.utils import validate_submission_name, validate_submission_rev, validate_submission_document_date, remote_ip
from ietf.submit.parsers.pdf_parser import PDFParser
from ietf.submit.parsers.plain_parser import PlainParser from ietf.submit.parsers.plain_parser import PlainParser
from ietf.submit.parsers.xml_parser import XMLParser from ietf.submit.parsers.xml_parser import XMLParser
from ietf.utils import log from ietf.utils import log
@ -49,6 +48,9 @@ from ietf.utils.xmldraft import XMLDraft, XMLParseError
class SubmissionBaseUploadForm(forms.Form): class SubmissionBaseUploadForm(forms.Form):
xml = forms.FileField(label='.xml format', required=True) xml = forms.FileField(label='.xml format', required=True)
formats: Tuple[str, ...] = ('xml',) # allowed formats
base_formats: Tuple[str, ...] = ('xml',) # at least one of these is required
def __init__(self, request, *args, **kwargs): def __init__(self, request, *args, **kwargs):
super(SubmissionBaseUploadForm, self).__init__(*args, **kwargs) super(SubmissionBaseUploadForm, self).__init__(*args, **kwargs)
@ -66,18 +68,11 @@ class SubmissionBaseUploadForm(forms.Form):
self.title = None self.title = None
self.abstract = None self.abstract = None
self.authors = [] self.authors = []
self.parsed_draft = None
self.file_types = [] self.file_types = []
self.file_info = {} # indexed by file field name, e.g., 'txt', 'xml', ... self.file_info = {} # indexed by file field name, e.g., 'txt', 'xml', ...
self.xml_version = None self.xml_version = None
# No code currently (14 Sep 2017) uses this class directly; it is
# only used through its subclasses. The two assignments below are self._extracted_filenames_and_revisions = {}
# set to trigger an exception if it is used directly only to make
# sure that adequate consideration is made if it is decided to use it
# directly in the future. Feel free to set these appropriately to
# avoid the exceptions in that case:
self.formats = None # None will raise an exception in clean() if this isn't changed in a subclass
self.base_formats = None # None will raise an exception in clean() if this isn't changed in a subclass
def set_cutoff_warnings(self): def set_cutoff_warnings(self):
now = timezone.now() now = timezone.now()
@ -126,20 +121,17 @@ class SubmissionBaseUploadForm(forms.Form):
'The I-D submission tool will be reopened after %s (IETF-meeting local time).' % (cutoff_01_str, reopen_str)) 'The I-D submission tool will be reopened after %s (IETF-meeting local time).' % (cutoff_01_str, reopen_str))
self.shutdown = True self.shutdown = True
def clean_file(self, field_name, parser_class): def _clean_file(self, field_name, parser_class):
f = self.cleaned_data[field_name] f = self.cleaned_data[field_name]
if not f: if not f:
return f return f
self.file_info[field_name] = parser_class(f).critical_parse() self.file_info[field_name] = parser_class(f).critical_parse()
if self.file_info[field_name].errors: if self.file_info[field_name].errors:
raise forms.ValidationError(self.file_info[field_name].errors) raise forms.ValidationError(self.file_info[field_name].errors, code="critical_error")
return f return f
def clean_xml(self): def clean_xml(self):
return self.clean_file("xml", XMLParser)
def clean(self):
def format_messages(where, e, log_msgs): def format_messages(where, e, log_msgs):
m = str(e) m = str(e)
if m: if m:
@ -148,38 +140,12 @@ class SubmissionBaseUploadForm(forms.Form):
import traceback import traceback
typ, val, tb = sys.exc_info() typ, val, tb = sys.exc_info()
m = traceback.format_exception(typ, val, tb) m = traceback.format_exception(typ, val, tb)
m = [ l.replace('\n ', ':\n ') for l in m ] m = [l.replace('\n ', ':\n ') for l in m]
msgs = [s for s in (["Error from xml2rfc (%s):" % (where,)] + m + log_msgs) if s] msgs = [s for s in ([f"Error from xml2rfc ({where}):"] + m + log_msgs) if s]
return msgs return msgs
if self.shutdown and not has_role(self.request.user, "Secretariat"): xml_file = self._clean_file("xml", XMLParser)
raise forms.ValidationError('The submission tool is currently shut down') if xml_file:
# check general submission rate thresholds before doing any more work
today = date_today()
self.check_submissions_thresholds(
"for the same submitter",
dict(remote_ip=self.remote_ip, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER, settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER_SIZE,
)
self.check_submissions_thresholds(
"across all submitters",
dict(submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
)
for ext in self.formats:
f = self.cleaned_data.get(ext, None)
if not f:
continue
self.file_types.append('.%s' % ext)
if not ('.txt' in self.file_types or '.xml' in self.file_types):
if not self.errors:
raise forms.ValidationError('Unexpected submission file types; found %s, but %s is required' % (', '.join(self.file_types), ' or '.join(self.base_formats)))
# Determine the draft name and revision. Try XML first.
if self.cleaned_data.get('xml'):
xml_file = self.cleaned_data.get('xml')
tfn = None tfn = None
with ExitStack() as stack: with ExitStack() as stack:
@stack.callback @stack.callback
@ -204,86 +170,131 @@ class SubmissionBaseUploadForm(forms.Form):
xml_draft = XMLDraft(tfn) xml_draft = XMLDraft(tfn)
except XMLParseError as e: except XMLParseError as e:
msgs = format_messages('xml', e, e.parser_msgs()) msgs = format_messages('xml', e, e.parser_msgs())
self.add_error('xml', msgs) raise forms.ValidationError(msgs, code="xml_parse_error")
return
except Exception as e: except Exception as e:
self.add_error('xml', f'Error parsing XML Internet-Draft: {e}') raise forms.ValidationError(f"Error parsing XML Internet-Draft: {e}", code="parse_exception")
return if not xml_draft.filename:
raise forms.ValidationError(
"Could not extract a valid Internet-Draft name from the XML. "
"Please make sure that the top-level <rfc/> "
"element has a docName attribute which provides the full Internet-Draft name including "
"revision number.",
code="parse_error_filename",
)
if not xml_draft.revision:
raise forms.ValidationError(
"Could not extract a valid Internet-Draft revision from the XML. "
"Please make sure that the top-level <rfc/> "
"element has a docName attribute which provides the full Internet-Draft name including "
"revision number.",
code="parse_error_revision",
)
self._extracted_filenames_and_revisions['xml'] = (xml_draft.filename, xml_draft.revision)
return xml_file
self.filename = xml_draft.filename def clean(self):
self.revision = xml_draft.revision if self.shutdown and not has_role(self.request.user, "Secretariat"):
elif self.cleaned_data.get('txt'): raise forms.ValidationError('The submission tool is currently shut down')
# no XML available, extract from the text if we have it
# n.b., this code path is unused until a subclass with a 'txt' field is created.
txt_file = self.cleaned_data['txt']
txt_file.seek(0)
bytes = txt_file.read()
try:
text = bytes.decode(self.file_info['txt'].charset)
self.parsed_draft = PlaintextDraft(text, txt_file.name)
self.filename = self.parsed_draft.filename
self.revision = self.parsed_draft.revision
except (UnicodeDecodeError, LookupError) as e:
self.add_error('txt', 'Failed decoding the uploaded file: "%s"' % str(e))
rev_error = validate_submission_rev(self.filename, self.revision) # check general submission rate thresholds before doing any more work
if rev_error: today = date_today()
raise forms.ValidationError(rev_error) self.check_submissions_thresholds(
"for the same submitter",
dict(remote_ip=self.remote_ip, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER, settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER_SIZE,
)
self.check_submissions_thresholds(
"across all submitters",
dict(submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
)
for ext in self.formats:
f = self.cleaned_data.get(ext, None)
if not f:
continue
self.file_types.append('.%s' % ext)
if not any(f".{bt}" in self.file_types for bt in self.base_formats):
if not self.errors:
raise forms.ValidationError(
"Unexpected submission file types; found {}, but {} is required".format(
", ".join(ft.lstrip(".") for ft in self.file_types),
" or ".join(self.base_formats),
)
)
# The following errors are likely noise if we have previous field # The following errors are likely noise if we have previous field
# errors: # errors:
if self.errors: if self.errors:
raise forms.ValidationError('') raise forms.ValidationError('')
# Check that all formats agree on draft name/rev
filename_from = None
for fmt, (extracted_name, extracted_rev) in self._extracted_filenames_and_revisions.items():
if self.filename is None:
filename_from = fmt
self.filename = extracted_name
self.revision = extracted_rev
elif self.filename != extracted_name:
raise forms.ValidationError(
{fmt: f"Extracted filename '{extracted_name}' does not match filename '{self.filename}' from {filename_from} format"},
code="filename_mismatch",
)
elif self.revision != extracted_rev:
raise forms.ValidationError(
{fmt: f"Extracted revision ({extracted_rev}) does not match revision from {filename_from} format ({self.revision})"},
code="revision_mismatch",
)
# Not expected to encounter missing filename/revision here because
# the individual fields should fail validation, but just in case
if not self.filename: if not self.filename:
raise forms.ValidationError("Could not extract a valid Internet-Draft name from the upload. " raise forms.ValidationError(
"To fix this in a text upload, please make sure that the full Internet-Draft name including " "Unable to extract a filename from any uploaded format.",
"revision number appears centered on its own line below the document title on the " code="no_filename",
"first page. In an xml upload, please make sure that the top-level <rfc/> " )
"element has a docName attribute which provides the full Internet-Draft name including "
"revision number.")
if not self.revision: if not self.revision:
raise forms.ValidationError("Could not extract a valid Internet-Draft revision from the upload. " raise forms.ValidationError(
"To fix this in a text upload, please make sure that the full Internet-Draft name including " "Unable to extract a revision from any uploaded format.",
"revision number appears centered on its own line below the document title on the " code="no_revision",
"first page. In an xml upload, please make sure that the top-level <rfc/> " )
"element has a docName attribute which provides the full Internet-Draft name including "
"revision number.") name_error = validate_submission_name(self.filename)
if name_error:
raise forms.ValidationError(name_error)
rev_error = validate_submission_rev(self.filename, self.revision)
if rev_error:
raise forms.ValidationError(rev_error)
self.check_for_old_uppercase_collisions(self.filename) self.check_for_old_uppercase_collisions(self.filename)
if self.cleaned_data.get('txt') or self.cleaned_data.get('xml'): # check group
# check group self.group = self.deduce_group(self.filename)
self.group = self.deduce_group(self.filename) # check existing
# check existing existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel", "waiting-for-draft"))
existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel", "waiting-for-draft")) if existing:
if existing: raise forms.ValidationError(
raise forms.ValidationError( format_html(
format_html( 'A submission with same name and revision is currently being processed. <a href="{}">Check the status here.</a>',
'A submission with same name and revision is currently being processed. <a href="{}">Check the status here.</a>', urlreverse("ietf.submit.views.submission_status", kwargs={'submission_id': existing[0].pk}),
urljoin(
settings.IDTRACKER_BASE_URL,
urlreverse("ietf.submit.views.submission_status", kwargs={'submission_id': existing[0].pk}),
)
)
) )
# cut-off
if self.revision == '00' and self.in_first_cut_off:
raise forms.ValidationError(mark_safe(self.cutoff_warning))
# check thresholds that depend on the draft / group
self.check_submissions_thresholds(
"for the Internet-Draft %s" % self.filename,
dict(name=self.filename, rev=self.revision, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
) )
if self.group:
self.check_submissions_thresholds( # cut-off
"for the group \"%s\"" % (self.group.acronym), if self.revision == '00' and self.in_first_cut_off:
dict(group=self.group, submission_date=today), raise forms.ValidationError(mark_safe(self.cutoff_warning))
settings.IDSUBMIT_MAX_DAILY_SAME_GROUP, settings.IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE, # check thresholds that depend on the draft / group
) self.check_submissions_thresholds(
"for the Internet-Draft %s" % self.filename,
dict(name=self.filename, rev=self.revision, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
)
if self.group:
self.check_submissions_thresholds(
"for the group \"%s\"" % (self.group.acronym),
dict(group=self.group, submission_date=today),
settings.IDSUBMIT_MAX_DAILY_SAME_GROUP, settings.IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE,
)
return super().clean() return super().clean()
@staticmethod @staticmethod
@ -613,26 +624,6 @@ class DeprecatedSubmissionBaseUploadForm(SubmissionBaseUploadForm):
return super().clean() return super().clean()
class SubmissionManualUploadForm(DeprecatedSubmissionBaseUploadForm):
xml = forms.FileField(label='.xml format', required=False) # xml field with required=False instead of True
txt = forms.FileField(label='.txt format', required=False)
# We won't permit html upload until we can verify that the content
# reasonably matches the text and/or xml upload. Till then, we generate
# html for version 3 xml submissions.
# html = forms.FileField(label='.html format', required=False)
pdf = forms.FileField(label='.pdf format', required=False)
def __init__(self, request, *args, **kwargs):
super(SubmissionManualUploadForm, self).__init__(request, *args, **kwargs)
self.formats = settings.IDSUBMIT_FILE_TYPES
self.base_formats = ['txt', 'xml', ]
def clean_txt(self):
return self.clean_file("txt", PlainParser)
def clean_pdf(self):
return self.clean_file("pdf", PDFParser)
class DeprecatedSubmissionAutoUploadForm(DeprecatedSubmissionBaseUploadForm): class DeprecatedSubmissionAutoUploadForm(DeprecatedSubmissionBaseUploadForm):
"""Full-service upload form, replaced by the asynchronous version""" """Full-service upload form, replaced by the asynchronous version"""
user = forms.EmailField(required=True) user = forms.EmailField(required=True)
@ -642,17 +633,50 @@ class DeprecatedSubmissionAutoUploadForm(DeprecatedSubmissionBaseUploadForm):
self.formats = ['xml', ] self.formats = ['xml', ]
self.base_formats = ['xml', ] self.base_formats = ['xml', ]
class SubmissionManualUploadForm(SubmissionBaseUploadForm):
txt = forms.FileField(label='.txt format', required=False)
formats = SubmissionBaseUploadForm.formats + ('txt',)
base_formats = SubmissionBaseUploadForm.base_formats + ('txt',)
def __init__(self, request, *args, **kwargs):
super().__init__(request, *args, **kwargs)
self.fields['xml'].required = False
def clean_txt(self):
txt_file = self._clean_file("txt", PlainParser)
if txt_file is not None:
bytes = txt_file.read()
try:
text = bytes.decode(self.file_info["txt"].charset)
parsed_draft = PlaintextDraft(text, txt_file.name)
self._extracted_filenames_and_revisions["txt"] = (parsed_draft.filename, parsed_draft.revision)
except (UnicodeDecodeError, LookupError) as e:
raise forms.ValidationError(f'Failed decoding the uploaded file: "{str(e)}"', code="decode_failed")
if not parsed_draft.filename:
raise forms.ValidationError(
"Could not extract a valid Internet-Draft name from the plaintext. "
"Please make sure that the full Internet-Draft name including "
"revision number appears centered on its own line below the document title on the "
"first page.",
code="parse_error_filename",
)
if not parsed_draft.revision:
raise forms.ValidationError(
"Could not extract a valid Internet-Draft revision from the plaintext. "
"Please make sure that the full Internet-Draft name including "
"revision number appears centered on its own line below the document title on the "
"first page.",
code="parse_error_revision",
)
return txt_file
class SubmissionAutoUploadForm(SubmissionBaseUploadForm): class SubmissionAutoUploadForm(SubmissionBaseUploadForm):
user = forms.EmailField(required=True) user = forms.EmailField(required=True)
replaces = forms.CharField(required=False, max_length=1000, strip=True) replaces = forms.CharField(required=False, max_length=1000, strip=True)
def __init__(self, request, *args, **kwargs):
super().__init__(request, *args, **kwargs)
self.formats = ['xml', ]
self.base_formats = ['xml', ]
def clean(self): def clean(self):
super().clean() cleaned_data = super().clean()
# Clean the replaces field after the rest of the cleaning so we know the name of the # Clean the replaces field after the rest of the cleaning so we know the name of the
# uploaded draft via self.filename # uploaded draft via self.filename
@ -692,6 +716,7 @@ class SubmissionAutoUploadForm(SubmissionBaseUploadForm):
alias.name + " is approved by the IESG and cannot be replaced" alias.name + " is approved by the IESG and cannot be replaced"
), ),
) )
return cleaned_data
class NameEmailForm(forms.Form): class NameEmailForm(forms.Form):

13
ietf/submit/tasks.py
View file

@ -9,7 +9,8 @@ from django.conf import settings
from django.utils import timezone from django.utils import timezone
from ietf.submit.models import Submission from ietf.submit.models import Submission
from ietf.submit.utils import cancel_submission, create_submission_event, process_uploaded_submission from ietf.submit.utils import (cancel_submission, create_submission_event, process_uploaded_submission,
process_and_accept_uploaded_submission)
from ietf.utils import log from ietf.utils import log
@ -23,6 +24,16 @@ def process_uploaded_submission_task(submission_id):
process_uploaded_submission(submission) process_uploaded_submission(submission)
@shared_task
def process_and_accept_uploaded_submission_task(submission_id):
try:
submission = Submission.objects.get(pk=submission_id)
except Submission.DoesNotExist:
log.log(f'process_uploaded_submission_task called for missing submission_id={submission_id}')
else:
process_and_accept_uploaded_submission(submission)
@shared_task @shared_task
def cancel_stale_submissions(): def cancel_stale_submissions():
now = timezone.now() now = timezone.now()

0
ietf/submit/test_submission.nonascii → ietf/submit/test_submission.nonascii.txt
View file

2
ietf/submit/test_submission_invalid_yang.txt
View file

@ -2,7 +2,7 @@
Network Working Group A. Name Network Working Group %(firstpagename)37s
Internet-Draft Test Centre Inc. Internet-Draft Test Centre Inc.
Intended status: Informational %(month)s %(year)s Intended status: Informational %(month)s %(year)s
Expires: %(expiration)s Expires: %(expiration)s

424
ietf/submit/tests.py
View file

@ -5,13 +5,14 @@
import datetime import datetime
import email import email
import io import io
import mock
import os import os
import re import re
import sys import sys
import mock
from io import StringIO from io import StringIO
from pyquery import PyQuery from pyquery import PyQuery
from typing import Tuple
from pathlib import Path from pathlib import Path
@ -28,7 +29,9 @@ import debug # pyflakes:ignore
from ietf.submit.utils import (expirable_submissions, expire_submission, find_submission_filenames, from ietf.submit.utils import (expirable_submissions, expire_submission, find_submission_filenames,
post_submission, validate_submission_name, validate_submission_rev, post_submission, validate_submission_name, validate_submission_rev,
process_uploaded_submission, SubmissionError, process_submission_text) process_and_accept_uploaded_submission, SubmissionError, process_submission_text,
process_submission_xml, process_uploaded_submission,
process_and_validate_submission)
from ietf.doc.factories import (DocumentFactory, WgDraftFactory, IndividualDraftFactory, IndividualRfcFactory, from ietf.doc.factories import (DocumentFactory, WgDraftFactory, IndividualDraftFactory, IndividualRfcFactory,
ReviewFactory, WgRfcFactory) ReviewFactory, WgRfcFactory)
from ietf.doc.models import ( Document, DocAlias, DocEvent, State, from ietf.doc.models import ( Document, DocAlias, DocEvent, State,
@ -47,7 +50,7 @@ from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactor
from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm
from ietf.submit.models import Submission, Preapproval, SubmissionExtResource from ietf.submit.models import Submission, Preapproval, SubmissionExtResource
from ietf.submit.mail import add_submission_email, process_response_email from ietf.submit.mail import add_submission_email, process_response_email
from ietf.submit.tasks import cancel_stale_submissions, process_uploaded_submission_task from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task
from ietf.utils.accesstoken import generate_access_token from ietf.utils.accesstoken import generate_access_token
from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.models import VersionInfo from ietf.utils.models import VersionInfo
@ -91,7 +94,28 @@ class BaseSubmitTestCase(TestCase):
def archive_dir(self): def archive_dir(self):
return settings.INTERNET_DRAFT_ARCHIVE_DIR return settings.INTERNET_DRAFT_ARCHIVE_DIR
def submission_file(name_in_doc, name_in_post, group, templatename, author=None, email=None, title=None, year=None, ascii=True): def post_to_upload_submission(self, *args, **kwargs):
"""POST to the upload_submission endpoint
Use this instead of directly POSTing to be sure that the appropriate celery
tasks would be queued (but are not actually queued during testing)
"""
# Mock task so we can check that it's called without actually submitting a celery task.
# Also mock on_commit() because otherwise the test transaction prevents the call from
# ever being made.
with mock.patch("ietf.submit.views.process_uploaded_submission_task") as mocked_task:
with mock.patch("ietf.submit.views.transaction.on_commit", side_effect=lambda x: x()):
response = self.client.post(*args, **kwargs)
if response.status_code == 302:
# A 302 indicates we're being redirected to the status page, meaning the upload
# was accepted. Check that the task would have been queued.
self.assertTrue(mocked_task.delay.called)
else:
self.assertFalse(mocked_task.delay.called)
return response
def submission_file_contents(name_in_doc, group, templatename, author=None, email=None, title=None, year=None, ascii=True):
_today = date_today() _today = date_today()
# construct appropriate text draft # construct appropriate text draft
f = io.open(os.path.join(settings.BASE_DIR, "submit", templatename)) f = io.open(os.path.join(settings.BASE_DIR, "submit", templatename))
@ -128,10 +152,18 @@ def submission_file(name_in_doc, name_in_post, group, templatename, author=None,
email=email, email=email,
title=title, title=title,
) )
return submission_text, author
def submission_file(name_in_doc, name_in_post, group, templatename, author=None, email=None, title=None, year=None, ascii=True):
submission_text, author = submission_file_contents(
name_in_doc, group, templatename, author, email, title, year, ascii
)
file = StringIO(submission_text) file = StringIO(submission_text)
file.name = name_in_post file.name = name_in_post
return file, author return file, author
def create_draft_submission_with_rev_mismatch(rev='01'): def create_draft_submission_with_rev_mismatch(rev='01'):
"""Create a draft and submission with mismatched version """Create a draft and submission with mismatched version
@ -172,31 +204,44 @@ class SubmitTests(BaseSubmitTestCase):
# Submit views assume there is a "next" IETF to look for cutoff dates against # Submit views assume there is a "next" IETF to look for cutoff dates against
MeetingFactory(type_id='ietf', date=date_today()+datetime.timedelta(days=180)) MeetingFactory(type_id='ietf', date=date_today()+datetime.timedelta(days=180))
def create_and_post_submission(self, name, rev, author, group=None, formats=("txt",), base_filename=None): def create_and_post_submission(self, name, rev, author, group=None, formats=("txt",), base_filename=None, ascii=True):
"""Helper to create and post a submission """Helper to create and post a submission
If base_filename is None, defaults to 'test_submission' If base_filename is None, defaults to 'test_submission'.
""" """
url = urlreverse('ietf.submit.views.upload_submission') url = urlreverse('ietf.submit.views.upload_submission')
files = dict() files = dict()
for format in formats: for format in formats:
fn = '.'.join((base_filename or 'test_submission', format)) fn = '.'.join((base_filename or 'test_submission', format))
files[format], __ = submission_file(f'{name}-{rev}', f'{name}-{rev}.{format}', group, fn, author=author) files[format], __ = submission_file(f'{name}-{rev}', f'{name}-{rev}.{format}', group, fn, author=author, ascii=ascii)
r = self.client.post(url, files) r = self.post_to_upload_submission(url, files)
if r.status_code != 302: if r.status_code == 302:
# A redirect means the upload was accepted and queued for processing
process_submission = True
last_submission = Submission.objects.order_by("-pk").first()
self.assertEqual(last_submission.state_id, "validating")
else:
process_submission = False
q = PyQuery(r.content) q = PyQuery(r.content)
print(q('div.invalid-feedback').text()) print(q('div.invalid-feedback').text())
self.assertNoFormPostErrors(r, ".invalid-feedback,.alert-danger") self.assertNoFormPostErrors(r, ".invalid-feedback,.alert-danger")
for format in formats: # Now process the submission like the task would do
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, format)))) if process_submission:
if format == 'xml': process_uploaded_submission(Submission.objects.order_by('-pk').first())
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, 'html')))) for format in formats:
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, format))))
if format == 'xml':
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, 'html'))))
return r return r
def do_submission(self, name, rev, group=None, formats=["txt",], author=None): def do_submission(self, name, rev, group=None, formats: Tuple[str, ...]=("txt",), author=None, base_filename=None, ascii=True):
"""Simulate uploading a draft and waiting for validation results
Returns the "full access" status URL and the author associated with the submitted draft.
"""
# break early in case of missing configuration # break early in case of missing configuration
self.assertTrue(os.path.exists(settings.IDSUBMIT_IDNITS_BINARY)) self.assertTrue(os.path.exists(settings.IDSUBMIT_IDNITS_BINARY))
@ -211,7 +256,9 @@ class SubmitTests(BaseSubmitTestCase):
# submit # submit
if author is None: if author is None:
author = PersonFactory() author = PersonFactory()
r = self.create_and_post_submission(name, rev, author, group, formats) r = self.create_and_post_submission(
name=name, rev=rev, author=author, group=group, formats=formats, base_filename=base_filename, ascii=ascii
)
status_url = r["Location"] status_url = r["Location"]
self.assertEqual(Submission.objects.filter(name=name).count(), 1) self.assertEqual(Submission.objects.filter(name=name).count(), 1)
@ -223,8 +270,10 @@ class SubmitTests(BaseSubmitTestCase):
sys.stderr.write("Author initials: %s\n" % author.initials()) sys.stderr.write("Author initials: %s\n" % author.initials())
self.assertEqual(len(submission.authors), 1) self.assertEqual(len(submission.authors), 1)
a = submission.authors[0] a = submission.authors[0]
self.assertEqual(a["name"], author.ascii_name()) if ascii:
self.assertEqual(a["email"], author.email().address.lower()) self.assertEqual(a["name"], author.ascii_name())
if author.email():
self.assertEqual(a["email"], author.email().address.lower())
self.assertEqual(a["affiliation"], "Test Centre Inc.") self.assertEqual(a["affiliation"], "Test Centre Inc.")
self.assertEqual(a["country"], "UK") self.assertEqual(a["country"], "UK")
@ -1262,11 +1311,8 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_new_wg_with_dash(self): def test_submit_new_wg_with_dash(self):
group = Group.objects.create(acronym="mars-special", name="Mars Special", type_id="wg", state_id="active") group = Group.objects.create(acronym="mars-special", name="Mars Special", type_id="wg", state_id="active")
name = "draft-ietf-%s-testing-tests" % group.acronym name = "draft-ietf-%s-testing-tests" % group.acronym
self.create_and_post_submission(name=name, rev="00", author=PersonFactory())
self.do_submission(name, "00")
self.assertEqual(Submission.objects.get(name=name).group.acronym, group.acronym) self.assertEqual(Submission.objects.get(name=name).group.acronym, group.acronym)
def test_submit_new_wg_v2_country_only(self): def test_submit_new_wg_v2_country_only(self):
@ -1292,19 +1338,15 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_new_irtf(self): def test_submit_new_irtf(self):
group = Group.objects.create(acronym="saturnrg", name="Saturn", type_id="rg", state_id="active") group = Group.objects.create(acronym="saturnrg", name="Saturn", type_id="rg", state_id="active")
name = "draft-irtf-%s-testing-tests" % group.acronym name = "draft-irtf-%s-testing-tests" % group.acronym
self.create_and_post_submission(name=name, rev="00", author=PersonFactory())
self.do_submission(name, "00") submission = Submission.objects.get(name=name)
self.assertEqual(submission.group.acronym, group.acronym)
self.assertEqual(Submission.objects.get(name=name).group.acronym, group.acronym) self.assertEqual(submission.group.type_id, group.type_id)
self.assertEqual(Submission.objects.get(name=name).group.type_id, group.type_id)
def test_submit_new_iab(self): def test_submit_new_iab(self):
name = "draft-iab-testing-tests" name = "draft-iab-testing-tests"
self.create_and_post_submission(name=name, rev="00", author=PersonFactory())
self.do_submission(name, "00")
self.assertEqual(Submission.objects.get(name=name).group.acronym, "iab") self.assertEqual(Submission.objects.get(name=name).group.acronym, "iab")
def test_cancel_submission(self): def test_cancel_submission(self):
@ -1514,7 +1556,7 @@ class SubmitTests(BaseSubmitTestCase):
rev = "00" rev = "00"
group = "mars" group = "mars"
self.do_submission(name, rev, group, ["txt", "xml", "pdf"]) self.do_submission(name, rev, group, ["txt", "xml"])
self.assertEqual(Submission.objects.filter(name=name).count(), 1) self.assertEqual(Submission.objects.filter(name=name).count(), 1)
@ -1523,8 +1565,6 @@ class SubmitTests(BaseSubmitTestCase):
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev)))) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))))
self.assertTrue(name in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read()) self.assertTrue(name in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read())
self.assertTrue('<?xml version="1.0" encoding="UTF-8"?>' in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read()) self.assertTrue('<?xml version="1.0" encoding="UTF-8"?>' in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read())
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))))
self.assertTrue('This is PDF' in io.open(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))).read())
def test_expire_submissions(self): def test_expire_submissions(self):
s = Submission.objects.create(name="draft-ietf-mars-foo", s = Submission.objects.create(name="draft-ietf-mars-foo",
@ -1630,7 +1670,7 @@ class SubmitTests(BaseSubmitTestCase):
for format in formats: for format in formats:
files[format], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.bad', group, "test_submission.bad") files[format], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.bad', group, "test_submission.bad")
r = self.client.post(url, files) r = self.post_to_upload_submission(url, files)
self.assertEqual(r.status_code, 200) self.assertEqual(r.status_code, 200)
q = PyQuery(r.content) q = PyQuery(r.content)
@ -1649,14 +1689,9 @@ class SubmitTests(BaseSubmitTestCase):
files[format], author = submission_file(name_in_doc, name_in_post, group, "test_submission.%s" % format) files[format], author = submission_file(name_in_doc, name_in_post, group, "test_submission.%s" % format)
files[format].name = name_in_post files[format].name = name_in_post
r = self.client.post(url, files) r = self.post_to_upload_submission(url, files)
self.assertEqual(r.status_code, 200) self.assertEqual(r.status_code, 200)
q = PyQuery(r.content) return r
self.assertTrue(len(q("form .invalid-feedback")) > 0)
m = q('div.invalid-feedback').text()
return r, q, m
def test_submit_bad_file_txt(self): def test_submit_bad_file_txt(self):
r, q, m = self.submit_bad_file("some name", ["txt"]) r, q, m = self.submit_bad_file("some name", ["txt"])
@ -1666,15 +1701,15 @@ class SubmitTests(BaseSubmitTestCase):
self.assertIn('document does not contain a legitimate name', m) self.assertIn('document does not contain a legitimate name', m)
def test_submit_bad_doc_name(self): def test_submit_bad_doc_name(self):
r, q, m = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo.dot-bar", name_in_post="draft-foo.dot-bar", formats=["txt"]) r = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo.dot-bar", name_in_post="draft-foo.dot-bar", formats=["txt"])
self.assertIn('contains a disallowed character with byte code: 46', m) self.assertContains(r, "contains a disallowed character with byte code: 46")
# This actually is allowed by the existing code. A significant rework of the validation mechanics is needed. # This actually is allowed by the existing code. A significant rework of the validation mechanics is needed.
# r, q, m = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo-bar-00.txt", name_in_post="draft-foo-bar-00.txt", formats=["txt"]) # r, q, m = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo-bar-00.txt", name_in_post="draft-foo-bar-00.txt", formats=["txt"])
# self.assertIn('Did you include a filename extension in the name by mistake?', m) # self.assertIn('Did you include a filename extension in the name by mistake?', m)
r, q, m = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo-bar-00.xml", name_in_post="draft-foo-bar-00.xml", formats=["xml"]) r = self.submit_bad_doc_name_with_ext(name_in_doc="draft-foo-bar-00.xml", name_in_post="draft-foo-bar-00.xml", formats=["xml"])
self.assertIn('Did you include a filename extension in the name by mistake?', m) self.assertContains(r, "Could not extract a valid Internet-Draft revision from the XML")
r, q, m = self.submit_bad_doc_name_with_ext(name_in_doc="../malicious-name-in-content-00", name_in_post="../malicious-name-in-post-00.xml", formats=["xml"]) r = self.submit_bad_doc_name_with_ext(name_in_doc="../malicious-name-in-content-00", name_in_post="../malicious-name-in-post-00.xml", formats=["xml"])
self.assertIn('Did you include a filename extension in the name by mistake?', m) self.assertContains(r, "Did you include a filename extension in the name by mistake?")
def test_submit_bad_file_xml(self): def test_submit_bad_file_xml(self):
r, q, m = self.submit_bad_file("some name", ["xml"]) r, q, m = self.submit_bad_file("some name", ["xml"])
@ -1682,12 +1717,6 @@ class SubmitTests(BaseSubmitTestCase):
self.assertIn('Expected the XML file to have extension ".xml"', m) self.assertIn('Expected the XML file to have extension ".xml"', m)
self.assertIn('Expected an XML file of type "application/xml"', m) self.assertIn('Expected an XML file of type "application/xml"', m)
def test_submit_bad_file_pdf(self):
r, q, m = self.submit_bad_file("some name", ["pdf"])
self.assertIn('Invalid characters were found in the name', m)
self.assertIn('Expected the PDF file to have extension ".pdf"', m)
self.assertIn('Expected an PDF file of type "application/pdf"', m)
def test_submit_file_in_archive(self): def test_submit_file_in_archive(self):
name = "draft-authorname-testing-file-exists" name = "draft-authorname-testing-file-exists"
rev = '00' rev = '00'
@ -1711,7 +1740,7 @@ class SubmitTests(BaseSubmitTestCase):
with io.open(fn, 'w') as f: with io.open(fn, 'w') as f:
f.write("a" * 2000) f.write("a" * 2000)
files[format], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.{format}', group, "test_submission.%s" % format) files[format], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.{format}', group, "test_submission.%s" % format)
r = self.client.post(url, files) r = self.post_to_upload_submission(url, files)
self.assertEqual(r.status_code, 200) self.assertEqual(r.status_code, 200)
q = PyQuery(r.content) q = PyQuery(r.content)
@ -1722,25 +1751,11 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_nonascii_name(self): def test_submit_nonascii_name(self):
name = "draft-authorname-testing-nonascii" name = "draft-authorname-testing-nonascii"
rev = "00" rev = "00"
group = None
# get
url = urlreverse('ietf.submit.views.upload_submission')
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
# submit
#author = PersonFactory(name=u"Jörgen Nilsson".encode('latin1'))
user = UserFactory(first_name="Jörgen", last_name="Nilsson") user = UserFactory(first_name="Jörgen", last_name="Nilsson")
author = PersonFactory(user=user) author = PersonFactory(user=user)
file, __ = submission_file(f'{name}-{rev}', f'{name}-{rev}.txt', group, "test_submission.nonascii", author=author, ascii=False) status_url, _ = self.do_submission(name=name, rev=rev, author=author, base_filename="test_submission.nonascii", ascii=False)
files = {"txt": file }
r = self.client.post(url, files)
self.assertEqual(r.status_code, 302)
status_url = r["Location"]
r = self.client.get(status_url) r = self.client.get(status_url)
q = PyQuery(r.content) q = PyQuery(r.content)
m = q('p.alert-warning').text() m = q('p.alert-warning').text()
@ -1750,19 +1765,12 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_missing_author_email(self): def test_submit_missing_author_email(self):
name = "draft-authorname-testing-noemail" name = "draft-authorname-testing-noemail"
rev = "00" rev = "00"
group = None
author = PersonFactory() author = PersonFactory()
for e in author.email_set.all(): for e in author.email_set.all():
e.delete() e.delete()
files = {"txt": submission_file(f'{name}-{rev}', f'{name}-{rev}.txt', group, "test_submission.txt", author=author, ascii=True)[0] } status_url, _ = self.do_submission(name=name, rev=rev, author=author)
# submit
url = urlreverse('ietf.submit.views.upload_submission')
r = self.client.post(url, files)
self.assertEqual(r.status_code, 302)
status_url = r["Location"]
r = self.client.get(status_url) r = self.client.get(status_url)
q = PyQuery(r.content) q = PyQuery(r.content)
m = q('p.text-danger').text() m = q('p.text-danger').text()
@ -1773,20 +1781,13 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_bad_author_email(self): def test_submit_bad_author_email(self):
name = "draft-authorname-testing-bademail" name = "draft-authorname-testing-bademail"
rev = "00" rev = "00"
group = None
author = PersonFactory() author = PersonFactory()
email = author.email_set.first() email = author.email_set.first()
email.address = '@bad.email' email.address = '@bad.email'
email.save() email.save()
files = {"xml": submission_file(f'{name}-{rev}',f'{name}-{rev}.xml', group, "test_submission.xml", author=author, ascii=False)[0] } status_url, _ = self.do_submission(name=name, rev=rev, author=author, formats=('xml',))
# submit
url = urlreverse('ietf.submit.views.upload_submission')
r = self.client.post(url, files)
self.assertEqual(r.status_code, 302)
status_url = r["Location"]
r = self.client.get(status_url) r = self.client.get(status_url)
q = PyQuery(r.content) q = PyQuery(r.content)
m = q('p.text-danger').text() m = q('p.text-danger').text()
@ -1797,15 +1798,8 @@ class SubmitTests(BaseSubmitTestCase):
def test_submit_invalid_yang(self): def test_submit_invalid_yang(self):
name = "draft-yang-testing-invalid" name = "draft-yang-testing-invalid"
rev = "00" rev = "00"
group = None
# submit status_url, _ = self.do_submission(name=name, rev=rev, base_filename="test_submission_invalid_yang")
files = {"txt": submission_file(f'{name}-{rev}', f'{name}-{rev}.txt', group, "test_submission_invalid_yang.txt")[0] }
url = urlreverse('ietf.submit.views.upload_submission')
r = self.client.post(url, files)
self.assertEqual(r.status_code, 302)
status_url = r["Location"]
r = self.client.get(status_url) r = self.client.get(status_url)
q = PyQuery(r.content) q = PyQuery(r.content)
# #
@ -2693,7 +2687,7 @@ Subject: test
for format in formats: for format in formats:
files[format], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.{format}', group, "test_submission.%s" % format) files[format], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.{format}', group, "test_submission.%s" % format)
r = self.client.post(url, files) r = self.post_to_upload_submission(url, files)
if r.status_code != 302: if r.status_code != 302:
q = PyQuery(r.content) q = PyQuery(r.content)
print(q('div.invalid-feedback span.form-text div').text()) print(q('div.invalid-feedback span.form-text div').text())
@ -2742,6 +2736,8 @@ Subject: test
@mock.patch.object(transaction, 'on_commit', lambda x: x()) @mock.patch.object(transaction, 'on_commit', lambda x: x())
@override_settings(IDTRACKER_BASE_URL='https://datatracker.example.com') @override_settings(IDTRACKER_BASE_URL='https://datatracker.example.com')
class ApiSubmissionTests(BaseSubmitTestCase): class ApiSubmissionTests(BaseSubmitTestCase):
TASK_TO_MOCK = "ietf.submit.views.process_and_accept_uploaded_submission_task"
def test_upload_draft(self): def test_upload_draft(self):
"""api_submission accepts a submission and queues it for processing""" """api_submission accepts a submission and queues it for processing"""
url = urlreverse('ietf.submit.views.api_submission') url = urlreverse('ietf.submit.views.api_submission')
@ -2750,7 +2746,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'xml': xml, 'xml': xml,
'user': author.user.username, 'user': author.user.username,
} }
with mock.patch('ietf.submit.views.process_uploaded_submission_task') as mock_task: with mock.patch(self.TASK_TO_MOCK) as mock_task:
r = self.client.post(url, data) r = self.client.post(url, data)
self.assertEqual(r.status_code, 200) self.assertEqual(r.status_code, 200)
response = r.json() response = r.json()
@ -2788,7 +2784,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'replaces': existing_draft.name, 'replaces': existing_draft.name,
} }
# mock out the task so we don't call to celery during testing! # mock out the task so we don't call to celery during testing!
with mock.patch('ietf.submit.views.process_uploaded_submission_task'): with mock.patch(self.TASK_TO_MOCK):
r = self.client.post(url, data) r = self.client.post(url, data)
self.assertEqual(r.status_code, 200) self.assertEqual(r.status_code, 200)
submission = Submission.objects.last() submission = Submission.objects.last()
@ -2806,7 +2802,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'xml': xml, 'xml': xml,
'user': 'i.dont.exist@nowhere.example.com', 'user': 'i.dont.exist@nowhere.example.com',
} }
with mock.patch('ietf.submit.views.process_uploaded_submission_task') as mock_task: with mock.patch(self.TASK_TO_MOCK) as mock_task:
r = self.client.post(url, data) r = self.client.post(url, data)
self.assertEqual(r.status_code, 400) self.assertEqual(r.status_code, 400)
response = r.json() response = r.json()
@ -2820,7 +2816,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'xml': xml, 'xml': xml,
'user': author.user.username, 'user': author.user.username,
} }
with mock.patch('ietf.submit.views.process_uploaded_submission_task') as mock_task: with mock.patch(self.TASK_TO_MOCK) as mock_task:
r = self.client.post(url, data) r = self.client.post(url, data)
self.assertEqual(r.status_code, 400) self.assertEqual(r.status_code, 400)
response = r.json() response = r.json()
@ -2834,7 +2830,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'xml': xml, 'xml': xml,
'user': author.user.username, 'user': author.user.username,
} }
with mock.patch('ietf.submit.views.process_uploaded_submission_task') as mock_task: with mock.patch(self.TASK_TO_MOCK) as mock_task:
r = self.client.post(url, data) r = self.client.post(url, data)
self.assertEqual(r.status_code, 400) self.assertEqual(r.status_code, 400)
response = r.json() response = r.json()
@ -2850,7 +2846,7 @@ class ApiSubmissionTests(BaseSubmitTestCase):
'xml': xml, 'xml': xml,
'user': author.user.username, 'user': author.user.username,
} }
with mock.patch('ietf.submit.views.process_uploaded_submission_task') as mock_task: with mock.patch(self.TASK_TO_MOCK) as mock_task:
r = self.client.post(url, data) r = self.client.post(url, data)
self.assertEqual(r.status_code, 400) self.assertEqual(r.status_code, 400)
response = r.json() response = r.json()
@ -3105,8 +3101,8 @@ class SubmissionUploadFormTests(BaseSubmitTestCase):
class AsyncSubmissionTests(BaseSubmitTestCase): class AsyncSubmissionTests(BaseSubmitTestCase):
"""Tests of async submission-related tasks""" """Tests of async submission-related tasks"""
def test_process_uploaded_submission(self): def test_process_and_accept_uploaded_submission(self):
"""process_uploaded_submission should properly process a submission""" """process_and_accept_uploaded_submission should properly process a submission"""
_today = date_today() _today = date_today()
xml, author = submission_file('draft-somebody-test-00', 'draft-somebody-test-00.xml', None, 'test_submission.xml') xml, author = submission_file('draft-somebody-test-00', 'draft-somebody-test-00.xml', None, 'test_submission.xml')
xml_data = xml.read() xml_data = xml.read()
@ -3126,7 +3122,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
self.assertFalse(txt_path.exists()) self.assertFalse(txt_path.exists())
html_path = xml_path.with_suffix('.html') html_path = xml_path.with_suffix('.html')
self.assertFalse(html_path.exists()) self.assertFalse(html_path.exists())
process_uploaded_submission(submission) process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'auth', 'accepted submission should be in auth state') self.assertEqual(submission.state_id, 'auth', 'accepted submission should be in auth state')
@ -3144,8 +3140,8 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
self.assertEqual(submission.file_size, os.stat(txt_path).st_size) self.assertEqual(submission.file_size, os.stat(txt_path).st_size)
self.assertIn('Completed submission validation checks', submission.submissionevent_set.last().desc) self.assertIn('Completed submission validation checks', submission.submissionevent_set.last().desc)
def test_process_uploaded_submission_invalid(self): def test_process_and_accept_uploaded_submission_invalid(self):
"""process_uploaded_submission should properly process an invalid submission""" """process_and_accept_uploaded_submission should properly process an invalid submission"""
xml, author = submission_file('draft-somebody-test-00', 'draft-somebody-test-00.xml', None, 'test_submission.xml') xml, author = submission_file('draft-somebody-test-00', 'draft-somebody-test-00.xml', None, 'test_submission.xml')
xml_data = xml.read() xml_data = xml.read()
xml.close() xml.close()
@ -3166,7 +3162,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
with xml_path.open('w') as f: with xml_path.open('w') as f:
f.write(xml_data) f.write(xml_data)
process_uploaded_submission(submission) process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel') self.assertEqual(submission.state_id, 'cancel')
self.assertIn('not one of the document authors', submission.submissionevent_set.last().desc) self.assertIn('not one of the document authors', submission.submissionevent_set.last().desc)
@ -3182,10 +3178,10 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
with xml_path.open('w') as f: with xml_path.open('w') as f:
f.write(re.sub(r'<email>.*</email>', '', xml_data)) f.write(re.sub(r'<email>.*</email>', '', xml_data))
process_uploaded_submission(submission) process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel') self.assertEqual(submission.state_id, 'cancel')
self.assertIn('Missing email address', submission.submissionevent_set.last().desc) self.assertIn('Email address not found for all authors', submission.submissionevent_set.last().desc)
# no title # no title
submission = SubmissionFactory( submission = SubmissionFactory(
@ -3198,7 +3194,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
with xml_path.open('w') as f: with xml_path.open('w') as f:
f.write(re.sub(r'<title>.*</title>', '<title></title>', xml_data)) f.write(re.sub(r'<title>.*</title>', '<title></title>', xml_data))
process_uploaded_submission(submission) process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel') self.assertEqual(submission.state_id, 'cancel')
self.assertIn('Could not extract a valid title', submission.submissionevent_set.last().desc) self.assertIn('Could not extract a valid title', submission.submissionevent_set.last().desc)
@ -3214,10 +3210,10 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-different-name-00.xml' xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-different-name-00.xml'
with xml_path.open('w') as f: with xml_path.open('w') as f:
f.write(xml_data) f.write(xml_data)
process_uploaded_submission(submission) process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel') self.assertEqual(submission.state_id, 'cancel')
self.assertIn('Internet-Draft filename disagrees', submission.submissionevent_set.last().desc) self.assertIn('Submission rejected: XML Internet-Draft filename', submission.submissionevent_set.last().desc)
# rev mismatch # rev mismatch
submission = SubmissionFactory( submission = SubmissionFactory(
@ -3230,10 +3226,10 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-01.xml' xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-01.xml'
with xml_path.open('w') as f: with xml_path.open('w') as f:
f.write(xml_data) f.write(xml_data)
process_uploaded_submission(submission) process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel') self.assertEqual(submission.state_id, 'cancel')
self.assertIn('revision disagrees', submission.submissionevent_set.last().desc) self.assertIn('Submission rejected: XML Internet-Draft revision', submission.submissionevent_set.last().desc)
# not xml # not xml
submission = SubmissionFactory( submission = SubmissionFactory(
@ -3246,7 +3242,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.txt' txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.txt'
with txt_path.open('w') as f: with txt_path.open('w') as f:
f.write(txt_data) f.write(txt_data)
process_uploaded_submission(submission) process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel') self.assertEqual(submission.state_id, 'cancel')
self.assertIn('Only XML Internet-Draft submissions', submission.submissionevent_set.last().desc) self.assertIn('Only XML Internet-Draft submissions', submission.submissionevent_set.last().desc)
@ -3263,7 +3259,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
with xml_path.open('w') as f: with xml_path.open('w') as f:
f.write(xml_data) f.write(xml_data)
with mock.patch('ietf.submit.utils.process_submission_xml') as mock_proc_xml: with mock.patch('ietf.submit.utils.process_submission_xml') as mock_proc_xml:
process_uploaded_submission(submission) process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertFalse(mock_proc_xml.called, 'Should not process submission not in "validating" state') self.assertFalse(mock_proc_xml.called, 'Should not process submission not in "validating" state')
self.assertEqual(submission.state_id, 'uploaded', 'State should not be changed') self.assertEqual(submission.state_id, 'uploaded', 'State should not be changed')
@ -3291,80 +3287,192 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
symbol='x', symbol='x',
) )
): ):
process_uploaded_submission(submission) process_and_accept_uploaded_submission(submission)
submission = Submission.objects.get(pk=submission.pk) # refresh submission = Submission.objects.get(pk=submission.pk) # refresh
self.assertEqual(submission.state_id, 'cancel') self.assertEqual(submission.state_id, 'cancel')
self.assertIn('fake failure', submission.submissionevent_set.last().desc) self.assertIn('fake failure', submission.submissionevent_set.last().desc)
@mock.patch('ietf.submit.tasks.process_uploaded_submission') @mock.patch('ietf.submit.tasks.process_and_accept_uploaded_submission')
def test_process_uploaded_submission_task(self, mock_method): def test_process_and_accept_uploaded_submission_task(self, mock_method):
"""process_uploaded_submission_task task should properly call its method""" """process_and_accept_uploaded_submission_task task should properly call its method"""
s = SubmissionFactory() s = SubmissionFactory()
process_uploaded_submission_task(s.pk) process_and_accept_uploaded_submission_task(s.pk)
self.assertEqual(mock_method.call_count, 1) self.assertEqual(mock_method.call_count, 1)
self.assertEqual(mock_method.call_args.args, (s,)) self.assertEqual(mock_method.call_args.args, (s,))
@mock.patch('ietf.submit.tasks.process_uploaded_submission') @mock.patch('ietf.submit.tasks.process_and_accept_uploaded_submission')
def test_process_uploaded_submission_task_ignores_invalid_id(self, mock_method): def test_process_and_accept_uploaded_submission_task_ignores_invalid_id(self, mock_method):
"""process_uploaded_submission_task should ignore an invalid submission_id""" """process_and_accept_uploaded_submission_task should ignore an invalid submission_id"""
SubmissionFactory() # be sure there is a Submission SubmissionFactory() # be sure there is a Submission
bad_pk = 9876 bad_pk = 9876
self.assertEqual(Submission.objects.filter(pk=bad_pk).count(), 0) self.assertEqual(Submission.objects.filter(pk=bad_pk).count(), 0)
process_uploaded_submission_task(bad_pk) process_and_accept_uploaded_submission_task(bad_pk)
self.assertEqual(mock_method.call_count, 0) self.assertEqual(mock_method.call_count, 0)
def test_process_submission_text_consistency_checks(self): def test_process_submission_xml(self):
"""process_submission_text should check draft metadata against submission""" xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / "draft-somebody-test-00.xml"
submission = SubmissionFactory( xml, _ = submission_file(
name='draft-somebody-test', "draft-somebody-test-00",
rev='00', "draft-somebody-test-00.xml",
title='Correct Draft Title', None,
"test_submission.xml",
title="Correct Draft Title",
) )
txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.txt' xml_path.write_text(xml.read())
output = process_submission_xml("draft-somebody-test", "00")
self.assertEqual(output["filename"], "draft-somebody-test")
self.assertEqual(output["rev"], "00")
self.assertEqual(output["title"], "Correct Draft Title")
self.assertIsNone(output["abstract"])
self.assertEqual(len(output["authors"]), 1) # not checking in detail, parsing is unreliable
self.assertIsNone(output["document_date"])
self.assertIsNone(output["pages"])
self.assertIsNone(output["words"])
self.assertIsNone(output["first_two_pages"])
self.assertIsNone(output["file_size"])
self.assertIsNone(output["formal_languages"])
self.assertEqual(output["xml_version"], "3")
# name mismatch
xml, _ = submission_file(
"draft-somebody-wrong-name-00", # name that appears in the file
"draft-somebody-test-00.xml",
None,
"test_submission.xml",
title="Correct Draft Title",
)
xml_path.write_text(xml.read())
with self.assertRaisesMessage(SubmissionError, "disagrees with submission filename"):
process_submission_xml("draft-somebody-test", "00")
# rev mismatch
xml, _ = submission_file(
"draft-somebody-test-01", # name that appears in the file
"draft-somebody-test-00.xml",
None,
"test_submission.xml",
title="Correct Draft Title",
)
xml_path.write_text(xml.read())
with self.assertRaisesMessage(SubmissionError, "disagrees with submission revision"):
process_submission_xml("draft-somebody-test", "00")
# missing title
xml, _ = submission_file(
"draft-somebody-test-00", # name that appears in the file
"draft-somebody-test-00.xml",
None,
"test_submission.xml",
title="",
)
xml_path.write_text(xml.read())
with self.assertRaisesMessage(SubmissionError, "Could not extract a valid title"):
process_submission_xml("draft-somebody-test", "00")
def test_process_submission_text(self):
txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / "draft-somebody-test-00.txt"
txt, _ = submission_file(
"draft-somebody-test-00",
"draft-somebody-test-00.txt",
None,
"test_submission.txt",
title="Correct Draft Title",
)
txt_path.write_text(txt.read())
output = process_submission_text("draft-somebody-test", "00")
self.assertEqual(output["filename"], "draft-somebody-test")
self.assertEqual(output["rev"], "00")
self.assertEqual(output["title"], "Correct Draft Title")
self.assertEqual(output["abstract"].strip(), "This document describes how to test tests.")
self.assertEqual(len(output["authors"]), 1) # not checking in detail, parsing is unreliable
self.assertLessEqual(output["document_date"] - date_today(), datetime.timedelta(days=1))
self.assertEqual(output["pages"], 2)
self.assertGreater(output["words"], 0) # make sure it got something
self.assertGreater(len(output["first_two_pages"]), 0) # make sure it got something
self.assertGreater(output["file_size"], 0) # make sure it got something
self.assertEqual(output["formal_languages"].count(), 1)
self.assertIsNone(output["xml_version"])
# name mismatch # name mismatch
txt, _ = submission_file( txt, _ = submission_file(
'draft-somebody-wrong-name-00', # name that appears in the file "draft-somebody-wrong-name-00", # name that appears in the file
'draft-somebody-test-00.xml', "draft-somebody-test-00.txt",
None, None,
'test_submission.txt', "test_submission.txt",
title='Correct Draft Title', title="Correct Draft Title",
) )
txt_path.open('w').write(txt.read()) txt_path.write_text(txt.read())
with self.assertRaisesMessage(SubmissionError, 'disagrees with submission filename'): with self.assertRaisesMessage(SubmissionError, "disagrees with submission filename"):
process_submission_text(submission) process_submission_text("draft-somebody-test", "00")
# rev mismatch # rev mismatch
txt, _ = submission_file( txt, _ = submission_file(
'draft-somebody-test-01', # name that appears in the file "draft-somebody-test-01", # name that appears in the file
'draft-somebody-test-00.xml', "draft-somebody-test-00.txt",
None, None,
'test_submission.txt', "test_submission.txt",
title='Correct Draft Title', title="Correct Draft Title",
) )
txt_path.open('w').write(txt.read()) txt_path.write_text(txt.read())
with self.assertRaisesMessage(SubmissionError, 'disagrees with submission revision'): with self.assertRaisesMessage(SubmissionError, "disagrees with submission revision"):
process_submission_text(submission) process_submission_text("draft-somebody-test", "00")
# title mismatch def test_process_and_validate_submission(self):
txt, _ = submission_file( xml_data = {
'draft-somebody-test-00', # name that appears in the file "title": "The Title",
'draft-somebody-test-00.xml', "authors": [{
None, "name": "Jane Doe",
'test_submission.txt', "email": "jdoe@example.com",
title='Not Correct Draft Title', "affiliation": "Test Centre",
"country": "UK",
}],
"xml_version": "3",
}
text_data = {
"title": "The Title",
"abstract": "This is an abstract.",
"authors": [{
"name": "John Doh",
"email": "ignored@example.com",
"affiliation": "Ignored",
"country": "CA",
}],
"document_date": date_today(),
"pages": 25,
"words": 1234,
"first_two_pages": "Pages One and Two",
"file_size": 4321,
"formal_languages": FormalLanguageName.objects.none(),
}
submission = SubmissionFactory(
state_id="validating",
file_types=".xml,.txt",
) )
txt_path.open('w').write(txt.read()) with mock.patch("ietf.submit.utils.process_submission_xml", return_value=xml_data):
with self.assertRaisesMessage(SubmissionError, 'disagrees with submission title'): with mock.patch("ietf.submit.utils.process_submission_text", return_value=text_data):
process_submission_text(submission) with mock.patch("ietf.submit.utils.render_missing_formats") as mock_render:
with mock.patch("ietf.submit.utils.apply_checkers") as mock_checkers:
process_and_validate_submission(submission)
self.assertTrue(mock_render.called)
self.assertTrue(mock_checkers.called)
submission = Submission.objects.get(pk=submission.pk)
self.assertEqual(submission.title, text_data["title"])
self.assertEqual(submission.abstract, text_data["abstract"])
self.assertEqual(submission.authors, xml_data["authors"])
self.assertEqual(submission.document_date, text_data["document_date"])
self.assertEqual(submission.pages, text_data["pages"])
self.assertEqual(submission.words, text_data["words"])
self.assertEqual(submission.first_two_pages, text_data["first_two_pages"])
self.assertEqual(submission.file_size, text_data["file_size"])
self.assertEqual(submission.xml_version, xml_data["xml_version"])
def test_status_of_validating_submission(self): def test_status_of_validating_submission(self):
s = SubmissionFactory(state_id='validating') s = SubmissionFactory(state_id='validating')
url = urlreverse('ietf.submit.views.submission_status', kwargs={'submission_id': s.pk}) url = urlreverse('ietf.submit.views.submission_status', kwargs={'submission_id': s.pk})
r = self.client.get(url) r = self.client.get(url)
self.assertContains(r, s.name) self.assertContains(r, s.name)
self.assertContains(r, 'still being processed and validated', status_code=200) self.assertContains(r, 'This submission is being processed and validated.', status_code=200)
@override_settings(IDSUBMIT_MAX_VALIDATION_TIME=datetime.timedelta(minutes=30)) @override_settings(IDSUBMIT_MAX_VALIDATION_TIME=datetime.timedelta(minutes=30))
def test_cancel_stale_submissions(self): def test_cancel_stale_submissions(self):
@ -3648,5 +3756,5 @@ class TestOldNamesAreProtected(BaseSubmitTestCase):
url = urlreverse("ietf.submit.views.upload_submission") url = urlreverse("ietf.submit.views.upload_submission")
files = {} files = {}
files["xml"], _ = submission_file("draft-something-hascapitalletters-00", "draft-something-hascapitalletters-00.xml", None, "test_submission.xml") files["xml"], _ = submission_file("draft-something-hascapitalletters-00", "draft-something-hascapitalletters-00.xml", None, "test_submission.xml")
r = self.client.post(url, files) r = self.post_to_upload_submission(url, files)
self.assertContains(r,"Case-conflicting draft name found",status_code=200) self.assertContains(r,"Case-conflicting draft name found",status_code=200)

329
ietf/submit/utils.py
View file

@ -11,7 +11,7 @@ import time
import traceback import traceback
import xml2rfc import xml2rfc
from typing import Optional # pyflakes:ignore from typing import Optional, Union # pyflakes:ignore
from unidecode import unidecode from unidecode import unidecode
from django.conf import settings from django.conf import settings
@ -40,7 +40,7 @@ from ietf.name.models import StreamName, FormalLanguageName
from ietf.person.models import Person, Email from ietf.person.models import Person, Email
from ietf.community.utils import update_name_contains_indexes_with_new_doc from ietf.community.utils import update_name_contains_indexes_with_new_doc
from ietf.submit.mail import ( announce_to_lists, announce_new_version, announce_to_authors, from ietf.submit.mail import ( announce_to_lists, announce_new_version, announce_to_authors,
send_approval_request, send_submission_confirmation, announce_new_wg_00 ) send_approval_request, send_submission_confirmation, announce_new_wg_00, send_manual_post_request )
from ietf.submit.models import ( Submission, SubmissionEvent, Preapproval, DraftSubmissionStateName, from ietf.submit.models import ( Submission, SubmissionEvent, Preapproval, DraftSubmissionStateName,
SubmissionCheck, SubmissionExtResource ) SubmissionCheck, SubmissionExtResource )
from ietf.utils import log from ietf.utils import log
@ -911,6 +911,9 @@ class SubmissionError(Exception):
"""Exception for errors during submission processing""" """Exception for errors during submission processing"""
pass pass
class InconsistentRevisionError(SubmissionError):
"""SubmissionError caused by an inconsistent revision"""
def staging_path(filename, revision, ext): def staging_path(filename, revision, ext):
if len(ext) > 0 and ext[0] != '.': if len(ext) > 0 and ext[0] != '.':
@ -1128,102 +1131,169 @@ def _normalize_title(title):
return normalize_text(title) # normalize whitespace return normalize_text(title) # normalize whitespace
def process_submission_xml(submission): def process_submission_xml(filename, revision):
"""Validate and extract info from an uploaded submission""" """Validate and extract info from an uploaded submission"""
xml_path = staging_path(submission.name, submission.rev, '.xml') xml_path = staging_path(filename, revision, '.xml')
xml_draft = XMLDraft(xml_path) xml_draft = XMLDraft(xml_path)
if submission.name != xml_draft.filename: if filename != xml_draft.filename:
raise SubmissionError('XML Internet-Draft filename disagrees with submission filename') raise SubmissionError(
if submission.rev != xml_draft.revision: f"XML Internet-Draft filename ({xml_draft.filename}) "
raise SubmissionError('XML Internet-Draft revision disagrees with submission revision') f"disagrees with submission filename ({filename})"
)
authors = xml_draft.get_author_list() if revision != xml_draft.revision:
for a in authors: raise SubmissionError(
if not a['email']: f"XML Internet-Draft revision ({xml_draft.revision}) "
raise SubmissionError(f'Missing email address for author {a}') f"disagrees with submission revision ({revision})"
)
author_emails = [a['email'].lower() for a in authors] title = _normalize_title(xml_draft.get_title())
submitter = get_person_from_name_email(**submission.submitter_parsed()) # the ** expands dict into kwargs if not title:
if not any( raise SubmissionError("Could not extract a valid title from the XML")
email.address.lower() in author_emails
for email in submitter.email_set.filter(active=True) return {
): "filename": xml_draft.filename,
raise SubmissionError(f'Submitter ({submitter}) is not one of the document authors') "rev": xml_draft.revision,
"title": title,
# Fill in the submission data "authors": [
submission.title = _normalize_title(xml_draft.get_title()) {key: auth[key] for key in ('name', 'email', 'affiliation', 'country')}
if not submission.title: for auth in xml_draft.get_author_list()
raise SubmissionError('Could not extract a valid title from the XML') ],
submission.authors = [ "abstract": None, # not supported from XML
{key: auth[key] for key in ('name', 'email', 'affiliation', 'country')} "document_date": None, # not supported from XML
for auth in authors "pages": None, # not supported from XML
] "words": None, # not supported from XML
submission.xml_version = xml_draft.xml_version "first_two_pages": None, # not supported from XML
submission.save() "file_size": None, # not supported from XML
"formal_languages": None, # not supported from XML
"xml_version": xml_draft.xml_version,
}
def process_submission_text(submission): def _turn_into_unicode(s: Optional[Union[str, bytes]]):
"""Validate/extract data from the text version of a submitted draft """Decode a possibly null string-like item as a string
This assumes the draft was uploaded as XML and extracts data that is not Copied from ietf.submit.utils.get_draft_meta(), would be nice to
currently available directly from the XML. Additional processing, e.g. from ditch this.
get_draft_meta(), would need to be added in order to support direct text
draft uploads.
""" """
text_path = staging_path(submission.name, submission.rev, '.txt') if s is None:
return ""
if isinstance(s, str):
return s
else:
try:
return s.decode("utf-8")
except UnicodeDecodeError:
try:
return s.decode("latin-1")
except UnicodeDecodeError:
return ""
def _is_valid_email(addr):
try:
validate_email(addr)
except ValidationError:
return False
return True
def process_submission_text(filename, revision):
"""Validate/extract data from the text version of a submitted draft"""
text_path = staging_path(filename, revision, '.txt')
text_draft = PlaintextDraft.from_file(text_path) text_draft = PlaintextDraft.from_file(text_path)
if submission.name != text_draft.filename: if filename != text_draft.filename:
raise SubmissionError( raise SubmissionError(
f'Text Internet-Draft filename ({text_draft.filename}) disagrees with submission filename ({submission.name})' f"Text Internet-Draft filename ({text_draft.filename}) "
f"disagrees with submission filename ({filename})"
) )
if submission.rev != text_draft.revision: if revision != text_draft.revision:
raise SubmissionError( raise SubmissionError(
f'Text Internet-Draft revision ({text_draft.revision}) disagrees with submission revision ({submission.rev})') f"Text Internet-Draft revision ({text_draft.revision}) "
text_title = _normalize_title(text_draft.get_title()) f"disagrees with submission revision ({revision})"
if not text_title: )
raise SubmissionError('Could not extract a valid title from the text') title = _normalize_title(text_draft.get_title())
if text_title != submission.title: if not title:
raise SubmissionError( # This test doesn't work well - the text_draft parser tends to grab "Abstract" as
f'Text Internet-Draft title ({text_title}) disagrees with submission title ({submission.title})') # the title if there's an empty title.
raise SubmissionError("Could not extract a title from the text")
submission.abstract = text_draft.get_abstract() # Drops \r, \n, <, >. Based on get_draft_meta() behavior
submission.document_date = text_draft.get_creation_date() trans_table = str.maketrans("", "", "\r\n<>")
submission.pages = text_draft.get_pagecount() authors = [
submission.words = text_draft.get_wordcount() {
submission.first_two_pages = ''.join(text_draft.pages[:2]) "name": fullname.translate(trans_table).strip(),
submission.file_size = os.stat(text_path).st_size "email": _turn_into_unicode(email if _is_valid_email(email) else ""),
submission.save() "affiliation": _turn_into_unicode(company),
"country": _turn_into_unicode(country),
submission.formal_languages.set( }
FormalLanguageName.objects.filter( for (fullname, _, _, _, _, email, country, company) in text_draft.get_author_list()
]
return {
"filename": text_draft.filename,
"rev": text_draft.revision,
"title": _normalize_title(text_draft.get_title()),
"authors": authors,
"abstract": text_draft.get_abstract(),
"document_date": text_draft.get_creation_date(),
"pages": text_draft.get_pagecount(),
"words": text_draft.get_wordcount(),
"first_two_pages": ''.join(text_draft.pages[:2]),
"file_size": os.stat(text_path).st_size,
"formal_languages": FormalLanguageName.objects.filter(
slug__in=text_draft.get_formal_languages() slug__in=text_draft.get_formal_languages()
) ),
) "xml_version": None, # not supported from text
}
def process_uploaded_submission(submission): def process_and_validate_submission(submission):
def abort_submission(error): """Process and validate a submission
cancel_submission(submission)
create_submission_event(None, submission, f'Submission rejected: {error}')
if submission.state_id != 'validating': Raises SubmissionError if an error is encountered.
log.log(f'Submission {submission.pk} is not in "validating" state, skipping.') """
return # do nothing if len(set(submission.file_types.split(",")).intersection({".xml", ".txt"})) == 0:
raise SubmissionError("Require XML and/or text format to process an Internet-Draft submission.")
if submission.file_types != '.xml':
abort_submission('Only XML Internet-Draft submissions can be processed.')
try: try:
process_submission_xml(submission) xml_metadata = None
if check_submission_revision_consistency(submission): # Parse XML first, if we have it
if ".xml" in submission.file_types:
xml_metadata = process_submission_xml(submission.name, submission.rev)
render_missing_formats(submission) # makes HTML and text, unless text was uploaded
# Parse text, whether uploaded or generated from XML
text_metadata = process_submission_text(submission.name, submission.rev)
if xml_metadata and xml_metadata["title"] != text_metadata["title"]:
raise SubmissionError( raise SubmissionError(
'Document revision inconsistency error in the database. ' f"Text Internet-Draft title ({text_metadata['title']}) "
'Please contact the secretariat for assistance.' f"disagrees with XML Internet-Draft title ({xml_metadata['title']})"
) )
render_missing_formats(submission)
process_submission_text(submission) # Fill in the submission from the parsed XML/text metadata
if xml_metadata is not None:
# Items preferred / only available from XML
submission.xml_version = xml_metadata["xml_version"]
submission.authors = xml_metadata["authors"]
else:
# Items to get from text only if XML not available
submission.authors = text_metadata["authors"]
# Items always to get from text, even when XML is available
submission.title = text_metadata["title"] # verified above this agrees with XML, if present
submission.abstract = text_metadata["abstract"]
submission.document_date = text_metadata["document_date"]
submission.pages = text_metadata["pages"]
submission.words = text_metadata["words"]
submission.first_two_pages = text_metadata["first_two_pages"]
submission.file_size = text_metadata["file_size"]
submission.save()
submission.formal_languages.set(text_metadata["formal_languages"])
consistency_error = check_submission_revision_consistency(submission)
if consistency_error:
raise InconsistentRevisionError(consistency_error)
set_extresources_from_existing_draft(submission) set_extresources_from_existing_draft(submission)
apply_checkers( apply_checkers(
submission, submission,
@ -1235,16 +1305,105 @@ def process_uploaded_submission(submission):
errors = [c.message for c in submission.checks.filter(passed__isnull=False) if not c.passed] errors = [c.message for c in submission.checks.filter(passed__isnull=False) if not c.passed]
if len(errors) > 0: if len(errors) > 0:
raise SubmissionError('Checks failed: ' + ' / '.join(errors)) raise SubmissionError('Checks failed: ' + ' / '.join(errors))
except SubmissionError as err: except SubmissionError:
abort_submission(err) raise # pass SubmissionErrors up the stack
except Exception: except Exception:
# convert other exceptions into SubmissionErrors
log.log(f'Unexpected exception while processing submission {submission.pk}.') log.log(f'Unexpected exception while processing submission {submission.pk}.')
log.log(traceback.format_exc()) log.log(traceback.format_exc())
abort_submission('A system error occurred while processing the submission.') raise SubmissionError('A system error occurred while processing the submission.')
# if we get here and are still "validating", accept the draft
if submission.state_id == 'validating': def submitter_is_author(submission):
submission.state_id = 'uploaded' submitter = get_person_from_name_email(**submission.submitter_parsed())
if submitter:
author_emails = [
author["email"].strip().lower()
for author in submission.authors
if "email" in author
]
return any(
email.address.lower() in author_emails
for email in submitter.email_set.filter(active=True)
)
return False
def all_authors_have_emails(submission):
return all(a["email"] for a in submission.authors)
def process_and_accept_uploaded_submission(submission):
"""Process, validate, and, if valid, accept an uploaded submission
Requires that the submitter already be set and is an author of the submitted draft.
The submission must be in the "validating" state. On success, it will be in the
"posted" state. On error, it wil be in the "cancel" state.
"""
if submission.state_id != "validating":
log.log(f'Submission {submission.pk} is not in "validating" state, skipping.')
return # do nothing
if submission.file_types != '.xml':
# permit only XML uploads for automatic acceptance
cancel_submission(submission)
create_submission_event(
None,
submission,
"Only XML Internet-Draft submissions can be processed.",
)
return
try:
process_and_validate_submission(submission)
except SubmissionError as err:
cancel_submission(submission) # changes Submission.state
create_submission_event(None, submission, f"Submission rejected: {err}")
return
if not all_authors_have_emails(submission):
cancel_submission(submission) # changes Submission.state
create_submission_event(
None,
submission,
"Submission rejected: Email address not found for all authors"
)
return
if not submitter_is_author(submission):
cancel_submission(submission) # changes Submission.state
create_submission_event(
None,
submission,
f"Submission rejected: Submitter ({submission.submitter}) is not one of the document authors",
)
return
create_submission_event(None, submission, desc="Completed submission validation checks")
accept_submission(submission)
def process_uploaded_submission(submission):
"""Process and validate an uploaded submission
The submission must be in the "validating" state. On success, it will be in the "uploaded"
state. On error, it will be in the "cancel" state.
"""
if submission.state_id != "validating":
log.log(f'Submission {submission.pk} is not in "validating" state, skipping.')
return # do nothing
try:
process_and_validate_submission(submission)
except InconsistentRevisionError as consistency_error:
submission.state_id = "manual"
submission.save()
create_submission_event(None, submission, desc="Uploaded submission (diverted to manual process)")
send_manual_post_request(None, submission, errors=dict(consistency=str(consistency_error)))
except SubmissionError as err:
cancel_submission(submission) # changes Submission.state
create_submission_event(None, submission, f"Submission rejected: {err}")
else:
submission.state_id = "uploaded"
submission.save() submission.save()
create_submission_event(None, submission, desc="Completed submission validation checks") create_submission_event(None, submission, desc="Completed submission validation checks")
accept_submission(submission)

161
ietf/submit/views.py
View file

@ -12,7 +12,7 @@ from urllib.parse import urljoin
from django.conf import settings from django.conf import settings
from django.contrib import messages from django.contrib import messages
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.db import DataError, transaction from django.db import transaction
from django.urls import reverse as urlreverse from django.urls import reverse as urlreverse
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden, HttpResponse, JsonResponse from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden, HttpResponse, JsonResponse
@ -30,13 +30,13 @@ from ietf.ietfauth.utils import has_role, role_required
from ietf.mailtrigger.utils import gather_address_lists from ietf.mailtrigger.utils import gather_address_lists
from ietf.message.models import Message, MessageAttachment from ietf.message.models import Message, MessageAttachment
from ietf.person.models import Email from ietf.person.models import Email
from ietf.submit.forms import ( SubmissionManualUploadForm, SubmissionAutoUploadForm, AuthorForm, from ietf.submit.forms import (SubmissionAutoUploadForm, AuthorForm, SubmitterForm, EditSubmissionForm,
SubmitterForm, EditSubmissionForm, PreapprovalForm, ReplacesForm, SubmissionEmailForm, MessageModelForm, PreapprovalForm, ReplacesForm, SubmissionEmailForm, MessageModelForm,
DeprecatedSubmissionAutoUploadForm ) DeprecatedSubmissionAutoUploadForm, SubmissionManualUploadForm)
from ietf.submit.mail import send_full_url, send_manual_post_request, add_submission_email, get_reply_to from ietf.submit.mail import send_full_url, send_manual_post_request, add_submission_email, get_reply_to
from ietf.submit.models import (Submission, Preapproval, SubmissionExtResource, from ietf.submit.models import (Submission, Preapproval, SubmissionExtResource,
DraftSubmissionStateName, SubmissionEmailEvent ) DraftSubmissionStateName, SubmissionEmailEvent )
from ietf.submit.tasks import process_uploaded_submission_task, poke from ietf.submit.tasks import process_uploaded_submission_task, process_and_accept_uploaded_submission_task, poke
from ietf.submit.utils import ( approvable_submissions_for_user, preapprovals_for_user, from ietf.submit.utils import ( approvable_submissions_for_user, preapprovals_for_user,
recently_approved_by_user, validate_submission, create_submission_event, docevent_from_submission, recently_approved_by_user, validate_submission, create_submission_event, docevent_from_submission,
post_submission, cancel_submission, rename_submission_files, remove_submission_files, get_draft_meta, post_submission, cancel_submission, rename_submission_files, remove_submission_files, get_draft_meta,
@ -52,64 +52,35 @@ from ietf.utils.timezone import date_today
def upload_submission(request): def upload_submission(request):
if request.method == 'POST': if request.method == "POST":
try: form = SubmissionManualUploadForm(
form = SubmissionManualUploadForm(request, data=request.POST, files=request.FILES) request, data=request.POST, files=request.FILES
if form.is_valid(): )
log('got valid submission form for %s' % form.filename) if form.is_valid():
saved_files = save_files(form) submission = get_submission(form)
authors, abstract, file_name, file_size = get_draft_meta(form, saved_files) submission.state = DraftSubmissionStateName.objects.get(slug="validating")
submission.remote_ip = form.remote_ip
submission = get_submission(form) submission.file_types = ",".join(form.file_types)
try: submission.submission_date = date_today()
fill_in_submission(form, submission, authors, abstract, file_size) submission.save()
except Exception as e: clear_existing_files(form)
log("Exception: %s\n" % e) save_files(form)
if submission and submission.id: create_submission_event(request, submission, desc="Uploaded submission")
submission.delete() # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
raise transaction.on_commit(
lambda: process_uploaded_submission_task.delay(submission.pk)
apply_checkers(submission, file_name) )
return redirect(
consistency_error = check_submission_revision_consistency(submission) "ietf.submit.views.submission_status",
if consistency_error: submission_id=submission.pk,
# A data consistency problem diverted this to manual processing - send notification access_token=submission.access_token(),
submission.state = DraftSubmissionStateName.objects.get(slug="manual") )
submission.save()
create_submission_event(request, submission, desc="Uploaded submission (diverted to manual process)")
send_manual_post_request(request, submission, errors=dict(consistency=consistency_error))
else:
# This is the usual case
create_submission_event(request, submission, desc="Uploaded submission")
# Don't add an "Uploaded new revision doevent yet, in case of cancellation
return redirect("ietf.submit.views.submission_status", submission_id=submission.pk, access_token=submission.access_token())
except IOError as e:
if "read error" in str(e): # The server got an IOError when trying to read POST data
form = SubmissionManualUploadForm(request=request)
form._errors = {}
form._errors["__all__"] = form.error_class(["There was a failure receiving the complete form data -- please try again."])
else:
raise
except ValidationError as e:
form = SubmissionManualUploadForm(request=request)
form._errors = {}
form._errors["__all__"] = form.error_class(["There was a failure converting the xml file to text -- please verify that your xml file is valid. (%s)" % e.message])
if debug.debug:
raise
except DataError as e:
form = SubmissionManualUploadForm(request=request)
form._errors = {}
form._errors["__all__"] = form.error_class(["There was a failure processing your upload -- please verify that your Internet-Draft passes idnits. (%s)" % e.message])
if debug.debug:
raise
else: else:
form = SubmissionManualUploadForm(request=request) form = SubmissionManualUploadForm(request=request)
return render(request, 'submit/upload_submission.html', return render(
{'selected': 'index', request, "submit/upload_submission.html", {"selected": "index", "form": form}
'form': form}) )
@csrf_exempt @csrf_exempt
def api_submission(request): def api_submission(request):
@ -173,7 +144,7 @@ def api_submission(request):
# Wrap in on_commit so the delayed task cannot start until the view is done with the DB # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
transaction.on_commit( transaction.on_commit(
lambda: process_uploaded_submission_task.delay(submission.pk) lambda: process_and_accept_uploaded_submission_task.delay(submission.pk)
) )
return JsonResponse( return JsonResponse(
{ {
@ -222,7 +193,7 @@ def api_submit(request):
submission = None submission = None
def err(code, text): def err(code, text):
return HttpResponse(text, status=code, content_type='text/plain') return HttpResponse(text, status=code, content_type='text/plain')
if request.method == 'GET': if request.method == 'GET':
return render(request, 'submit/api_submit_info.html') return render(request, 'submit/api_submit_info.html')
elif request.method == 'POST': elif request.method == 'POST':
@ -301,7 +272,7 @@ def api_submit(request):
except Exception as e: except Exception as e:
exception = e exception = e
raise raise
return err(500, "Exception: %s" % str(e)) return err(500, "Exception: %s" % str(e))
finally: finally:
if exception and submission: if exception and submission:
remove_submission_files(submission) remove_submission_files(submission)
@ -470,7 +441,7 @@ def submission_status(request, submission_id, access_token=None):
update_submission_external_resources(submission, extresources) update_submission_external_resources(submission, extresources)
approvals_received = submitter_form.cleaned_data['approvals_received'] approvals_received = submitter_form.cleaned_data['approvals_received']
if submission.rev == '00' and submission.group and not submission.group.is_active: if submission.rev == '00' and submission.group and not submission.group.is_active:
permission_denied(request, 'Posting a new Internet-Draft for an inactive group is not permitted.') permission_denied(request, 'Posting a new Internet-Draft for an inactive group is not permitted.')
@ -710,7 +681,7 @@ def confirm_submission(request, submission_id, auth_token):
messages.error(request, 'The submission is not in a state where it can be cancelled.') messages.error(request, 'The submission is not in a state where it can be cancelled.')
return redirect("ietf.submit.views.submission_status", submission_id=submission_id) return redirect("ietf.submit.views.submission_status", submission_id=submission_id)
else: else:
raise RuntimeError("Unexpected state in confirm_submission()") raise RuntimeError("Unexpected state in confirm_submission()")
@ -783,7 +754,7 @@ def manualpost(request):
''' '''
manual = Submission.objects.filter(state_id = "manual").distinct() manual = Submission.objects.filter(state_id = "manual").distinct()
for s in manual: for s in manual:
s.passes_checks = all([ c.passed!=False for c in s.checks.all() ]) s.passes_checks = all([ c.passed!=False for c in s.checks.all() ])
s.errors = validate_submission(s) s.errors = validate_submission(s)
@ -799,7 +770,7 @@ def manualpost(request):
def cancel_waiting_for_draft(request): def cancel_waiting_for_draft(request):
if request.method == 'POST': if request.method == 'POST':
can_cancel = has_role(request.user, "Secretariat") can_cancel = has_role(request.user, "Secretariat")
if not can_cancel: if not can_cancel:
permission_denied(request, 'You do not have permission to perform this action.') permission_denied(request, 'You do not have permission to perform this action.')
@ -808,12 +779,12 @@ def cancel_waiting_for_draft(request):
submission = get_submission_or_404(submission_id, access_token = access_token) submission = get_submission_or_404(submission_id, access_token = access_token)
cancel_submission(submission) cancel_submission(submission)
create_submission_event(request, submission, "Cancelled submission") create_submission_event(request, submission, "Cancelled submission")
if (submission.rev != "00"): if (submission.rev != "00"):
# Add a doc event # Add a doc event
docevent_from_submission(submission, "Cancelled submission for rev {}".format(submission.rev)) docevent_from_submission(submission, "Cancelled submission for rev {}".format(submission.rev))
return redirect("ietf.submit.views.manualpost") return redirect("ietf.submit.views.manualpost")
@ -826,19 +797,19 @@ def add_manualpost_email(request, submission_id=None, access_token=None):
button_text = request.POST.get('submit', '') button_text = request.POST.get('submit', '')
if button_text == 'Cancel': if button_text == 'Cancel':
return redirect("submit/manual_post.html") return redirect("submit/manual_post.html")
form = SubmissionEmailForm(request.POST) form = SubmissionEmailForm(request.POST)
if form.is_valid(): if form.is_valid():
submission_pk = form.cleaned_data['submission_pk'] submission_pk = form.cleaned_data['submission_pk']
message = form.cleaned_data['message'] message = form.cleaned_data['message']
#in_reply_to = form.cleaned_data['in_reply_to'] #in_reply_to = form.cleaned_data['in_reply_to']
# create Message # create Message
if form.cleaned_data['direction'] == 'incoming': if form.cleaned_data['direction'] == 'incoming':
msgtype = 'msgin' msgtype = 'msgin'
else: else:
msgtype = 'msgout' msgtype = 'msgout'
submission, submission_email_event = ( submission, submission_email_event = (
add_submission_email(request=request, add_submission_email(request=request,
remote_ip=remote_ip(request), remote_ip=remote_ip(request),
@ -848,15 +819,15 @@ def add_manualpost_email(request, submission_id=None, access_token=None):
message = message, message = message,
by = request.user.person, by = request.user.person,
msgtype = msgtype) ) msgtype = msgtype) )
messages.success(request, 'Email added.') messages.success(request, 'Email added.')
try: try:
draft = Document.objects.get(name=submission.name) draft = Document.objects.get(name=submission.name)
except Document.DoesNotExist: except Document.DoesNotExist:
# Assume this is revision 00 - we'll do this later # Assume this is revision 00 - we'll do this later
draft = None draft = None
if (draft != None): if (draft != None):
e = AddedMessageEvent(type="added_message", doc=draft) e = AddedMessageEvent(type="added_message", doc=draft)
e.message = submission_email_event.submissionemailevent.message e.message = submission_email_event.submissionemailevent.message
@ -866,7 +837,7 @@ def add_manualpost_email(request, submission_id=None, access_token=None):
e.desc = submission_email_event.desc e.desc = submission_email_event.desc
e.time = submission_email_event.time e.time = submission_email_event.time
e.save() e.save()
return redirect("ietf.submit.views.manualpost") return redirect("ietf.submit.views.manualpost")
except ValidationError as e: except ValidationError as e:
form = SubmissionEmailForm(request.POST) form = SubmissionEmailForm(request.POST)
@ -883,7 +854,7 @@ def add_manualpost_email(request, submission_id=None, access_token=None):
initial['submission_pk'] = submission.pk initial['submission_pk'] = submission.pk
else: else:
initial['direction'] = 'incoming' initial['direction'] = 'incoming'
form = SubmissionEmailForm(initial=initial) form = SubmissionEmailForm(initial=initial)
return render(request, 'submit/add_submit_email.html',dict(form=form)) return render(request, 'submit/add_submit_email.html',dict(form=form))
@ -914,20 +885,20 @@ def send_submission_email(request, submission_id, message_id=None):
reply_to = form.cleaned_data['reply_to'], reply_to = form.cleaned_data['reply_to'],
body = form.cleaned_data['body'] body = form.cleaned_data['body']
) )
in_reply_to_id = form.cleaned_data['in_reply_to_id'] in_reply_to_id = form.cleaned_data['in_reply_to_id']
in_reply_to = None in_reply_to = None
rp = "" rp = ""
if in_reply_to_id: if in_reply_to_id:
rp = " reply" rp = " reply"
try: try:
in_reply_to = Message.objects.get(id=in_reply_to_id) in_reply_to = Message.objects.get(id=in_reply_to_id)
except Message.DoesNotExist: except Message.DoesNotExist:
log("Unable to retrieve in_reply_to message: %s" % in_reply_to_id) log("Unable to retrieve in_reply_to message: %s" % in_reply_to_id)
desc = "Sent message {} - manual post - {}-{}".format(rp, desc = "Sent message {} - manual post - {}-{}".format(rp,
submission.name, submission.name,
submission.rev) submission.rev)
SubmissionEmailEvent.objects.create( SubmissionEmailEvent.objects.create(
submission = submission, submission = submission,
@ -941,14 +912,14 @@ def send_submission_email(request, submission_id, message_id=None):
send_mail_message(None,msg) send_mail_message(None,msg)
messages.success(request, 'Email sent.') messages.success(request, 'Email sent.')
return redirect('ietf.submit.views.submission_status', return redirect('ietf.submit.views.submission_status',
submission_id=submission.id, submission_id=submission.id,
access_token=submission.access_token()) access_token=submission.access_token())
else: else:
reply_to = get_reply_to() reply_to = get_reply_to()
msg = None msg = None
if not message_id: if not message_id:
addrs = gather_address_lists('sub_confirmation_requested',submission=submission).as_strings(compact=False) addrs = gather_address_lists('sub_confirmation_requested',submission=submission).as_strings(compact=False)
to_email = addrs.to to_email = addrs.to
@ -958,7 +929,7 @@ def send_submission_email(request, submission_id, message_id=None):
try: try:
submitEmail = SubmissionEmailEvent.objects.get(id=message_id) submitEmail = SubmissionEmailEvent.objects.get(id=message_id)
msg = submitEmail.message msg = submitEmail.message
if msg: if msg:
to_email = msg.frm to_email = msg.frm
cc = msg.cc cc = msg.cc
@ -979,24 +950,24 @@ def send_submission_email(request, submission_id, message_id=None):
'subject': subject, 'subject': subject,
'reply_to': reply_to, 'reply_to': reply_to,
} }
if msg: if msg:
initial['in_reply_to_id'] = msg.id initial['in_reply_to_id'] = msg.id
form = MessageModelForm(initial=initial) form = MessageModelForm(initial=initial)
return render(request, "submit/email.html", { return render(request, "submit/email.html", {
'submission': submission, 'submission': submission,
'access_token': submission.access_token(), 'access_token': submission.access_token(),
'form':form}) 'form':form})
def show_submission_email_message(request, submission_id, message_id, access_token=None): def show_submission_email_message(request, submission_id, message_id, access_token=None):
submission = get_submission_or_404(submission_id, access_token) submission = get_submission_or_404(submission_id, access_token)
submitEmail = get_object_or_404(SubmissionEmailEvent, pk=message_id) submitEmail = get_object_or_404(SubmissionEmailEvent, pk=message_id)
attachments = submitEmail.message.messageattachment_set.all() attachments = submitEmail.message.messageattachment_set.all()
return render(request, 'submit/submission_email.html', return render(request, 'submit/submission_email.html',
{'submission': submission, {'submission': submission,
'message': submitEmail, 'message': submitEmail,
@ -1007,25 +978,25 @@ def show_submission_email_attachment(request, submission_id, message_id, filenam
message = get_object_or_404(SubmissionEmailEvent, pk=message_id) message = get_object_or_404(SubmissionEmailEvent, pk=message_id)
attach = get_object_or_404(MessageAttachment, attach = get_object_or_404(MessageAttachment,
message=message.message, message=message.message,
filename=filename) filename=filename)
if attach.encoding == "base64": if attach.encoding == "base64":
body = base64.b64decode(attach.body) body = base64.b64decode(attach.body)
else: else:
body = attach.body.encode('utf-8') body = attach.body.encode('utf-8')
if attach.content_type is None: if attach.content_type is None:
content_type='text/plain' content_type='text/plain'
else: else:
content_type=attach.content_type content_type=attach.content_type
response = HttpResponse(body, content_type=content_type) response = HttpResponse(body, content_type=content_type)
response['Content-Disposition'] = 'attachment; filename=%s' % attach.filename response['Content-Disposition'] = 'attachment; filename=%s' % attach.filename
response['Content-Length'] = len(body) response['Content-Length'] = len(body)
return response return response
def get_submission_or_404(submission_id, access_token=None): def get_submission_or_404(submission_id, access_token=None):
submission = get_object_or_404(Submission, pk=submission_id) submission = get_object_or_404(Submission, pk=submission_id)

28
ietf/templates/submit/submission_status.html
View file

@ -133,17 +133,31 @@
This submission is awaiting the first Internet-Draft upload. This submission is awaiting the first Internet-Draft upload.
</p> </p>
{% elif submission.state_id == 'validating' %} {% elif submission.state_id == 'validating' %}
<p class="alert alert-warning my-3"> <div class="alert alert-danger my-3">
This submission is still being processed and validated. This normally takes a few minutes after Notice: The Internet-Draft submission process has changed as of Datatracker version 10.3.0.
Your Internet-Draft is currently being processed and validated asynchronously. Results will be
displayed at this URL when they are available. If JavaScript is enabled in your
browser, this page will refreshed automatically. If JavaScript is not enabled, or if you
disable the automatic refresh with the toggle below, please reload this page in a few
minutes to see the results.
</div>
<div class="alert alert-warning my-3">
This submission is being processed and validated. This normally takes a few minutes after
submission. submission.
{% with earliest_event=submission.submissionevent_set.last %} {% with earliest_event=submission.submissionevent_set.last %}
{% if earliest_event %} {% if earliest_event %}
It has been {{ earliest_event.time|timesince }} since submission. Your draft was uploaded at {{ earliest_event.time }}<span id="time-since-uploaded" class="d-none">
{% endif %} ({{ earliest_event.time|timesince }} ago)</span>.
{% endif %}
{% endwith %} {% endwith %}
Please contact the secretariat for assistance if it has been more than an hour. Please contact the secretariat for assistance if it has been more than an hour.
</p>
{% else %} <div class="form-check form-switch mt-3 d-none">{# hide with d-none unless javascript makes it visible #}
<input class="form-check-input" type="checkbox" id="enableAutoReload" checked>
<label class="form-check-label" for="enableAutoReload"> Refresh automatically </label>
</div>
</div>
{% else %}
<h2 class="mt-5">Meta-data from the submission</h2> <h2 class="mt-5">Meta-data from the submission</h2>
{% if errors %} {% if errors %}
<div class="alert alert-danger my-3"> <div class="alert alert-danger my-3">

11
ietf/templates/submit/upload_submission.html
View file

@ -47,7 +47,8 @@
aria-controls="other-formats"> aria-controls="other-formats">
<input class="form-check-input" <input class="form-check-input"
id="checkbox" id="checkbox"
type="checkbox"> type="checkbox"
{% if form.errors.txt %}checked {% endif %}>
Submit other formats Submit other formats
</label> </label>
</div> </div>
@ -60,14 +61,8 @@
However, if you cannot for some reason submit XML, you must However, if you cannot for some reason submit XML, you must
submit a plaintext rendering of your I-D. submit a plaintext rendering of your I-D.
</p> </p>
{% bootstrap_label '<i class="bi bi-file-pdf"></i> PDF rendering of the I-D' label_class="form-label fw-bold" %}
{% bootstrap_field form.pdf show_label=False %}
<p class="form-text">
Optional to submit, will be auto-generated based
on the submitted XML.
</p>
</div> </div>
{% bootstrap_form_errors form %} {% bootstrap_form_errors form type="non_fields" %}
{% bootstrap_button button_type="submit" name="upload" content="Upload" %} {% bootstrap_button button_type="submit" name="upload" content="Upload" %}
</form> </form>
{% include "submit/problem-reports-footer.html" %} {% include "submit/problem-reports-footer.html" %}