Merge forward to 6.31.1.dev0

- Legacy-Id: 11899

ietf/doc/migrations/0014_auto_20160524_2147.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('message', '__first__'),
+        ('doc', '0013_auto_20151027_1127'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AddedMessageEvent',
+            fields=[
+                ('docevent_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='doc.DocEvent')),
+                ('msgtype', models.CharField(max_length=25)),
+                ('in_reply_to', models.ForeignKey(related_name='doc_irtomanual', blank=True, to='message.Message', null=True)),
+                ('message', models.ForeignKey(related_name='doc_manualevents', blank=True, to='message.Message', null=True)),
+            ],
+            options={
+            },
+            bases=('doc.docevent',),
+        ),
+        migrations.AlterField(
+            model_name='docevent',
+            name='type',
+            field=models.CharField(max_length=50, choices=[(b'new_revision', b'Added new revision'), (b'changed_document', b'Changed document metadata'), (b'added_comment', b'Added comment'), (b'added_message', b'Added message'), (b'deleted', b'Deleted document'), (b'changed_state', b'Changed state'), (b'changed_stream', b'Changed document stream'), (b'expired_document', b'Expired document'), (b'extended_expiry', b'Extended expiry of document'), (b'requested_resurrect', b'Requested resurrect'), (b'completed_resurrect', b'Completed resurrect'), (b'changed_consensus', b'Changed consensus'), (b'published_rfc', b'Published RFC'), (b'added_suggested_replaces', b'Added suggested replacement relationships'), (b'reviewed_suggested_replaces', b'Reviewed suggested replacement relationships'), (b'changed_group', b'Changed group'), (b'changed_protocol_writeup', b'Changed protocol writeup'), (b'changed_charter_milestone', b'Changed charter milestone'), (b'initial_review', b'Set initial review time'), (b'changed_review_announcement', b'Changed WG Review text'), (b'changed_action_announcement', b'Changed WG Action text'), (b'started_iesg_process', b'Started IESG process on document'), (b'created_ballot', b'Created ballot'), (b'closed_ballot', b'Closed ballot'), (b'sent_ballot_announcement', b'Sent ballot announcement'), (b'changed_ballot_position', b'Changed ballot position'), (b'changed_ballot_approval_text', b'Changed ballot approval text'), (b'changed_ballot_writeup_text', b'Changed ballot writeup text'), (b'changed_rfc_editor_note_text', b'Changed RFC Editor Note text'), (b'changed_last_call_text', b'Changed last call text'), (b'requested_last_call', b'Requested last call'), (b'sent_last_call', b'Sent last call'), (b'scheduled_for_telechat', b'Scheduled for telechat'), (b'iesg_approved', b'IESG approved document (no problem)'), (b'iesg_disapproved', b'IESG disapproved document (do not publish)'), (b'approved_in_minute', b'Approved in minute'), (b'iana_review', b'IANA review comment'), (b'rfc_in_iana_registry', b'RFC is in IANA registry'), (b'rfc_editor_received_announcement', b'Announcement was received by RFC Editor'), (b'requested_publication', b'Publication at RFC Editor requested')]),
+            preserve_default=True,
+        ),
+    ]

ietf/doc/migrations/0015_auto_20160824_2218.py

@@ -7,7 +7,7 @@ from django.db import models, migrations
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('doc', '0013_auto_20151027_1127'),
+        ('doc', '0014_auto_20160524_2147'),
     ]
 
     operations = [

ietf/doc/models.py

@@ -619,6 +619,7 @@ EVENT_TYPES = [
     ("new_revision", "Added new revision"),
     ("changed_document", "Changed document metadata"),
     ("added_comment", "Added comment"),
+    ("added_message", "Added message"),
 
     ("deleted", "Deleted document"),
 
@@ -811,6 +812,13 @@ class InitialReviewDocEvent(DocEvent):
     expires = models.DateTimeField(blank=True, null=True)
 
 
+class AddedMessageEvent(DocEvent):
+    import ietf.message.models
+    message     = models.ForeignKey(ietf.message.models.Message, null=True, blank=True,related_name='doc_manualevents')
+    msgtype     = models.CharField(max_length=25)
+    in_reply_to = models.ForeignKey(ietf.message.models.Message, null=True, blank=True,related_name='doc_irtomanual')
+
+
 # dumping store for removed events
 class DeletedEvent(models.Model):
     content_type = models.ForeignKey(ContentType)

ietf/doc/resources.py

@@ -11,7 +11,7 @@ from ietf.doc.models import (BallotType, DeletedEvent, StateType, State, Documen
     DocumentAuthor, DocEvent, StateDocEvent, DocHistory, ConsensusDocEvent, DocAlias,
     TelechatDocEvent, DocReminder, LastCallDocEvent, NewRevisionDocEvent, WriteupDocEvent,
     InitialReviewDocEvent, DocHistoryAuthor, BallotDocEvent, RelatedDocument,
-    RelatedDocHistory, BallotPositionDocEvent)
+    RelatedDocHistory, BallotPositionDocEvent, AddedMessageEvent)
 
 
 from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource
@@ -513,3 +513,32 @@ class BallotPositionDocEventResource(ModelResource):
         }
 api.doc.register(BallotPositionDocEventResource())
 
+
+
+from ietf.person.resources import PersonResource
+from ietf.message.resources import MessageResource
+class AddedMessageEventResource(ModelResource):
+    by               = ToOneField(PersonResource, 'by')
+    doc              = ToOneField(DocumentResource, 'doc')
+    docevent_ptr     = ToOneField(DocEventResource, 'docevent_ptr')
+    message          = ToOneField(MessageResource, 'message', null=True)
+    in_reply_to      = ToOneField(MessageResource, 'in_reply_to', null=True)
+    class Meta:
+        queryset = AddedMessageEvent.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'addedmessageevent'
+        filtering = { 
+            "id": ALL,
+            "time": ALL,
+            "type": ALL,
+            "desc": ALL,
+            "msgtype": ALL,
+            "by": ALL_WITH_RELATIONS,
+            "doc": ALL_WITH_RELATIONS,
+            "docevent_ptr": ALL_WITH_RELATIONS,
+            "message": ALL_WITH_RELATIONS,
+            "in_reply_to": ALL_WITH_RELATIONS,
+        }
+api.doc.register(AddedMessageEventResource())
+

ietf/doc/utils.py

@@ -276,6 +276,16 @@ def add_links_in_new_revision_events(doc, events, diff_revisions):
         prev = diff_url
 
 
+def add_events_message_info(events):
+    for e in events:
+        if not e.type == "added_message":
+            continue
+
+        e.message = e.addedmessageevent.message
+        e.msgtype = e.addedmessageevent.msgtype
+        e.in_reply_to = e.addedmessageevent.in_reply_to
+
+
 def get_document_content(key, filename, split=True, markup=True):
     try:
         with open(filename, 'rb') as f:

ietf/doc/views_doc.py

@@ -50,7 +50,8 @@ from ietf.doc.models import ( Document, DocAlias, DocHistory, DocEvent, BallotDo
 from ietf.doc.utils import ( add_links_in_new_revision_events, augment_events_with_revision,
     can_adopt_draft, get_chartering_type, get_document_content, get_tags_for_stream_id,
     needed_ballot_positions, nice_consensus, prettify_std_name, update_telechat, has_same_ballot,
-    get_initial_notify, make_notify_changed_event, crawl_history, default_consensus)
+    get_initial_notify, make_notify_changed_event, crawl_history, default_consensus,
+    add_events_message_info)
 from ietf.community.utils import augment_docs_with_tracking_info
 from ietf.group.models import Role
 from ietf.group.utils import can_manage_group, can_manage_materials
@@ -657,6 +658,7 @@ def document_history(request, name):
 
     augment_events_with_revision(doc, events)
     add_links_in_new_revision_events(doc, events, diff_revisions)
+    add_events_message_info(events)
 
     # figure out if the current user can add a comment to the history
     if doc.type_id == "draft" and doc.group != None:

ietf/mailtrigger/migrations/0006_auto_20160707_1933.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+
+    Recipient=apps.get_model('mailtrigger','Recipient')
+
+    rc = Recipient.objects.create
+
+    rc(slug='manualpost_message',
+       desc='The IETF manual post processing system',
+       template='<ietf-manualpost@ietf.org>')
+
+
+def reverse(apps, schema_editor):
+    Recipient=apps.get_model('mailtrigger','Recipient')
+
+    Recipient.objects.filter(slug='manualpost_message').delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mailtrigger', '0005_interim_trigger'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]

ietf/mailtrigger/utils.py

@@ -80,6 +80,9 @@ def gather_relevant_expansions(**kwargs):
             rule_list.append((mailtrigger.slug,mailtrigger.desc,addrs.to,addrs.cc))
     return sorted(rule_list)
 
+def get_base_submission_message_address():
+    return Recipient.objects.get(slug='manualpost_message').gather()[0]
+
 def get_base_ipr_request_address():
     return Recipient.objects.get(slug='ipr_requests').gather()[0]
 

ietf/message/models.py

@@ -28,6 +28,19 @@ class Message(models.Model):
     def __unicode__(self):
         return "'%s' %s -> %s" % (self.subject, self.frm, self.to)
 
+
+class MessageAttachment(models.Model):
+    message = models.ForeignKey(Message)
+    filename = models.CharField(max_length=255, db_index=True, blank=True)
+    content_type = models.CharField(max_length=255, blank=True)
+    encoding = models.CharField(max_length=255, blank=True)
+    removed = models.BooleanField(default=False)
+    body = models.TextField()
+
+    def __unicode__(self):
+        return self.filename
+
+
 class SendQueue(models.Model):
     time = models.DateTimeField(default=datetime.datetime.now)
     by = models.ForeignKey(Person)

ietf/message/resources.py

@@ -7,12 +7,12 @@ from tastypie.cache import SimpleCache
 
 from ietf import api
 
-from ietf.message.models import Message, SendQueue
-
+from ietf.message.models import Message, SendQueue, MessageAttachment
 
 from ietf.person.resources import PersonResource
 from ietf.group.resources import GroupResource
 from ietf.doc.resources import DocumentResource
+
 class MessageResource(ModelResource):
     by               = ToOneField(PersonResource, 'by')
     related_groups   = ToManyField(GroupResource, 'related_groups', null=True)
@@ -59,3 +59,21 @@ class SendQueueResource(ModelResource):
         }
 api.message.register(SendQueueResource())
 
+
+
+class MessageAttachmentResource(ModelResource):
+    message          = ToOneField(MessageResource, 'message')
+    class Meta:
+        queryset = MessageAttachment.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'messageattachment'
+        filtering = { 
+            "id": ALL,
+            "filename": ALL,
+            "removed": ALL,
+            "body": ALL,
+            "message": ALL_WITH_RELATIONS,
+        }
+api.message.register(MessageAttachmentResource())
+

ietf/name/fixtures/names.json

@@ -188,7 +188,7 @@
   "order": 0,
   "revname": "Conflict reviewed by",
   "used": true,
-  "name": "Conflict reviews",
+  "name": "conflict reviews",
   "desc": ""
  },
  "model": "name.docrelationshipname",
@@ -872,6 +872,20 @@
  "model": "name.draftsubmissionstatename",
  "pk": "posted"
 },
+{
+ "fields": {
+  "order": 8,
+  "next_states": [
+   "cancel",
+   "posted"
+  ],
+  "used": true,
+  "name": "Manual Post Awaiting Draft",
+  "desc": ""
+ },
+ "model": "name.draftsubmissionstatename",
+ "pk": "manual-awaiting-draft"
+},
 {
  "fields": {
   "order": 0,
@@ -4903,6 +4917,14 @@
  "model": "mailtrigger.recipient",
  "pk": "logged_in_person"
 },
+{
+ "fields": {
+  "template": "<ietf-manualpost@ietf.org>",
+  "desc": "The IETF manual post processing system"
+ },
+ "model": "mailtrigger.recipient",
+ "pk": "manualpost_message"
+},
 {
  "fields": {
   "template": "<new-work@ietf.org>",

ietf/submit/forms.py

@@ -1,6 +1,7 @@
 import os
 import re
 import datetime
+import email
 import pytz
 import xml2rfc
 import tempfile
@@ -16,7 +17,9 @@ from ietf.doc.models import Document
 from ietf.group.models import Group
 from ietf.ietfauth.utils import has_role
 from ietf.doc.fields import SearchableDocAliasesField
+from ietf.ipr.mail import utc_from_string
 from ietf.meeting.models import Meeting
+from ietf.message.models import Message
 from ietf.submit.models import Submission, Preapproval
 from ietf.submit.utils import validate_submission_rev, validate_submission_document_date
 from ietf.submit.parsers.pdf_parser import PDFParser
@@ -223,7 +226,7 @@ class SubmissionUploadForm(forms.Form):
             self.group = self.deduce_group()
 
             # check existing
-            existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel"))
+            existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel", "manual-awaiting-draft"))
             if existing:
                 raise forms.ValidationError(mark_safe('A submission with same name and revision is currently being processed. <a href="%s">Check the status here.</a>' % urlreverse("submit_submission_status", kwargs={ 'submission_id': existing[0].pk })))
 
@@ -320,6 +323,9 @@ class NameEmailForm(forms.Form):
     name = forms.CharField(required=True)
     email = forms.EmailField(label=u'Email address')
 
+    #Fields for secretariat only
+    approvals_received = forms.BooleanField(label=u'Approvals received', required=False, initial=False)
+
     def __init__(self, *args, **kwargs):
         email_required = kwargs.pop("email_required", True)
         super(NameEmailForm, self).__init__(*args, **kwargs)
@@ -421,3 +427,86 @@ class PreapprovalForm(forms.Form):
             raise forms.ValidationError("A draft with this name has already been submitted and accepted. A pre-approval would not make any difference.")
 
         return n
+
+
+class SubmissionEmailForm(forms.Form):
+    '''
+    Used to add a message to a submission or to create a new submission.
+    This message is NOT a reply to a previous message but has arrived out of band
+    
+    if submission_pk is None we are startign a new submission and name
+    must be unique. Otehrwise the name must match the submission.name.
+    '''
+    name = forms.CharField(required=True, max_length=255, label="Draft name")
+    submission_pk = forms.IntegerField(required=False, widget=forms.HiddenInput())
+    direction = forms.ChoiceField(choices=(("incoming", "Incoming"), ("outgoing", "Outgoing")),
+                                  widget=forms.RadioSelect)
+    message = forms.CharField(required=True, widget=forms.Textarea,
+                              help_text="Copy the entire message including headers. To do so, view the source, select all, copy then paste into the text area above")
+    #in_reply_to = MessageModelChoiceField(queryset=Message.objects,label="In Reply To",required=False)
+
+    def __init__(self, *args, **kwargs):
+        super(SubmissionEmailForm, self).__init__(*args, **kwargs)
+
+    def clean_message(self):
+        '''Returns a ietf.message.models.Message object'''
+        self.message_text = self.cleaned_data['message']
+        try:
+            message = email.message_from_string(self.message_text)
+        except Exception as e:
+            self.add_error('message', e)
+            return None
+            
+        for field in ('to','from','subject','date'):
+            if not message[field]:
+                raise forms.ValidationError('Error parsing email: {} field not found.'.format(field))
+        date = utc_from_string(message['date'])
+        if not isinstance(date,datetime.datetime):
+            raise forms.ValidationError('Error parsing email date field')
+        return message
+
+    def clean(self):
+        if any(self.errors):
+            return self.cleaned_data
+        super(SubmissionEmailForm, self).clean()
+        name = self.cleaned_data['name']
+        match = re.search(r"(draft-[a-z0-9-]*)-(\d\d)", name)
+        if not match:
+            self.add_error('name', 
+                           "Submission name {} must start with 'draft-' and only contain digits, lowercase letters and dash characters and end with revision.".format(name))
+        else:
+            self.draft_name = match.group(1)    
+            self.revision = match.group(2)
+
+            error = validate_submission_rev(self.draft_name, self.revision)
+            if error:
+                raise forms.ValidationError(error)
+
+        #in_reply_to = self.cleaned_data['in_reply_to']
+        #message = self.cleaned_data['message']
+        direction = self.cleaned_data['direction']
+        if direction != 'incoming' and direction != 'outgoing':
+            self.add_error('direction', "Must be one of 'outgoing' or 'incoming'")
+
+        #if in_reply_to:
+        #    if direction != 'incoming':
+        #        raise forms.ValidationError('Only incoming messages can have In Reply To selected')
+        #    date = utc_from_string(message['date'])
+        #    if date < in_reply_to.time:
+        #        raise forms.ValidationError('The incoming message must have a date later than the message it is replying to')
+
+        return self.cleaned_data
+
+class MessageModelForm(forms.ModelForm):
+    in_reply_to_id = forms.CharField(required=False, widget=forms.HiddenInput())
+    
+    class Meta:
+        model = Message
+        fields = ['to','frm','cc','bcc','reply_to','subject','body']
+        exclude = ['time','by','content_type','related_groups','related_docs']
+
+    def __init__(self, *args, **kwargs):
+        super(MessageModelForm, self).__init__(*args, **kwargs)
+        self.fields['frm'].label='From'
+        self.fields['frm'].widget.attrs['readonly'] = 'True'
+        self.fields['reply_to'].widget.attrs['readonly'] = 'True'

ietf/submit/mail.py

@@ -1,14 +1,28 @@
+import re
+import email
+import datetime
+import base64
+import os
+
+import pyzmail
+
 from django.conf import settings
 from django.core.urlresolvers import reverse as urlreverse
+from django.core.validators import ValidationError
 from django.contrib.sites.models import Site
 from django.template.loader import render_to_string
 
+from ietf.utils.log import log
 from ietf.utils.mail import send_mail, send_mail_message
 from ietf.doc.models import Document
+from ietf.ipr.mail import utc_from_string
+from ietf.mailtrigger.utils import gather_address_lists, \
+    get_base_submission_message_address
 from ietf.person.models import Person
-from ietf.message.models import Message
+from ietf.message.models import Message, MessageAttachment
 from ietf.utils.accesstoken import generate_access_token
-from ietf.mailtrigger.utils import gather_address_lists
+from ietf.submit.models import SubmissionEmail, Submission
+
 
 def send_submission_confirmation(request, submission):
     subject = 'Confirm submission of I-D %s' % submission.name
@@ -120,3 +134,183 @@ def announce_to_authors(request, submission):
               {'submission': submission,
                'group': group},
               cc=cc)
+
+
+def get_reply_to():
+    """Returns a new reply-to address for use with an outgoing message.  This is an
+    address with "plus addressing" using a random string.  Guaranteed to be unique"""
+    local,domain = get_base_submission_message_address().split('@')
+    while True:
+        rand = base64.urlsafe_b64encode(os.urandom(12))
+        address = "{}+{}@{}".format(local,rand,domain)
+        q = Message.objects.filter(reply_to=address)
+        if not q:
+            return address
+
+
+def process_response_email(msg):
+    """Saves an incoming message.  msg=string.  Message "To" field is expected to
+    be in the format ietf-submit+[identifier]@ietf.org.  Expect to find a message with
+    a matching value in the reply_to field, associated to a submission.
+    Create a Message object for the incoming message and associate it to
+    the original message via new SubmissionEvent"""
+    message = email.message_from_string(msg)
+    to = message.get('To')
+
+    # exit if this isn't a response we're interested in (with plus addressing)
+    local,domain = get_base_submission_message_address().split('@')
+    if not re.match(r'^{}\+[a-zA-Z0-9_\-]{}@{}'.format(local,'{16}',domain),to):
+        return None
+
+    try:
+        to_message = Message.objects.get(reply_to=to)
+    except Message.DoesNotExist:
+        log('Error finding matching message ({})'.format(to))
+        return None
+
+    try:
+        submission = to_message.manualevents.first().submission
+    except:
+        log('Error processing message ({})'.format(to))
+        return None
+
+    if not submission:
+        log('Error processing message - no submission ({})'.format(to))
+        return None
+
+    parts = pyzmail.parse.get_mail_parts(message)
+    body=''
+    for part in parts:
+        if part.is_body == 'text/plain' and part.disposition == None:
+            payload, used_charset = pyzmail.decode_text(part.get_payload(), part.charset, None)
+            body = body + payload + '\n'
+
+    by = Person.objects.get(name="(System)")
+    msg = submit_message_from_message(message, body, by)
+
+    desc = "Email: received message - manual post - {}-{}".format(
+            submission.name,
+            submission.rev)
+    
+    submission_email_event = SubmissionEmail.objects.create(
+            submission = submission,
+            desc = desc,
+            msgtype = 'msgin',
+            by = by,
+            message = msg,
+            in_reply_to = to_message
+    )
+
+    save_submission_email_attachments(submission_email_event, parts)
+
+    log(u"Received submission email from %s" % msg.frm)
+    return msg
+
+
+def add_submission_email(request, remote_ip, name, rev, submission_pk, message, by, msgtype):
+    """Add email to submission history"""
+
+    #in_reply_to = form.cleaned_data['in_reply_to']
+    # create Message
+    parts = pyzmail.parse.get_mail_parts(message)
+    body=''
+    for part in parts:
+        if part.is_body == 'text/plain' and part.disposition == None:
+            payload, used_charset = pyzmail.decode_text(part.get_payload(), part.charset, None)
+            body = body + payload + '\n'
+
+    msg = submit_message_from_message(message, body, by)
+
+    if (submission_pk != None):
+        # Must exist - we're adding a message to an existing submission
+        submission = Submission.objects.get(pk=submission_pk)
+    else:
+        # Must not exist
+        submissions = Submission.objects.filter(name=name,rev=rev).exclude(state_id='cancel')
+        if submissions.count() > 0:
+            raise ValidationError("Submission {} already exists".format(name))
+            
+        # create Submission using the name
+        try:
+            submission = Submission.objects.create(
+                    state_id="manual-awaiting-draft",
+                    remote_ip=remote_ip,
+                    name=name,
+                    rev=rev,
+                    title=name,
+                    note="",
+                    submission_date=datetime.date.today(),
+                    replaces="",
+            )
+            from ietf.submit.utils import create_submission_event, docevent_from_submission
+            desc = "Submission created for rev {} in response to email".format(rev)
+            create_submission_event(request, 
+                                    submission,
+                                    desc)
+            docevent_from_submission(request,
+                                     submission,
+                                     desc)
+        except Exception as e:
+            log("Exception: %s\n" % e)
+            raise
+
+    if msgtype == 'msgin':
+        rs = "Received"
+    else:
+        rs = "Sent"
+
+    desc = "{} message - manual post - {}-{}".format(rs, name, rev)
+    submission_email_event = SubmissionEmail.objects.create(
+            desc = desc,
+            submission = submission,
+            msgtype = msgtype,
+            by = by,
+            message = msg)
+    #in_reply_to = in_reply_to
+
+    save_submission_email_attachments(submission_email_event, parts)
+    return submission, submission_email_event
+        
+        
+def submit_message_from_message(message,body,by=None):
+    """Returns a ietf.message.models.Message.  msg=email.Message
+        A copy of mail.message_from_message with different body handling
+    """
+    if not by:
+        by = Person.objects.get(name="(System)")
+    msg = Message.objects.create(
+            by = by,
+            subject = message.get('subject',''),
+            frm = message.get('from',''),
+            to = message.get('to',''),
+            cc = message.get('cc',''),
+            bcc = message.get('bcc',''),
+            reply_to = message.get('reply_to',''),
+            body = body,
+            time = utc_from_string(message.get('date', ''))
+    )
+    return msg
+
+def save_submission_email_attachments(submission_email_event, parts):
+    for part in parts:
+        if part.disposition != 'attachment':
+            continue
+
+        if part.type == 'text/plain':
+            payload, used_charset = pyzmail.decode_text(part.get_payload(), 
+                                                        part.charset, 
+                                                        None)
+            encoding = ""
+        else:
+            # Need a better approach - for the moment we'll just handle these 
+            # and encode as base64
+            payload = base64.b64encode(part.get_payload())
+            encoding = "base64"
+
+        #name = submission_email_event.submission.name
+
+        MessageAttachment.objects.create(message = submission_email_event.message,
+                                         content_type = part.type,
+                                         encoding = encoding,
+                                         filename=part.filename,
+                                         body=payload)

ietf/submit/management/commands/manualpost_email.py

@@ -0,0 +1,27 @@
+import sys
+from optparse import make_option
+
+from django.core.management.base import BaseCommand, CommandError
+
+from ietf.submit.mail import process_response_email
+
+import debug                            # pyflakes:ignore
+
+class Command(BaseCommand):
+    help = (u"Process incoming manual post email responses")
+    option_list = BaseCommand.option_list + (
+         make_option('--email-file', dest='email', help='File containing email (default: stdin)'),)
+
+    def handle(self, *args, **options):
+        email = options.get('email', None)
+        msg = None
+
+        if not email:
+            msg = sys.stdin.read()
+        else:
+            msg = open(email, "r").read()
+
+        try:
+            process_response_email(msg)
+        except ValueError as e:
+            raise CommandError(e)

ietf/submit/migrations/0011_submissionemail.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('message', '__first__'),
+        ('submit', '0010_data_set_submission_check_symbol'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SubmissionEmail',
+            fields=[
+                ('submissionevent_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='submit.SubmissionEvent')),
+                ('msgtype', models.CharField(max_length=25)),
+                ('in_reply_to', models.ForeignKey(related_name='irtomanual', blank=True, to='message.Message', null=True)),
+                ('message', models.ForeignKey(related_name='manualevents', blank=True, to='message.Message', null=True)),
+            ],
+            options={
+                'ordering': ['-time', '-id'],
+            },
+            bases=('submit.submissionevent',),
+        ),
+    ]

ietf/submit/migrations/0012_auto_20160414_1902.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    def add_draft_submission_state_name(apps, schema_editor):
+        # We can't import the model directly as it may be a newer
+        # version than this migration expects. We use the historical version.
+        DraftSubmissionStateName = apps.get_model("name", "DraftSubmissionStateName")
+        DraftSubmissionStateName.objects.create(slug="manual-awaiting-draft",
+                                                name="Manual Post Awaiting Draft",
+                                                desc="",
+                                                used=True,
+                                                order=8)
+
+
+    dependencies = [
+        ('submit', '0011_submissionemail'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_draft_submission_state_name),
+    ]

ietf/submit/migrations/0013_auto_20160415_2120.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from datetime import date
+
+from django.db import migrations
+from ietf.submit.utils import remove_submission_files
+
+
+class Migration(migrations.Migration):
+    def remove_old_submissions(apps, schema_editor):
+        """
+        We'll remove any submissions awaiting manual post that are older
+        than a date provided here.
+        
+        These all showed up when we added the ability to list submissions
+        awaiting manual post and go back many years
+        """
+
+        # We can't import the model directly as it may be a newer
+        # version than this migration expects. We use the historical version.
+        before=date(2016, 3, 1)
+        Submission = apps.get_model("submit", "Submission")
+        DraftSubmissionStateName = apps.get_model("name", "DraftSubmissionStateName")
+
+        cancelled = DraftSubmissionStateName.objects.get(slug="cancel")     
+        for submission in Submission.objects.filter(state_id = "manual", submission_date__lt=before).distinct():
+            submission.state = cancelled
+            submission.save()
+        
+            remove_submission_files(submission)
+
+    dependencies = [
+        ('submit', '0012_auto_20160414_1902'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_old_submissions),
+    ]

ietf/submit/migrations/0014_auto_20160627_1945.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    def add_next_states(apps, schema_editor):
+        DraftSubmissionStateName = apps.get_model("name", "DraftSubmissionStateName")
+
+        cancelled = DraftSubmissionStateName.objects.get(slug="cancel")
+        posted = DraftSubmissionStateName.objects.get(slug="posted")
+        mad = DraftSubmissionStateName.objects.get(slug="manual-awaiting-draft")
+        
+        mad.next_states.add(cancelled)
+        mad.next_states.add(posted)
+        mad.save()
+
+    dependencies = [
+        ('submit', '0013_auto_20160415_2120'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_next_states),
+    ]

ietf/submit/models.py

@@ -7,6 +7,7 @@ import jsonfield
 from ietf.doc.models import Document
 from ietf.person.models import Person
 from ietf.group.models import Group
+from ietf.message.models import Message
 from ietf.name.models import DraftSubmissionStateName
 from ietf.utils.accesstoken import generate_random_key, generate_access_token
 
@@ -106,3 +107,15 @@ class Preapproval(models.Model):
 
     def __unicode__(self):
         return self.name
+
+class SubmissionEmail(SubmissionEvent):
+    message     = models.ForeignKey(Message, null=True, blank=True,related_name='manualevents')
+    msgtype     = models.CharField(max_length=25)
+    in_reply_to = models.ForeignKey(Message, null=True, blank=True,related_name='irtomanual')
+
+    def __unicode__(self):
+        return u"%s %s by %s at %s" % (self.submission.name, self.desc, self.by.plain_name() if self.by else "(unknown)", self.time)
+
+    class Meta:
+        ordering = ['-time', '-id']
+

ietf/submit/resources.py

@@ -6,7 +6,8 @@ from tastypie.cache import SimpleCache
 
 from ietf import api
 
-from ietf.submit.models import Preapproval, SubmissionCheck, Submission, SubmissionEvent
+from ietf.submit.models import Preapproval, \
+    SubmissionCheck, Submission, SubmissionEmail, SubmissionEvent
 
 
 from ietf.person.resources import PersonResource
@@ -100,3 +101,31 @@ class SubmissionCheckResource(ModelResource):
         }
 api.submit.register(SubmissionCheckResource())
 
+
+
+from ietf.person.resources import PersonResource
+from ietf.message.resources import MessageResource
+class SubmissionEmailResource(ModelResource):
+    submission       = ToOneField(SubmissionResource, 'submission')
+    by               = ToOneField(PersonResource, 'by', null=True)
+    submissionevent_ptr = ToOneField(SubmissionEventResource, 'submissionevent_ptr')
+    message          = ToOneField(MessageResource, 'message', null=True)
+    in_reply_to      = ToOneField(MessageResource, 'in_reply_to', null=True)
+    class Meta:
+        queryset = SubmissionEmail.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'submissionemail'
+        filtering = { 
+            "id": ALL,
+            "time": ALL,
+            "desc": ALL,
+            "msgtype": ALL,
+            "submission": ALL_WITH_RELATIONS,
+            "by": ALL_WITH_RELATIONS,
+            "submissionevent_ptr": ALL_WITH_RELATIONS,
+            "message": ALL_WITH_RELATIONS,
+            "in_reply_to": ALL_WITH_RELATIONS,
+        }
+api.submit.register(SubmissionEmailResource())
+

ietf/submit/tests.py

@@ -1,3 +1,5 @@
+import email
+
 import datetime
 import os
 import shutil
@@ -9,17 +11,38 @@ from django.core.urlresolvers import reverse as urlreverse
 
 import debug                            # pyflakes:ignore
 
-from ietf.utils.test_utils import login_testing_unauthorized, unicontent
-from ietf.utils.test_data import make_test_data
-from ietf.utils.mail import outbox
-from ietf.utils.test_utils import TestCase
-from ietf.meeting.models import Meeting
 from ietf.submit.utils import expirable_submissions, expire_submission, ensure_person_email_info_exists
-from ietf.person.models import Person
-from ietf.group.models import Group
 from ietf.doc.models import Document, DocAlias, DocEvent, State, BallotDocEvent, BallotPositionDocEvent, DocumentAuthor
-from ietf.submit.models import Submission, Preapproval
+from ietf.group.models import Group
 from ietf.group.utils import setup_default_community_list_for_group
+from ietf.meeting.models import Meeting
+from ietf.message.models import Message
+from ietf.person.models import Person
+from ietf.submit.models import Submission, Preapproval
+from ietf.submit.mail import add_submission_email, process_response_email
+from ietf.utils.mail import outbox, empty_outbox
+from ietf.utils.test_data import make_test_data
+from ietf.utils.test_utils import login_testing_unauthorized, unicontent, TestCase
+
+
+def submission_file(name, rev, group, format, templatename):
+    # construct appropriate text draft
+    f = open(os.path.join(settings.BASE_DIR, "submit", templatename))
+    template = f.read()
+    f.close()
+
+    submission_text = template % dict(
+            date=datetime.date.today().strftime("%d %B %Y"),
+            expiration=(datetime.date.today() + datetime.timedelta(days=100)).strftime("%d %B, %Y"),
+            year=datetime.date.today().strftime("%Y"),
+            month=datetime.date.today().strftime("%B"),
+            name="%s-%s" % (name, rev),
+            group=group or "",
+    )
+
+    file = StringIO(str(submission_text))
+    file.name = "%s-%s.%s" % (name, rev, format)
+    return file
 
 class SubmitTests(TestCase):
     def setUp(self):
@@ -70,25 +93,6 @@ class SubmitTests(TestCase):
         settings.YANG_INVAL_MODEL_DIR = self.saved_yang_inval_model_dir
 
 
-    def submission_file(self, name, rev, group, format, templatename):
-        # construct appropriate text draft
-        f = open(os.path.join(settings.BASE_DIR, "submit", templatename))
-        template = f.read()
-        f.close()
-
-        submission_text = template % dict(
-            date=datetime.date.today().strftime("%d %B %Y"),
-            expiration=(datetime.date.today() + datetime.timedelta(days=100)).strftime("%d %B, %Y"),
-            year=datetime.date.today().strftime("%Y"),
-            month=datetime.date.today().strftime("%B"),
-            name="%s-%s" % (name, rev),
-            group=group or "",
-            )
-
-        file = StringIO(str(submission_text))
-        file.name = "%s-%s.%s" % (name, rev, format)
-        return file
-
     def do_submission(self, name, rev, group=None, formats=["txt",]):
         # break early in case of missing configuration
         self.assertTrue(os.path.exists(settings.IDSUBMIT_IDNITS_BINARY))
@@ -104,7 +108,7 @@ class SubmitTests(TestCase):
         # submit
         files = {}
         for format in formats:
-            files[format] = self.submission_file(name, rev, group, format, "test_submission.%s" % format)
+            files[format] = submission_file(name, rev, group, format, "test_submission.%s" % format)
 
         r = self.client.post(url, files)
         if r.status_code != 302:
@@ -334,6 +338,12 @@ class SubmitTests(TestCase):
         r = self.client.post(confirm_url)
         self.assertEqual(r.status_code, 302)
 
+        # check we have document events 
+        doc_events = draft.docevent_set.filter(type="added_comment")
+        edescs = '::'.join([x.desc for x in doc_events])
+        self.assertTrue('New version approved' in edescs)
+        self.assertTrue('Uploaded new revision' in edescs)
+
         draft = Document.objects.get(docalias__name=name)
         self.assertEqual(draft.rev, rev)
         self.assertEqual(draft.group.acronym, name.split("-")[2])
@@ -781,7 +791,7 @@ class SubmitTests(TestCase):
         # submit
         files = {}
         for format in formats:
-            files[format] = self.submission_file(name, rev, group, "bad", "test_submission.bad")
+            files[format] = submission_file(name, rev, group, "bad", "test_submission.bad")
 
         r = self.client.post(url, files)
 
@@ -893,3 +903,452 @@ class ApprovalsTestCase(TestCase):
         self.assertEqual(r.status_code, 302)
 
         self.assertEqual(len(Preapproval.objects.filter(name=preapproval.name)), 0)
+
+class ManualPostsTestCase(TestCase):
+    def test_manual_posts(self):
+        make_test_data()
+
+        url = urlreverse('submit_manualpost')
+        # Secretariat has access
+        self.client.login(username="secretary", password="secretary+password")
+
+        Submission.objects.create(name="draft-ietf-mars-foo",
+                                  group=Group.objects.get(acronym="mars"),
+                                  submission_date=datetime.date.today(),
+                                  state_id="manual")
+        Submission.objects.create(name="draft-ietf-mars-bar",
+                                  group=Group.objects.get(acronym="mars"),
+                                  submission_date=datetime.date.today(),
+                                  rev="00",
+                                  state_id="grp-appr")
+
+        # get
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+
+        self.assertEqual(len(q('.submissions a:contains("draft-ietf-mars-foo")')), 1)
+        self.assertEqual(len(q('.submissions a:contains("draft-ietf-mars-bar")')), 0)
+
+    def test_awaiting_draft(self):
+        message_string = """To: somebody@ietf.org
+From: joe@test.com
+Date: {}
+Subject: test submission via email
+
+Please submit my draft at http://test.com/mydraft.txt
+
+Thank you
+""".format(datetime.datetime.now().ctime())
+        message = email.message_from_string(message_string)
+        submission, submission_email_event =\
+            add_submission_email(request=None,
+                                 remote_ip ="192.168.0.1",
+                                 name = "draft-my-new-draft",
+                                 rev='00',
+                                 submission_pk=None,
+                                 message = message,
+                                 by = Person.objects.get(name="(System)"),
+                                 msgtype = "msgin")
+
+        url = urlreverse('submit_manualpost')
+        # Secretariat has access
+        self.client.login(username="secretary", password="secretary+password")
+
+        # get
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+
+        self.assertEqual(len(q('.awaiting-draft a:contains("draft-my-new-draft")')), 1)
+
+        # Same name should raise an error
+        with self.assertRaises(Exception):
+            add_submission_email(request=None,
+                                 remote_ip ="192.168.0.1",
+                                 name = "draft-my-new-draft",
+                                 rev='00',
+                                 submission_pk=None,
+                                 message = message,
+                                 by = Person.objects.get(name="(System)"),
+                                 msgtype = "msgin")
+
+        # Cancel this one
+        r = self.client.post(urlreverse("submit_cancel_awaiting_draft_by_hash"), {
+            "submission_id": submission.pk,
+            "access_token": submission.access_token(),
+        })
+        self.assertEqual(r.status_code, 302)
+        url = r["Location"]
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('.awaiting-draft a:contains("draft-my-new-draft")')), 0)
+
+        # Should now be able to add it again
+        submission, submission_email_event = \
+            add_submission_email(request=None,
+                                 remote_ip ="192.168.0.1",
+                                 name = "draft-my-new-draft",
+                                 rev='00',
+                                 submission_pk=None,
+                                 message = message,
+                                 by = Person.objects.get(name="(System)"),
+                                 msgtype = "msgin")
+
+
+    def test_awaiting_draft_with_attachment(self):
+        frm = "joe@test.com"
+        
+        message_string = """To: somebody@ietf.org
+From: {}
+Date: {}
+Subject: A very important message with a small attachment
+Content-Type: multipart/mixed; boundary="------------090908050800030909090207"
+
+This is a multi-part message in MIME format.
+--------------090908050800030909090207
+Content-Type: text/plain; charset=utf-8; format=flowed
+Content-Transfer-Encoding: 7bit
+
+The message body will probably say something about the attached document
+
+--------------090908050800030909090207
+Content-Type: text/plain; charset=UTF-8; name="attach.txt"
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="attach.txt"
+
+QW4gZXhhbXBsZSBhdHRhY2htZW50IHd0aG91dCB2ZXJ5IG11Y2ggaW4gaXQuCgpBIGNvdXBs
+ZSBvZiBsaW5lcyAtIGJ1dCBpdCBjb3VsZCBiZSBhIGRyYWZ0Cg==
+--------------090908050800030909090207--
+""".format(frm, datetime.datetime.now().ctime())
+        message = email.message_from_string(message_string)
+        submission, submission_email_event = \
+            add_submission_email(request=None,
+                                 remote_ip ="192.168.0.1",
+                                 name = "draft-my-new-draft",
+                                 rev='00',
+                                 submission_pk=None,
+                                 message = message,
+                                 by = Person.objects.get(name="(System)"),
+                                 msgtype = "msgin")
+
+        manualpost_page_url = urlreverse('submit_manualpost')
+        # Secretariat has access
+        self.client.login(username="secretary", password="secretary+password")
+
+        self.check_manualpost_page(submission=submission, 
+                                   submission_email_event=submission_email_event,
+                                   the_url=manualpost_page_url, 
+                                   submission_name_fragment='draft-my-new-draft',
+                                   frm=frm,
+                                   is_secretariat=True)
+ 
+        # Try the status page with no credentials
+        self.client.logout()
+
+        self.check_manualpost_page(submission=submission, 
+                                   submission_email_event=submission_email_event,
+                                   the_url=manualpost_page_url, 
+                                   submission_name_fragment='draft-my-new-draft',
+                                   frm=frm,
+                                   is_secretariat=False)
+        
+        # Post another message to this submission using the link
+        message_string = """To: somebody@ietf.org
+From: joe@test.com
+Date: {}
+Subject: A new submission message with a small attachment
+Content-Type: multipart/mixed; boundary="------------090908050800030909090207"
+
+This is a multi-part message in MIME format.
+--------------090908050800030909090207
+Content-Type: text/plain; charset=utf-8; format=flowed
+Content-Transfer-Encoding: 7bit
+
+The message body will probably say something more about the attached document
+
+--------------090908050800030909090207
+Content-Type: text/plain; charset=UTF-8; name="attach.txt"
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="attachment.txt"
+
+QW4gZXhhbXBsZSBhdHRhY2htZW50IHd0aG91dCB2ZXJ5IG11Y2ggaW4gaXQuCgpBIGNvdXBs
+ZSBvZiBsaW5lcyAtIGJ1dCBpdCBjb3VsZCBiZSBhIGRyYWZ0Cg==
+--------------090908050800030909090207--
+""".format(datetime.datetime.now().ctime())
+
+        # Back to secretariat
+        self.client.login(username="secretary", password="secretary+password")
+
+        r, q = self.request_and_parse(manualpost_page_url)
+
+        url = self.get_href(q, "a#new-submission-email:contains('New submission from email')")
+
+        # Get the form
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        #self.assertEqual(len(q('input[name=edit-title]')), 1)
+
+        # Post the new message
+        r = self.client.post(url, {
+            "name": "draft-my-next-new-draft-00",
+            "direction": "incoming",
+            "message": message_string,
+        })
+
+        if r.status_code != 302:
+            q = PyQuery(r.content)
+            print q
+
+        self.assertEqual(r.status_code, 302)
+        
+
+        #self.check_manualpost_page(submission, submission_email_event,
+        #                        url, 'draft-my-next-new-draft'
+        #                        'Another very important message',
+        #                        true)
+
+    def check_manualpost_page(self, submission, submission_email_event,
+                              the_url, submission_name_fragment,
+                              frm,
+                              is_secretariat):
+        # get the page listing manual posts
+        r, q = self.request_and_parse(the_url)
+        selector = "#awaiting-draft a#add-submission-email{}:contains('Add email')". \
+            format(submission.pk, submission_name_fragment)
+
+        if is_secretariat:
+            # Can add an email to the submission
+            add_email_url = self.get_href(q, selector)
+        else:
+            # No add email button button
+            self.assertEqual(len(q(selector)), 0)
+
+        # Find the link for our submission in those awaiting drafts
+        submission_url = self.get_href(q, "#awaiting-draft a#aw{}:contains({})".
+                                       format(submission.pk, submission_name_fragment))
+
+        # Follow the link to the status page for this submission
+        r, q = self.request_and_parse(submission_url)
+        
+        selector = "#history a#reply{}:contains('Reply')".\
+            format(submission.pk)
+
+        if is_secretariat:
+            # check that reply button is visible and get the form
+            reply_url = self.get_href(q, selector)
+
+            # Get the form
+            r = self.client.get(reply_url)
+            self.assertEqual(r.status_code, 200)
+            reply_q = PyQuery(r.content)
+            self.assertEqual(len(reply_q('input[name=to]')), 1)
+        else:
+            # No reply button
+            self.assertEqual(len(q(selector)), 0)
+
+        if is_secretariat:
+            # Now try to send an email using the send email link
+    
+            selector = "a#send{}:contains('Send Email')". \
+                format(submission.pk)
+            send_url = self.get_href(q, selector)
+
+            self.do_submission_email(the_url = send_url,
+                                     to = frm,
+                                     body = "A new message")
+
+        # print q
+        # print submission.pk
+        # print submission_email_event.pk
+        
+        # Find the link for our message in the list
+        url = self.get_href(q, "#aw{}-{}:contains('{}')".format(submission.pk, 
+                                                                submission_email_event.message.pk,
+                                                                "Received message - manual post"))
+        
+        # Page displaying message details
+        r, q = self.request_and_parse(url)
+        
+        if is_secretariat:
+            # check that reply button is visible
+
+            reply_href = self.get_href(q, "#email-details a#reply{}:contains('Reply')". \
+                                       format(submission.pk))
+
+        else:
+            # No reply button
+            self.assertEqual(len(q(selector)), 0)
+            reply_href = None
+
+        # check that attachment link is visible
+
+        url = self.get_href(q, "#email-details a#attach{}:contains('attach.txt')".format(submission.pk))
+
+        # Fetch the attachment
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        
+        # Attempt a reply if we can
+        if reply_href == None:
+            return
+
+        self.do_submission_email(the_url = reply_href,
+                                 to = frm,
+                                 body = "A reply to the message")
+        
+        # try adding an email to the submission
+        # Use the add email link from the manual post listing page
+
+        if is_secretariat:
+            # Can add an email to the submission
+            # add_email_url set previously
+            r = self.client.get(add_email_url)
+            self.assertEqual(r.status_code, 200)
+            add_email_q = PyQuery(r.content)
+            self.assertEqual(len(add_email_q('input[name=submission_pk]')), 1)
+
+            # Add a simple email
+            new_message_string = """To: somebody@ietf.org
+From: joe@test.com
+Date: {}
+Subject: Another message
+
+About my submission
+
+Thank you
+""".format(datetime.datetime.now().ctime())
+
+            r = self.client.post(add_email_url, {
+                "name": "{}-{}".format(submission.name, submission.rev),
+                "direction": "incoming",
+                "submission_pk": submission.pk,
+                "message": new_message_string,
+            })
+
+            if r.status_code != 302:
+                q = PyQuery(r.content)
+                print q
+
+            self.assertEqual(r.status_code, 302)
+
+    def request_and_parse(self, url):
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        return r, PyQuery(r.content)
+
+        
+    def get_href(self, q, query):
+        link = q(query)
+        self.assertEqual(len(link), 1)
+
+        return PyQuery(link[0]).attr('href')
+
+
+    def do_submission_email(self, the_url, to, body):
+        # check the page
+        r = self.client.get(the_url)
+        q = PyQuery(r.content)
+        post_button = q('[type=submit]:contains("Send Email")')
+        self.assertEqual(len(post_button), 1)
+        action = post_button.parents("form").find('input[type=hidden][name="action"]').val()
+        subject = post_button.parents("form").find('input[name="subject"]').val()
+        frm = post_button.parents("form").find('input[name="frm"]').val()
+        cc = post_button.parents("form").find('input[name="cc"]').val()
+        reply_to = post_button.parents("form").find('input[name="reply_to"]').val()
+
+        empty_outbox()
+        
+        # post submitter info
+        r = self.client.post(the_url, {
+            "action": action,
+            "subject": subject,
+            "frm": frm,
+            "to": to,
+            "cc": cc,
+            "reply_to": reply_to,
+            "body": body,
+        })
+
+        self.assertEqual(r.status_code, 302)
+
+        self.assertEqual(len(outbox), 1)
+
+        outmsg = outbox[0]
+        self.assertTrue(to in outmsg['To'])
+        
+        reply_to = outmsg['Reply-to']
+        self.assertIsNotNone(reply_to, "Expected Reply-to")
+        
+        # Build a reply
+
+        message_string = """To: {}
+From: {}
+Date: {}
+Subject: test
+""".format(reply_to, to, datetime.datetime.now().ctime())
+        result = process_response_email(message_string)
+        self.assertIsInstance(result, Message)
+
+        return r
+
+    def do_submission(self, name, rev, group=None, formats=["txt",]):
+        # We're not testing the submission process - just the submission status 
+
+        # get
+        url = urlreverse('submit_upload_submission')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('input[type=file][name=txt]')), 1)
+        self.assertEqual(len(q('input[type=file][name=xml]')), 1)
+
+        # submit
+        files = {}
+        for format in formats:
+            files[format] = submission_file(name, rev, group, format, "test_submission.%s" % format)
+
+        r = self.client.post(url, files)
+        if r.status_code != 302:
+            q = PyQuery(r.content)
+            print(q('div.has-error span.help-block div').text)
+
+        self.assertEqual(r.status_code, 302)
+
+        status_url = r["Location"]
+        for format in formats:
+            self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.%s" % (name, rev, format))))
+        self.assertEqual(Submission.objects.filter(name=name).count(), 1)
+        submission = Submission.objects.get(name=name)
+        self.assertTrue(all([ c.passed!=False for c in submission.checks.all() ]))
+        self.assertEqual(len(submission.authors_parsed()), 1)
+        author = submission.authors_parsed()[0]
+        self.assertEqual(author["name"], "Author Name")
+        self.assertEqual(author["email"], "author@example.com")
+
+        return status_url
+
+
+    def supply_extra_metadata(self, name, status_url, submitter_name, submitter_email):
+        # check the page
+        r = self.client.get(status_url)
+        q = PyQuery(r.content)
+        post_button = q('[type=submit]:contains("Post")')
+        self.assertEqual(len(post_button), 1)
+        action = post_button.parents("form").find('input[type=hidden][name="action"]').val()
+
+        # post submitter info
+        r = self.client.post(status_url, {
+            "action": action,
+            "submitter-name": submitter_name,
+            "submitter-email": submitter_email,
+            "approvals_received": True,
+        })
+
+        if r.status_code == 302:
+            submission = Submission.objects.get(name=name)
+            self.assertEqual(submission.submitter, u"%s <%s>" % (submitter_name, submitter_email))
+
+        return r

ietf/submit/urls.py

@@ -15,4 +15,14 @@ urlpatterns = patterns('ietf.submit.views',
     url(r'^approvals/$', 'approvals', name='submit_approvals'),
     url(r'^approvals/addpreapproval/$', 'add_preapproval', name='submit_add_preapproval'),
     url(r'^approvals/cancelpreapproval/(?P<preapproval_id>[a-f\d]+)/$', 'cancel_preapproval', name='submit_cancel_preapproval'),
+
+    url(r'^manualpost/addemail$', 'add_manualpost_email', name='submit_manualpost_email'),
+    url(r'^manualpost/addemail/(?P<submission_id>\d+)/(?P<access_token>[a-f\d]*)/$', 'add_manualpost_email', name='submit_manualpost_email_by_hash'),
+    url(r'^awaitingdraft/cancel$', 'cancel_awaiting_draft', name='submit_cancel_awaiting_draft_by_hash'),
+    url(r'^manualpost/$', 'manualpost', name='submit_manualpost'),
+    url(r'^manualpost/email/(?P<submission_id>\d+)/(?P<message_id>\d+)/$', 'submission_email', name='submit_submission_email'),
+    url(r'^manualpost/email/(?P<submission_id>\d+)/(?P<message_id>\d+)/(?P<access_token>[a-f\d]*)/$', 'submission_email', name='submit_submission_email_by_hash'),
+    url(r'^manualpost/sendemail/(?P<submission_id>\d+)/$', 'send_email', name='submission_send_email'),
+    url(r'^manualpost/replyemail/(?P<submission_id>\d+)/(?P<message_id>\d+)/$', 'send_email', name='submission_reply_email'),
+    url(r'^manualpost/attachment/(?P<submission_id>\d+)/(?P<message_id>\d+)/(?P<filename>.*)$', 'submission_email_attachment', name='submit_submission_email_attachment'),
 )

ietf/submit/utils.py

@@ -3,7 +3,8 @@ import datetime
 
 from django.conf import settings
 
-from ietf.doc.models import Document, State, DocAlias, DocEvent, DocumentAuthor
+from ietf.doc.models import Document, State, DocAlias, DocEvent, \
+    DocumentAuthor, AddedMessageEvent
 from ietf.doc.models import NewRevisionDocEvent
 from ietf.doc.models import RelatedDocument, DocRelationshipName
 from ietf.doc.utils import add_state_change_event, rebuild_reference_relations
@@ -19,6 +20,7 @@ from ietf.submit.models import Submission, SubmissionEvent, Preapproval, DraftSu
 from ietf.utils import unaccent
 from ietf.utils.log import log
 
+
 def validate_submission(submission):
     errors = {}
 
@@ -108,8 +110,62 @@ def create_submission_event(request, submission, desc):
     SubmissionEvent.objects.create(submission=submission, by=by, desc=desc)
 
 
-def post_submission(request, submission):
-    # find out who did it
+def docevent_from_submission(request, submission, desc):
+    system = Person.objects.get(name="(System)")
+
+    try:
+        draft = Document.objects.get(name=submission.name)
+    except Document.DoesNotExist:
+        # Assume this is revision 00 - we'll do this later
+        return
+
+    submitter_parsed = submission.submitter_parsed()
+    if submitter_parsed["name"] and submitter_parsed["email"]:
+        submitter = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]).person
+    else:
+        submitter = system
+
+    e = DocEvent(doc=draft)
+    e.by = submitter
+    e.type = "added_comment"
+    e.desc = desc
+    e.save()
+
+
+def post_rev00_submission_events(draft, submission, submitter):
+    # Add previous submission events as docevents
+    # For now we'll filter based on the description
+    for subevent in submission.submissionevent_set.all():
+        desc = subevent.desc
+        if desc.startswith("Uploaded submission"):
+            desc = "Uploaded new revision"
+            e = DocEvent(type="added_comment", doc=draft)
+        elif desc.startswith("Submission created"):
+            e = DocEvent(type="added_comment", doc=draft)
+        elif desc.startswith("Set submitter to"):
+            pos = subevent.desc.find("sent confirmation email")
+            e = DocEvent(type="added_comment", doc=draft)
+            if pos > 0:
+                desc = "Request for posting confirmation emailed %s" % (subevent.desc[pos + 23:])
+            else:
+                pos = subevent.desc.find("sent appproval email")
+                if pos > 0:
+                    desc = "Request for posting approval emailed %s" % (subevent.desc[pos + 19:])
+        elif desc.startswith("Received message") or desc.startswith("Sent message"):
+            e = AddedMessageEvent(type="added_message", doc=draft)
+            e.message = subevent.submissionemail.message
+            e.msgtype = subevent.submissionemail.msgtype
+            e.in_reply_to = subevent.submissionemail.in_reply_to
+        else:
+            continue
+
+        e.time = subevent.time #submission.submission_date
+        e.by = submitter
+        e.desc = desc
+        e.save()
+
+
+def post_submission(request, submission, approvedDesc):
     system = Person.objects.get(name="(System)")
     submitter_parsed = submission.submitter_parsed()
     if submitter_parsed["name"] and submitter_parsed["email"]:
@@ -175,8 +231,29 @@ def post_submission(request, submission):
     trouble = rebuild_reference_relations(draft, filename=os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (submission.name, submission.rev)))
     if trouble:
         log('Rebuild_reference_relations trouble: %s'%trouble)
+    
+    if draft.rev == '00':
+        # Add all the previous submission events as docevents
+        post_rev00_submission_events(draft, submission, submitter)
+
+    # Add an approval docevent
+    e = DocEvent(type="added_comment", doc=draft)
+    e.time = draft.time #submission.submission_date
+    e.by = submitter
+    e.desc = approvedDesc
+    e.save()
+    
+    # new revision event
+    e = NewRevisionDocEvent(type="new_revision", doc=draft, rev=draft.rev)
+    e.time = draft.time #submission.submission_date
+    e.by = submitter
+    e.desc = "New version available: <b>%s-%s.txt</b>" % (draft.name, draft.rev)
+    e.save()
+
+    if draft.stream_id == "ietf" and draft.group.type_id == "wg" and draft.rev == "00":
+        # automatically set state "WG Document"
+        draft.set_state(State.objects.get(used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-doc"))
 
-    # automatic state changes
     if draft.get_state_slug("draft-iana-review") in ("ok-act", "ok-noact", "not-ok"):
         prev_state = draft.get_state("draft-iana-review")
         next_state = State.objects.get(used=True, type="draft-iana-review", slug="changed")
@@ -435,4 +512,4 @@ def expire_submission(submission, by):
     submission.state_id = "cancel"
     submission.save()
 
-    SubmissionEvent.objects.create(submission=submission, by=by, desc="Canceled expired submission")
+    SubmissionEvent.objects.create(submission=submission, by=by, desc="Cancelled expired submission")

ietf/submit/views.py

@@ -1,31 +1,41 @@
 # Copyright The IETF Trust 2007, All Rights Reserved
+import base64
 import datetime
 import os
 import xml2rfc
 
 from django.conf import settings
+from django.contrib import messages
 from django.core.urlresolvers import reverse as urlreverse
 from django.core.validators import validate_email, ValidationError
-from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden
+from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden, \
+    HttpResponse
 from django.shortcuts import get_object_or_404, redirect, render
 from django.utils.module_loading import import_string
 
 import debug                            # pyflakes:ignore
 
-from ietf.doc.models import Document, DocAlias
+from ietf.doc.models import Document, DocAlias, AddedMessageEvent
 from ietf.doc.utils import prettify_std_name
 from ietf.group.models import Group
 from ietf.ietfauth.utils import has_role, role_required
-from ietf.submit.forms import SubmissionUploadForm, NameEmailForm, EditSubmissionForm, PreapprovalForm, ReplacesForm
-from ietf.submit.mail import send_full_url, send_approval_request_to_group, send_submission_confirmation, send_manual_post_request
-from ietf.submit.models import Submission, SubmissionCheck, Preapproval, DraftSubmissionStateName
+from ietf.mailtrigger.utils import gather_address_lists
+from ietf.message.models import Message, MessageAttachment
+from ietf.submit.forms import SubmissionUploadForm, NameEmailForm, EditSubmissionForm, PreapprovalForm, ReplacesForm, \
+    SubmissionEmailForm, MessageModelForm
+from ietf.submit.mail import send_full_url, send_approval_request_to_group, \
+    send_submission_confirmation, send_manual_post_request, \
+    add_submission_email, get_reply_to
+from ietf.submit.models import Submission, SubmissionCheck, Preapproval, DraftSubmissionStateName, \
+    SubmissionEmail
 from ietf.submit.utils import approvable_submissions_for_user, preapprovals_for_user, recently_approved_by_user
 from ietf.submit.utils import validate_submission, create_submission_event
+from ietf.submit.utils import docevent_from_submission
 from ietf.submit.utils import post_submission, cancel_submission, rename_submission_files
 from ietf.utils.accesstoken import generate_random_key, generate_access_token
 from ietf.utils.draft import Draft
 from ietf.utils.log import log
-from ietf.mailtrigger.utils import gather_address_lists
+from ietf.utils.mail import send_mail_message
 
 
 def upload_submission(request):
@@ -93,29 +103,59 @@ def upload_submission(request):
                 else:
                     abstract = form.parsed_draft.get_abstract()
 
-                # save submission
-                try:
-                    submission = Submission.objects.create(
-                        state=DraftSubmissionStateName.objects.get(slug="uploaded"),
-                        remote_ip=form.remote_ip,
-                        name=form.filename,
-                        group=form.group,
-                        title=form.title,
-                        abstract=abstract,
-                        rev=form.revision,
-                        pages=form.parsed_draft.get_pagecount(),
-                        authors="\n".join(authors),
-                        note="",
-                        first_two_pages=''.join(form.parsed_draft.pages[:2]),
-                        file_size=file_size,
-                        file_types=','.join(form.file_types),
-                        submission_date=datetime.date.today(),
-                        document_date=form.parsed_draft.get_creation_date(),
-                        replaces="",
+                # See if there is a Submission in state manual-awaiting-upload
+                # for this revision.
+                # If so - we're going to update it otherwise we create a new object 
+
+                submission = Submission.objects.filter(name=form.filename, 
+                                                       rev=form.revision,
+                                                       state_id = "manual-awaiting-draft").distinct()
+                if (len(submission) == 0):
+                    submission = None
+                elif (len(submission) == 1):
+                    submission = submission[0]
+                    
+                    submission.state = DraftSubmissionStateName.objects.get(slug="uploaded")
+                    submission.remote_ip=form.remote_ip
+                    submission.title=form.title
+                    submission.abstract=abstract
+                    submission.rev=form.revision
+                    submission.pages=form.parsed_draft.get_pagecount()
+                    submission.authors="\n".join(authors)
+                    submission.first_two_pages=''.join(form.parsed_draft.pages[:2])
+                    submission.file_size=file_size
+                    submission.file_types=','.join(form.file_types)
+                    submission.submission_date=datetime.date.today()
+                    submission.document_date=form.parsed_draft.get_creation_date()
+                    submission.replaces=""
+                    
+                    submission.save()
+                else:
+                    raise Exception("Multiple submissions awaiting upload")
+
+                if (submission == None):
+                    try:
+                        submission = Submission.objects.create(
+                                state=DraftSubmissionStateName.objects.get(slug="uploaded"),
+                                remote_ip=form.remote_ip,
+                                name=form.filename,
+                                group=form.group,
+                                title=form.title,
+                                abstract=abstract,
+                                rev=form.revision,
+                                pages=form.parsed_draft.get_pagecount(),
+                                authors="\n".join(authors),
+                                note="",
+                                first_two_pages=''.join(form.parsed_draft.pages[:2]),
+                                file_size=file_size,
+                                file_types=','.join(form.file_types),
+                                submission_date=datetime.date.today(),
+                                document_date=form.parsed_draft.get_creation_date(),
+                                replaces="",
                         )
-                except Exception as e:
-                    log("Exception: %s\n" % e)
-                    raise
+                    except Exception as e:
+                        log("Exception: %s\n" % e)
+                        raise
 
                 # run submission checkers
                 def apply_check(submission, checker, method, fn):
@@ -135,6 +175,7 @@ def upload_submission(request):
                             break
 
                 create_submission_event(request, submission, desc="Uploaded submission")
+                docevent_from_submission(request, submission, desc="Uploaded new revision")
 
                 return redirect("submit_submission_status_by_hash", submission_id=submission.pk, access_token=submission.access_token())
         except IOError as e:
@@ -197,7 +238,7 @@ def submission_status(request, submission_id, access_token=None):
     can_edit = can_edit_submission(request.user, submission, access_token) and submission.state_id == "uploaded"
     can_cancel = (key_matched or is_secretariat) and submission.state.next_states.filter(slug="cancel")
     can_group_approve = (is_secretariat or is_chair) and submission.state_id == "grp-appr"
-    can_force_post = is_secretariat and submission.state.next_states.filter(slug="posted")
+    can_force_post = is_secretariat and submission.state.next_states.filter(slug="posted") and submission.state_id != "manual-awaiting-draft"
     show_send_full_url = not key_matched and not is_secretariat and submission.state_id not in ("cancel", "posted")
 
     addrs = gather_address_lists('sub_confirmation_requested',submission=submission)
@@ -211,7 +252,7 @@ def submission_status(request, submission_id, access_token=None):
     message = None
 
     if submission.state_id == "cancel":
-        message = ('error', 'This submission has been canceled, modification is no longer possible.')
+        message = ('error', 'This submission has been cancelled, modification is no longer possible.')
     elif submission.state_id == "auth":
         message = ('success', u'The submission is pending email authentication. An email has been sent to: %s' % ", ".join(confirmation_list))
     elif submission.state_id == "grp-appr":
@@ -237,35 +278,50 @@ def submission_status(request, submission_id, access_token=None):
                 replaces = replaces_form.cleaned_data.get("replaces", [])
                 submission.replaces = ",".join(o.name for o in replaces)
 
-                if requires_group_approval:
-                    submission.state = DraftSubmissionStateName.objects.get(slug="grp-appr")
-                    submission.save()
-
-                    sent_to = send_approval_request_to_group(request, submission)
-
-                    desc = "sent approval email to group chairs: %s" % u", ".join(sent_to)
+                approvals_received = submitter_form.cleaned_data['approvals_received']
+                
+                if approvals_received:
+                    if not is_secretariat:
+                        return HttpResponseForbidden('You do not have permission to perform this action')
 
+                    # go directly to posting submission
+                    desc = u"Secretariat manually posting. Approvals already received"
+                    post_submission(request, submission, desc)
+                    create_submission_event(request, submission, desc)
                 else:
-                    submission.auth_key = generate_random_key()
-                    if requires_prev_authors_approval:
-                        submission.state = DraftSubmissionStateName.objects.get(slug="aut-appr")
+                    if requires_group_approval:
+                        submission.state = DraftSubmissionStateName.objects.get(slug="grp-appr")
+                        submission.save()
+    
+                        sent_to = send_approval_request_to_group(request, submission)
+    
+                        desc = "sent approval email to group chairs: %s" % u", ".join(sent_to)
+                        docDesc = u"Request for posting approval emailed to group chairs: %s" % u", ".join(sent_to)
+
                     else:
-                        submission.state = DraftSubmissionStateName.objects.get(slug="auth")
-                    submission.save()
-
-                    sent_to = send_submission_confirmation(request, submission)
-
-                    if submission.state_id == "aut-appr":
-                        desc = u"sent confirmation email to previous authors: %s" % u", ".join(sent_to)
-                    else:
-                        desc = u"sent confirmation email to submitter and authors: %s" % u", ".join(sent_to)
-
-                msg = u"Set submitter to \"%s\", replaces to %s and %s" % (
-                    submission.submitter,
-                    ", ".join(prettify_std_name(r.name) for r in replaces) if replaces else "(none)",
-                    desc)
-                create_submission_event(request, submission, msg)
-
+                        submission.auth_key = generate_random_key()
+                        if requires_prev_authors_approval:
+                            submission.state = DraftSubmissionStateName.objects.get(slug="aut-appr")
+                        else:
+                            submission.state = DraftSubmissionStateName.objects.get(slug="auth")
+                        submission.save()
+    
+                        sent_to = send_submission_confirmation(request, submission)
+    
+                        if submission.state_id == "aut-appr":
+                            desc = u"sent confirmation email to previous authors: %s" % u", ".join(sent_to)
+                            docDesc = "Request for posting confirmation emailed to previous authors: %s" % u", ".join(sent_to)
+                        else:
+                            desc = u"sent confirmation email to submitter and authors: %s" % u", ".join(sent_to)
+                            docDesc = "Request for posting confirmation emailed to submitter and authors: %s" % u", ".join(sent_to)
+    
+                    msg = u"Set submitter to \"%s\", replaces to %s and %s" % (
+                        submission.submitter,
+                        ", ".join(prettify_std_name(r.name) for r in replaces) if replaces else "(none)",
+                        desc)
+                    create_submission_event(request, submission, msg)
+                    docevent_from_submission(request, submission, docDesc)
+    
                 if access_token:
                     return redirect("submit_submission_status_by_hash", submission_id=submission.pk, access_token=access_token)
                 else:
@@ -290,7 +346,7 @@ def submission_status(request, submission_id, access_token=None):
 
             cancel_submission(submission)
 
-            create_submission_event(request, submission, "Canceled submission")
+            create_submission_event(request, submission, "Cancelled submission")
 
             return redirect("submit_submission_status", submission_id=submission_id)
 
@@ -299,7 +355,7 @@ def submission_status(request, submission_id, access_token=None):
             if not can_group_approve:
                 return HttpResponseForbidden('You do not have permission to perform this action')
 
-            post_submission(request, submission)
+            post_submission(request, submission, "WG -00 approved")
 
             create_submission_event(request, submission, "Approved and posted submission")
 
@@ -310,13 +366,13 @@ def submission_status(request, submission_id, access_token=None):
             if not can_force_post:
                 return HttpResponseForbidden('You do not have permission to perform this action')
 
-            post_submission(request, submission)
-
             if submission.state_id == "manual":
                 desc = "Posted submission manually"
             else:
                 desc = "Forced post of submission"
 
+            post_submission(request, submission, desc)
+
             create_submission_event(request, submission, desc)
 
             return redirect("doc_view", name=submission.name)
@@ -432,7 +488,16 @@ def confirm_submission(request, submission_id, auth_token):
     if not key_matched: key_matched = auth_token == submission.auth_key # backwards-compat
 
     if request.method == 'POST' and submission.state_id in ("auth", "aut-appr") and key_matched:
-        post_submission(request, submission)
+        submitter_parsed = submission.submitter_parsed()
+        if submitter_parsed["name"] and submitter_parsed["email"]:
+            # We know who approved it
+            desc = "New version approved"
+        elif submission.state_id == "auth":
+            desc = "New version approved by author"
+        else:
+            desc = "New version approved by previous author"
+
+        post_submission(request, submission, desc)
 
         create_submission_event(request, submission, "Confirmed and posted submission")
 
@@ -499,3 +564,266 @@ def cancel_preapproval(request, preapproval_id):
     return render(request, 'submit/cancel_preapproval.html',
                               {'selected': 'approvals',
                                'preapproval': preapproval })
+
+
+def manualpost(request):
+    '''
+    Main view for manual post requests
+    '''
+
+    manual = Submission.objects.filter(state_id = "manual").distinct()
+    
+    for s in manual:
+        s.passes_checks = all([ c.passed!=False for c in s.checks.all() ])
+        s.errors = validate_submission(s)
+
+    awaiting_draft = Submission.objects.filter(state_id = "manual-awaiting-draft").distinct()
+
+    return render(request, 'submit/manual_post.html',
+                  {'manual': manual,
+                   'selected': 'manual_posts',
+                   'awaiting_draft': awaiting_draft})
+
+
+def cancel_awaiting_draft(request):
+    if request.method == 'POST':
+        can_cancel = has_role(request.user, "Secretariat")
+        
+        if not can_cancel:
+            return HttpResponseForbidden('You do not have permission to perform this action')
+
+        submission_id = request.POST.get('submission_id', '')
+        access_token = request.POST.get('access_token', '')
+
+        submission = get_submission_or_404(submission_id, access_token = access_token)
+        cancel_submission(submission)
+    
+        create_submission_event(request, submission, "Cancelled submission")
+        if (submission.rev != "00"):
+            # Add a doc event
+            docevent_from_submission(request, 
+                                     submission,
+                                     "Cancelled submission for rev {}".format(submission.rev))
+    
+    return redirect("submit_manualpost")
+
+
+@role_required('Secretariat',)
+def add_manualpost_email(request, submission_id=None, access_token=None):
+    """Add email to submission history"""
+
+    if request.method == 'POST':
+        try:
+            button_text = request.POST.get('submit', '')
+            if button_text == 'Cancel':
+                return redirect("submit/manual_post.html")
+    
+            form = SubmissionEmailForm(request.POST)
+            if form.is_valid():
+                submission_pk = form.cleaned_data['submission_pk']
+                message = form.cleaned_data['message']
+                #in_reply_to = form.cleaned_data['in_reply_to']
+                # create Message
+    
+                if form.cleaned_data['direction'] == 'incoming':
+                    msgtype = 'msgin'
+                else:
+                    msgtype = 'msgout'
+    
+                submission, submission_email_event = \
+                    add_submission_email(request=request,
+                                         remote_ip=request.META.get('REMOTE_ADDR', None),
+                                         name = form.draft_name,
+                                         rev=form.revision,
+                                         submission_pk = submission_pk,
+                                         message = message,
+                                         by = request.user.person,
+                                         msgtype = msgtype)
+    
+                messages.success(request, 'Email added.')
+    
+                try:
+                    draft = Document.objects.get(name=submission.name)
+                except Document.DoesNotExist:
+                    # Assume this is revision 00 - we'll do this later
+                    draft = None
+        
+                if (draft != None):
+                    e = AddedMessageEvent(type="added_message", doc=draft)
+                    e.message = submission_email_event.submissionemail.message
+                    e.msgtype = submission_email_event.submissionemail.msgtype
+                    e.in_reply_to = submission_email_event.submissionemail.in_reply_to
+                    e.by = request.user.person
+                    e.desc = submission_email_event.desc
+                    e.time = submission_email_event.time
+                    e.save()
+    
+                return redirect("submit_manualpost")
+        except ValidationError as e:
+            form = SubmissionEmailForm(request.POST)
+            form._errors = {}
+            form._errors["__all__"] = form.error_class(["There was a failure uploading your message. (%s)" % e.message])
+    else:
+        initial = {
+        }
+
+        if (submission_id != None):
+            submission = get_submission_or_404(submission_id, access_token)
+            initial['name'] = "{}-{}".format(submission.name, submission.rev)
+            initial['direction'] = 'incoming'
+            initial['submission_pk'] = submission.pk
+        else:
+            initial['direction'] = 'incoming'
+            
+        form = SubmissionEmailForm(initial=initial)
+
+    return render(request, 'submit/add_submit_email.html',dict(form=form))
+
+
+@role_required('Secretariat',)
+def send_email(request, submission_id, message_id=None):
+    """Send an email related to a submission"""
+    submission = get_submission_or_404(submission_id, access_token = None)
+
+    if request.method == 'POST':
+        button_text = request.POST.get('submit', '')
+        if button_text == 'Cancel':
+            return redirect('submit_submission_status_by_hash',
+                            submission_id=submission.id,
+                            access_token=submission.access_token())
+
+        form = MessageModelForm(request.POST)
+        if form.is_valid():
+            # create Message
+            msg = Message.objects.create(
+                    by = request.user.person,
+                    subject = form.cleaned_data['subject'],
+                    frm = form.cleaned_data['frm'],
+                    to = form.cleaned_data['to'],
+                    cc = form.cleaned_data['cc'],
+                    bcc = form.cleaned_data['bcc'],
+                    reply_to = form.cleaned_data['reply_to'],
+                    body = form.cleaned_data['body']
+            )
+            
+            in_reply_to_id = form.cleaned_data['in_reply_to_id']
+            in_reply_to = None
+            rp = ""
+            
+            if in_reply_to_id:
+                rp = " reply"
+                try:
+                    in_reply_to = Message.objects.get(id=in_reply_to_id)
+                except Message.DoesNotExist:
+                    log("Unable to retrieve in_reply_to message: %s" % in_reply_to_id)
+    
+            desc = "Sent message {} - manual post - {}-{}".format(rp,
+                                                                  submission.name, 
+                                                                  submission.rev)
+            SubmissionEmail.objects.create(
+                    submission = submission,
+                    desc = desc,
+                    msgtype = 'msgout',
+                    by = request.user.person,
+                    message = msg,
+                    in_reply_to = in_reply_to)
+
+            # send email
+            send_mail_message(None,msg)
+
+            messages.success(request, 'Email sent.')
+            return redirect('submit_submission_status_by_hash', 
+                            submission_id=submission.id,
+                            access_token=submission.access_token())
+
+    else:
+        reply_to = get_reply_to()
+        msg = None
+        
+        if not message_id:
+            addrs = gather_address_lists('sub_confirmation_requested',submission=submission).as_strings(compact=False)
+            to_email = addrs.to
+            cc = addrs.cc
+            subject = 'Regarding {}'.format(submission.name)
+        else:
+            try:
+                submitEmail = SubmissionEmail.objects.get(id=message_id)
+                msg = submitEmail.message
+                
+                if msg:
+                    to_email = msg.frm
+                    cc = msg.cc
+                    subject = 'Re:{}'.format(msg.subject)
+                else:
+                    to_email = None
+                    cc = None
+                    subject = 'Regarding {}'.format(submission.name)
+            except Message.DoesNotExist:
+                to_email = None
+                cc = None
+                subject = 'Regarding {}'.format(submission.name)
+
+        initial = {
+            'to': to_email,
+            'cc': cc,
+            'frm': settings.IDSUBMIT_FROM_EMAIL,
+            'subject': subject,
+            'reply_to': reply_to,
+        }
+        
+        if msg:
+            initial['in_reply_to_id'] = msg.id
+        
+        form = MessageModelForm(initial=initial)
+
+    return render(request, "submit/email.html",  {
+        'submission': submission,
+        'access_token': submission.access_token(),
+        'form':form})
+    
+
+def submission_email(request, submission_id, message_id, access_token=None):
+    submission = get_submission_or_404(submission_id, access_token)
+
+    submitEmail = get_object_or_404(SubmissionEmail, pk=message_id)    
+    attachments = submitEmail.message.messageattachment_set.all()
+    
+    return render(request, 'submit/submission_email.html',
+                  {'submission': submission,
+                   'message': submitEmail,
+                   'attachments': attachments})
+
+def submission_email_attachment(request, submission_id, message_id, filename, access_token=None):
+    get_submission_or_404(submission_id, access_token)
+
+    message = get_object_or_404(SubmissionEmail, pk=message_id)
+
+    attach = get_object_or_404(MessageAttachment, 
+                               message=message.message, 
+                               filename=filename)
+    
+    if attach.encoding == "base64":
+        body = base64.b64decode(attach.body)
+    else:
+        body = attach.body.encode('utf-8')
+    
+    if attach.content_type is None:
+        content_type='text/plain'
+    else:
+        content_type=attach.content_type
+        
+    response = HttpResponse(body, content_type=content_type)
+    response['Content-Disposition'] = 'attachment; filename=%s' % attach.filename
+    response['Content-Length'] = len(body)
+    return response
+    
+
+def get_submission_or_404(submission_id, access_token=None):
+    submission = get_object_or_404(Submission, pk=submission_id)
+
+    key_matched = access_token and submission.access_token() == access_token
+    if not key_matched: key_matched = submission.access_key == access_token # backwards-compat
+    if access_token and not key_matched:
+        raise Http404
+
+    return submission

ietf/templates/base/menu.html

@@ -43,7 +43,7 @@
     </a>
     <ul class="dropdown-menu" role="menu">
   {% endif %}
-  <li><a href="{% url "submit_upload_submission" %}">Submit a draft</a></li>
+  <li><a href="{% url "submit_upload_submission" %}">Draft submissions</a></li>
 
   {% if user|has_role:"WG Chair" %}
     {% if flavor == "top" %}<li class="divider hidden-xs"></li>{% endif %}

ietf/templates/submit/add_submit_email.html

@@ -0,0 +1,37 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin %}
+
+{% load bootstrap3 %}
+
+{% block title %}{% if submission == None %}Add new submission request email{% else %}Add submission request email to {{ submission.name }}{% endif %}{% endblock %}
+
+{% block content %}
+  {% origin %}
+    <h1>Add email</h1>
+
+  {% if submission == None %}
+    <p>
+      A new submission request will be created for the given name and revision. The
+      name must take the form "draft-xxx-nn" where xxx is lowercase letters, digits or dashes
+      and nn is the revision number - 00 for the initial revision. For example<br/>
+      &nbsp;&nbsp;&nbsp;draft-my-spec-00
+    </p>
+  {% else %}
+    <p>
+      The email will be added to the submission history for {{ submission.name }}
+    </p>
+  {% endif %}
+
+    <form class="add-email" method="post">
+        {% csrf_token %}
+
+        {% bootstrap_form form %}
+    
+        {% buttons %}
+            <button type="submit" class="btn btn-primary">Add Email</button>
+            <a class="btn btn-default pull-right" href="{% url "submit_manualpost" %}">Back</a>
+        {% endbuttons %}
+    </form>
+
+{% endblock %}

ietf/templates/submit/email.html

@@ -0,0 +1,30 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin %}
+{% load staticfiles %}
+{% load bootstrap3 %}
+
+{% block title %}Email related to{{ submission.name }}{% endblock %}
+
+{% block pagehead %}
+  <link rel="stylesheet" href="{% static 'bootstrap-datepicker/css/bootstrap-datepicker3.min.css' %}">
+{% endblock %}
+
+{% block content %}
+  {% origin %}
+  <h1>Email related to<br><small>{{ submission.name }}</small></h1>
+
+  <form method="post" class="show-required">
+    {% csrf_token %}
+
+    {% bootstrap_form form %}
+
+    {% buttons %}
+      <button type="submit" class="btn btn-primary">Send Email</button>
+    {% endbuttons %}
+  </form>
+{% endblock %}
+
+{% block js %}
+  <script src="{% static 'bootstrap-datepicker/js/bootstrap-datepicker.min.js' %}"></script>
+{% endblock %}

ietf/templates/submit/manual_post.html

@@ -0,0 +1,98 @@
+{% extends "submit/submit_base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin staticfiles %}
+
+{% block pagehead %}
+<link rel="stylesheet" href="{% static "jquery.tablesorter/css/theme.bootstrap.min.css" %}">
+{% endblock %}
+
+{% block title %}Draft submissions awaiting manual posting{% endblock %}
+
+{% load ietf_filters %}
+
+{% block submit_content %}
+  {% origin %}
+
+    <h2 class="anchor-target" id="approvals">Submissions needing manual posting</h2>
+
+    {% if not manual %}
+      <p id="no-manual">There are no submissions needing manual posting.</p>
+    {% else %}
+      <table id="manual" class="submissions table table-condensed table-striped tablesorter">
+        <thead>
+          <tr>
+            <th>Draft</th>
+            <th>Submitted</th>
+            <th>Passes Checks</th>
+            <th>Metadata</th>
+          </tr>
+        </thead>
+        <tbody>
+          {% for s in manual %}
+            <tr>
+              {% if user.is_authenticated %}
+                <td><a href="{% url "submit_submission_status_by_hash" submission_id=s.pk access_token=s.access_token %}">{{ s.name }}-{{ s.rev }}</a></td>
+              {% else %}
+                <td><a href="{% url "submit_submission_status" submission_id=s.pk %}">{{ s.name }}-{{ s.rev }}</a></td>
+              {% endif %}
+              <td>{{ s.submission_date }}</td>
+              <td>{% if s.passes_checks %}Ok{% else %}Fails{% endif %}</td>
+              <td>{% if s.errors %}Errors{% else %}Ok{% endif %}</td>
+            </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    {% endif %}
+
+    <h2 class="anchor-target" id="approvals">Submissions awaiting draft upload</h2>
+
+    {% if not awaiting_draft %}
+      <p id="no-awaiting-draft">There are no submissions awaiting draft upload.</p>
+    {% else %}
+      <table id="awaiting-draft" class="awaiting-draft table table-condensed table-striped tablesorter">
+        <thead>
+        <tr>
+          <th>Name</th>
+          <th>Rev</th>
+          <th>Submitted</th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for s in awaiting_draft %}
+          <tr>
+            {% if user.is_authenticated %}
+              <td><a id="aw{{ s.pk }}" href="{% url "submit_submission_status_by_hash" submission_id=s.pk access_token=s.access_token %}">{{ s.name }}</a></td>
+            {% else %}
+              <td><a id="aw{{ s.pk }}" href="{% url "submit_submission_status" submission_id=s.pk %}">{{ s.name }}</a></td>
+            {% endif %}
+            <td>{{ s.rev }}</td>
+            <td>{{ s.submission_date }}</td>
+
+            {% if user|has_role:"Secretariat" %}
+              <td>
+                <form id="cancel-submission" action="/submit/awaitingdraft/cancel" method="post">
+                  {% csrf_token %}
+                  <input type="hidden" name="submission_id" value="{{ s.pk }}">
+                  <input type="hidden" name="access_token" value="{{ s.access_token }}">
+                  <button class="btn btn-danger btn-xs" type="submit" data-toggle="tooltip" title="Cancels the submission permanently.">Cancel submission</button>
+                </form>
+              </td>
+            {% endif %}
+            {% if user|has_role:"Secretariat" %}
+              <td><a id="add-submission-email{{ s.pk }}" class="btn btn-default btn-xs" href="{% url "submit_manualpost_email_by_hash" submission_id=s.pk access_token=s.access_token %}">Add email</a></td>
+            {% endif %}
+          </tr>
+        {% endfor %}
+        </tbody>
+      </table>
+    {% endif %}
+
+    {% if user|has_role:"Secretariat" %}
+      <a id="new-submission-email" class="btn btn-default btn-sm" href="{% url "submit_manualpost_email" %}">New submission from email</a>
+    {% endif %}
+
+{% endblock %}
+
+{% block js %}
+<script src="{% static "jquery.tablesorter/js/jquery.tablesorter.combined.min.js" %}"></script>
+{% endblock %}

ietf/templates/submit/submission_email.html

@@ -0,0 +1,60 @@
+{% extends "submit/submit_base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin staticfiles %}
+
+{% block pagehead %}
+<link rel="stylesheet" href="{% static "jquery.tablesorter/css/theme.bootstrap.min.css" %}">
+{% endblock %}
+
+{% block title %}Submission email{% endblock %}
+
+{% load ietf_filters %}
+
+{% block submit_content %}
+  {% origin %}
+    <h2 class="anchor-target" id="approvals">Email for {{ submission.name }}</h2>
+
+    <table id="email-details" class="emails table table-condensed table-striped tablesorter">
+      <tr>
+        <th>Uploaded</th>
+        <td>{{ message.time }}</td>
+      </tr>
+
+      <tr>
+        <th>Date</th>
+        <td>{{ message.message.time }}</td>
+      </tr>
+
+      <tr>
+        <th>From</th>
+        <td>{{ message.message.frm }}</td>
+      </tr>
+
+      <tr>
+        <th>Subject</th>
+        <td>{{ message.message.subject }}</td>
+      </tr>
+
+      <tr>
+          <th>Message</th>
+          <td>{{ message.message.body|linebreaksbr }}</td>
+      </tr>
+        
+      {% for a in attachments %}
+        <tr>
+          <th>Attachment</th>
+          <td><a id="attach{{ submission.pk }}" href="{% url "submit_submission_email_attachment" submission_id=submission.pk message_id=message.pk filename=a.filename %}">{{ a.filename }}</a></td>
+        </tr>
+      {% endfor %}
+
+      {% if user|has_role:"Secretariat" %}
+        <tr>
+          <td><a id="reply{{ submission.pk }}" class="btn btn-default" href="{% url "submission_reply_email" submission_id=submission.pk message_id=message.pk %}" title="Reply"><span class="glyphicon glyphicon-envelope" aria-hidden="true"></span> Reply</a></td>
+        </tr>
+      {% endif %}
+    </table>
+{% endblock %}
+
+{% block js %}
+<script src="{% static "jquery.tablesorter/js/jquery.tablesorter.combined.min.js" %}"></script>
+{% endblock %}

ietf/templates/submit/submission_status.html

@@ -101,28 +101,33 @@
     </div>
   </div>
 
+  {% if submission.state_id == "manual-awaiting-draft" %}
+    <p class="alert alert-warning">
+      This submission is awaiting the first draft upload.
+    </p>
+  {% else %}
+    <h2>Meta-data from the submission</h2>
 
-  <h2>Meta-data from the submission</h2>
+    {% if errors %}
+      <div class="alert alert-danger">
+        <p><b>Meta-Data errors found!</b></p>
+      <p  >Please make sure that your Internet-Draft includes all of the required meta-data in the proper format.</p>
 
-  {% if errors %}
-    <div class="alert alert-danger">
-      <p><b>Meta-Data errors found!</b></p>
-      <p>Please make sure that your Internet-Draft includes all of the required meta-data in the proper format.</p>
+        <p>If your Internet-Draft <b>does</b> include all of the required meta-data in the proper format, and if
+          the error(s) identified below are due to the failure of the tool to extract the meta-data correctly,
+          then please use the "Adjust meta-data" button below, which will take you to the "Adjust screen" where
+          you can correct the improperly extracted meta-data. You will then be able to submit your Internet-Draft
+          to the Secretariat for manual posting.</p>
 
-      <p>If your Internet-Draft <b>does</b> include all of the required meta-data in the proper format, and if
-        the error(s) identified below are due to the failure of the tool to extract the meta-data correctly,
-        then please use the "Adjust meta-data" button below, which will take you to the "Adjust screen" where
-        you can correct the improperly extracted meta-data. You will then be able to submit your Internet-Draft
-        to the Secretariat for manual posting.</p>
+        <p>If your Internet-Draft <b>does not</b> include all of the required meta-data in the proper format, then
+          please cancel this submission, update your Internet-Draft, and resubmit it.</p>
 
-      <p>If your Internet-Draft <b>does not</b> include all of the required meta-data in the proper format, then
-        please cancel this submission, update your Internet-Draft, and resubmit it.</p>
-
-      <p><b>Note:</b> The secretariat will <b>not</b> add any
-        meta-data to your Internet-Draft or edit the meta-data. An
-        Internet-Draft that does not include all of the required meta-data in
-        the proper format <b>will</b> be returned to the submitter.</p>
-    </div>
+        <p><b>Note:</b> The secretariat will <b>not</b> add any
+          meta-data to your Internet-Draft or edit the meta-data. An
+          Internet-Draft that does not include all of the required meta-data in
+          the proper format <b>will</b> be returned to the submitter.</p>
+      </div>
+    {% endif %}
   {% endif %}
 
   <table class="table table-condensed table-striped">
@@ -305,6 +310,11 @@
     </form>
   {% endif %}
 
+  {% if user|has_role:"Secretariat" %}
+    <p></p>
+    <a id="send{{ submission.pk }}" class="btn btn-default" href="{% url "submission_send_email" submission_id=submission.pk %}" title="Email submitter"><span class="glyphicon glyphicon-envelope" aria-hidden="true"></span>Send Email</a>
+  {% endif %}
+
   {% if show_send_full_url %}
     <div class="alert alert-danger">
       <p>You are not allowed to modify or cancel this submission. You can
@@ -329,7 +339,7 @@
 
   <h2>History</h2>
 
-  <table class="table table-condensed table-striped">
+  <table  id="history" class="table table-condensed table-striped">
     <thead>
       <tr><th>Date</th><th>By</th><th>Event</th></tr>
     </thead>
@@ -338,7 +348,21 @@
         <tr>
           <td class="text-nowrap">{{ e.time|date:"Y-m-d" }}</td>
           <td>{{ e.by|default:"" }}</td>
-          <td>{{ e.desc }}</td>
+          {%  if e.desc|startswith:"Received message" or e.desc|startswith:"Sent message" %}
+            {%  with m=e.submissionemail.message %}
+              {% if user.is_authenticated %}
+                <td> 
+                  {% if  e.desc|startswith:"Received message" and user|has_role:"Secretariat" %}
+                    <a id="reply{{ submission.pk }}" class="btn btn-default btn-xs" href="{% url "submission_reply_email" submission_id=submission.pk message_id=e.submissionemail.pk %}" title="Reply"><span class="glyphicon glyphicon-envelope" aria-hidden="true"></span> Reply</a>
+                  {% endif %}
+                  Email: <a id="aw{{ submission.pk }}-{{ m.pk }}" href="{% url "submit_submission_email_by_hash" submission_id=submission.pk message_id=e.submissionemail.pk access_token=submission.access_token %}">{{ e.desc }}</a></td>
+              {% else %}
+                <td>Email: <a id="aw{{ submission.pk }}-{{ m.pk }}" href="{% url "submit_submission_email" submission_id=submission.pk message_id=e.submissionemail.pk %}">{{ e.desc }}</a></td>
+              {% endif %}
+            {% endwith %}
+          {%  else %}
+            <td>{{ e.desc }}</td>
+          {%  endif %}
         </tr>
       {% endfor %}
     </tbody>

ietf/templates/submit/submit_base.html

@@ -26,6 +26,9 @@
     <li {% if selected == "approvals" %}class="active"{% endif %}>
       <a href="{% url "submit_approvals" %}">Approvals</a>
     </li>
+    <li {% if selected == "manual_posts" %}class="active"{% endif %}>
+      <a href="{% url "submit_manualpost" %}">Manual Post Requests</a>
+    </li>
   </ul>
 
   {% block submit_content %}

ietf/templates/submit/submitter_form.html

@@ -8,10 +8,17 @@
   please manually enter your name and email address.
 </p>
 
+{% load ietf_filters %}
+
 {% buttons %}
   {% for author in submission.authors_parsed %}
     <input type="button" class="author btn btn-default" data-name="{{ author.name  }}" data-email="{{ author.email }}" value="{{ author.name }}">
   {% endfor %}
 {% endbuttons %}
 
-{% bootstrap_form submitter_form %}
+{% bootstrap_form_errors submitter_form %}
+{% bootstrap_field submitter_form.name %}
+{% bootstrap_field submitter_form.email %}
+{% if user|has_role:"Secretariat" %}
+  {% bootstrap_field submitter_form.approvals_received %}
+{% endif %}

requirements.txt

@@ -29,6 +29,7 @@ python-dateutil>=2.2
 python-magic>=0.4.6
 python-memcached>=1.48		# for django.core.cache.backends.memcached
 pytz>=2014.7
+pyzmail>=1.0.3
 selenium>=2.42
 six>=1.8.0
 tqdm>=3.7.0