Saving files in staging area.

Saving information of upload in database.
Executing idnits.
Fixes #589
 - Legacy-Id: 2835

ietf/settings.py

@@ -188,10 +188,17 @@ LIAISON_UNIVERSAL_FROM = 'Liaison Statement Management Tool <lsmt@' + IETF_DOMAI
 LIAISON_ATTACH_PATH = '/a/www/ietf-datatracker/documents/LIAISON/'
 LIAISON_ATTACH_URL = '/documents/LIAISON/'
 
+# ID Submission Tool settings
 # Days from meeting to cut off dates on submit
 FIRST_CUTOFF_DAYS = 5
 SECOND_CUTOFF_DAYS = 3
 
+STAGING_PATH = '/a/www/www6s/staging/'
+IDNITS_PATH = '/a/www/ietf-datatracker/release/idnits'
+MAX_PLAIN_DRAFT_SIZE = 6291456  # Max size of the txt draft in bytes
+
+# End of ID Submission Tool settings
+
 # Put SECRET_KEY in here, or any other sensitive or site-specific
 # changes.  DO NOT commit settings_local.py to svn.
 from settings_local import *

ietf/submit/error_manager.py

@@ -1,19 +0,0 @@
-from ietf.submit.models import IdSubmissionStatus
-
-class ErrorManager(object):
-    ERROR_CODES = {
-        'DEFAULT': 'Unknow error',
-        'INVALID_FILENAME': 111,
-        'EXCEEDED_SIZE': 102,
-    }
-
-    def get_error_str(self, key):
-        error_code = self.ERROR_CODES.get(key, self.ERROR_CODES['DEFAULT'])
-        if isinstance(error_code, basestring):
-            return '%s (%s)' % (key, error_code)
-        try:
-            return IdSubmissionStatus.objects.get(status_id=error_code).status_value
-        except IdSubmissionStatus.DoesNotExist:
-            return '%s (%s)' % (self.ERROR_CODES['DEFAULT'], key)
-
-MainErrorManager=ErrorManager()

ietf/submit/forms.py

@@ -1,13 +1,18 @@
+import os
+import subprocess
 import datetime
 
 from django import forms
+from django.conf import settings
 from django.template.loader import render_to_string
 
 from ietf.proceedings.models import Meeting
-from ietf.submit.parsers.plain_parser import PlainParser
+from ietf.submit.models import IdSubmissionDetail
 from ietf.submit.parsers.pdf_parser import PDFParser
+from ietf.submit.parsers.plain_parser import PlainParser
 from ietf.submit.parsers.ps_parser import PSParser
 from ietf.submit.parsers.xml_parser import XMLParser
+from ietf.utils.draft import Draft
 
 
 CUTOFF_HOUR = 17
@@ -28,7 +33,9 @@ class UploadForm(forms.Form):
     def __init__(self, *args, **kwargs):
         super(UploadForm, self).__init__(*args, **kwargs)
         self.in_first_cut_off = False
+        self.idnits_message = None
         self.shutdown = False
+        self.draft = None
         self.read_dates()
 
     def read_dates(self):
@@ -72,34 +79,102 @@ class UploadForm(forms.Form):
                 yield fieldset_dict
 
     def clean_txt(self):
-        if not self.cleaned_data['txt']:
-            return None
-        parsed_info = PlainParser(self.cleaned_data['txt']).critical_parse()
+        txt_file = self.cleaned_data['txt']
+        if not txt_file:
+            return txt_file
+        parsed_info = PlainParser(txt_file).critical_parse()
         if parsed_info.errors:
             raise forms.ValidationError(parsed_info.errors)
+        return txt_file
 
     def clean_pdf(self):
-        if not self.cleaned_data['pdf']:
-            return None
-        parsed_info = PDFParser(self.cleaned_data['pdf']).critical_parse()
+        pdf_file = self.cleaned_data['pdf']
+        if not pdf_file: 
+            return pdf_file
+        parsed_info = PDFParser(pdf_file).critical_parse()
         if parsed_info.errors:
             raise forms.ValidationError(parsed_info.errors)
+        return pdf_file
 
     def clean_ps(self):
-        if not self.cleaned_data['ps']:
-            return None
-        parsed_info = PSParser(self.cleaned_data['ps']).critical_parse()
+        ps_file = self.cleaned_data['ps']
+        if not ps_file: 
+            return ps_file
+        parsed_info = PSParser(ps_file).critical_parse()
         if parsed_info.errors:
             raise forms.ValidationError(parsed_info.errors)
+        return ps_file
 
     def clean_xml(self):
-        if not self.cleaned_data['xml']:
-            return None
-        parsed_info = XMLParser(self.cleaned_data['xml']).critical_parse()
+        xml_file = self.cleaned_data['xml']
+        if not xml_file: 
+            return xml_file
+        parsed_info = XMLParser(xml_file).critical_parse()
         if parsed_info.errors:
             raise forms.ValidationError(parsed_info.errors)
+        return xml_file
 
     def clean(self):
         if self.shutdown:
             raise forms.ValidationError('The tool is shut down')
+        self.staging_path = getattr(settings, 'STAGING_PATH', None)
+        self.idnits = getattr(settings, 'IDNITS_PATH', None)
+        if not self.staging_path:
+            raise forms.ValidationError('STAGING_PATH not defined on settings.py')
+        if not os.path.exists(self.staging_path):
+            raise forms.ValidationError('STAGING_PATH defined on settings.py does not exist')
+        if not self.idnits:
+            raise forms.ValidationError('IDNITS_PATH not defined on settings.py')
+        if not os.path.exists(self.idnits):
+            raise forms.ValidationError('IDNITS_PATH defined on settings.py does not exist')
+        if self.cleaned_data.get('txt', None):
+            self.get_draft()
+            self.check_previous_submission()
         return super(UploadForm, self).clean()
+
+    def check_previous_submission(self):
+        filename = self.draft.filename
+        revision = self.draft.revision
+        existing = IdSubmissionDetail.objects.filter(filename=filename, revision=revision,
+                                                     status__pk__gte=0, status__pk__lt=100)
+        if existing:
+            raise forms.ValidationError('Duplicate Internet-Draft submission is currently in process.')
+
+    def get_draft(self):
+        if self.draft:
+            return self.draft
+        txt_file = self.cleaned_data['txt']
+        txt_file.seek(0)
+        self.draft = Draft(txt_file.read())
+        txt_file.seek(0)
+        return self.draft
+    
+    def save(self):
+        for fd in [self.cleaned_data['txt'], self.cleaned_data['pdf'],
+                   self.cleaned_data['xml'], self.cleaned_data['ps']]:
+            if not fd:
+                continue
+            filename = os.path.join(self.staging_path, fd.name)
+            destination = open(filename, 'wb+')
+            for chunk in fd.chunks():
+                destination.write(chunk)
+                destination.close()
+        self.check_idnits()
+        return self.save_draft_info(self.draft)
+
+    def check_idnits(self):
+        filepath = os.path.join(self.staging_path, self.cleaned_data['txt'].name)
+        p = subprocess.Popen([self.idnits, '--submitcheck', '--nitcount', filepath], stdout=subprocess.PIPE)
+        self.idnits_message = p.stdout.read()
+
+    def save_draft_info(self, draft):
+        detail = IdSubmissionDetail.objects.create(
+            id_document_name=draft.get_title(),
+            filename=draft.filename,
+            revision=draft.revision,
+            txt_page_count=draft.get_pagecount(),
+            creation_date=draft.get_creation_date(),
+            idnits_message=self.idnits_message,
+            status_id=1,  # Status 1 - upload
+            )
+        return detail

ietf/submit/models.py

@@ -6,3 +6,37 @@ class IdSubmissionStatus(models.Model):
 
     class Meta:
         db_table = 'id_submission_status'
+
+class IdSubmissionDetail(models.Model):
+    submission_id = models.AutoField(primary_key=True)
+    temp_id_document_tag = models.IntegerField(null=True, blank=True)
+    status = models.ForeignKey(IdSubmissionStatus, db_column='status_id', null=True, blank=True)
+    last_updated_date = models.DateField(null=True, blank=True)
+    last_updated_time = models.CharField(blank=True, max_length=25)
+    id_document_name = models.CharField(blank=True, max_length=255)
+    group_acronym_id = models.IntegerField(null=True, blank=True)
+    filename = models.CharField(blank=True, max_length=255)
+    creation_date = models.DateField(null=True, blank=True)
+    submission_date = models.DateField(null=True, blank=True)
+    remote_ip = models.CharField(blank=True, max_length=100)
+    revision = models.CharField(blank=True, max_length=3)
+    submitter_tag = models.IntegerField(null=True, blank=True)
+    auth_key = models.CharField(blank=True, max_length=255)
+    idnits_message = models.TextField(blank=True)
+    file_type = models.CharField(blank=True, max_length=50)
+    comment_to_sec = models.TextField(blank=True)
+    abstract = models.TextField(blank=True)
+    txt_page_count = models.IntegerField(null=True, blank=True)
+    error_message = models.CharField(blank=True, max_length=255)
+    warning_message = models.TextField(blank=True)
+    wg_submission = models.IntegerField(null=True, blank=True)
+    filesize = models.IntegerField(null=True, blank=True)
+    man_posted_date = models.DateField(null=True, blank=True)
+    man_posted_by = models.CharField(blank=True, max_length=255)
+    first_two_pages = models.TextField(blank=True)
+    sub_email_priority = models.IntegerField(null=True, blank=True)
+    invalid_version = models.IntegerField(null=True, blank=True)
+    idnits_failed = models.IntegerField(null=True, blank=True)
+
+    class Meta:
+        db_table = 'id_submission_detail'

ietf/submit/parsers/plain_parser.py

@@ -1,11 +1,11 @@
 import datetime
 import re
 
+from django.conf import settings
 from ietf.idtracker.models import InternetDraft, IETFWG
-from ietf.submit.error_manager import MainErrorManager
+from django.template.defaultfilters import filesizeformat
 from ietf.submit.parsers.base import FileParser
 
-MAX_PLAIN_FILE_SIZE = 6000000
 NONE_WG_PK = 1027
 
 
@@ -24,8 +24,8 @@ class PlainParser(FileParser):
         return self.parsed_info
 
     def parse_max_size(self):
-        if self.fd.size > MAX_PLAIN_FILE_SIZE:
-            self.parsed_info.add_error(MainErrorManager.get_error_str('EXCEEDED_SIZE'))
+        if self.fd.size > settings.MAX_PLAIN_DRAFT_SIZE:
+            self.parsed_info.add_error('File size is larger than %s' % filesizeformat(settings.MAX_PLAIN_DRAFT_SIZE))
         self.parsed_info.metadraft.filesize = self.fd.size
         self.parsed_info.metadraft.submission_date = datetime.date.today()
 
@@ -62,7 +62,7 @@ class PlainParser(FileParser):
             filename = re.sub('-\d+$', '', filename)
             self.parsed_info.metadraft.filename = filename
             return
-        self.parsed_info.add_error(MainErrorManager.get_error_str('INVALID_FILENAME'))
+        self.parsed_info.add_error('The document does not contain a legitimate filename that start with draft-*')
 
     def parse_wg(self):
         filename = self.parsed_info.metadraft.filename

ietf/submit/urls.py

@@ -4,6 +4,7 @@ from django.conf.urls.defaults import patterns, url
 urlpatterns = patterns('ietf.submit.views',
     url(r'^$', 'submit_index', name='submit_index'),
     url(r'^status/$', 'submit_status', name='submit_status'),
+    url(r'^status/(?P<submission_id>\d+)/$', 'draft_status', name='draft_status'),
 )
 
 urlpatterns += patterns('django.views.generic.simple',

ietf/submit/utils.py

@@ -0,0 +1,7 @@
+import re
+
+def check_idnits_success(idnits_message):
+    success_re = re.compile('\s+Summary:\s+0\s+|No nits found')
+    if success_re.search(idnits_message):
+        return True
+    return False

ietf/submit/views.py

@@ -1,15 +1,21 @@
 # Copyright The IETF Trust 2007, All Rights Reserved
+from django.core.urlresolvers import reverse
+from django.http import HttpResponseRedirect
+from django.shortcuts import get_object_or_404
 from django.shortcuts import render_to_response
 from django.template import RequestContext
 
+from ietf.submit.models import IdSubmissionDetail
 from ietf.submit.forms import UploadForm
+from ietf.submit.utils import check_idnits_success
 
 
 def submit_index(request):
     if request.method == 'POST':
         form = UploadForm(data=request.POST, files=request.FILES)
         if form.is_valid():
-            pass
+            submit = form.save()
+            return HttpResponseRedirect(reverse(draft_status, None, kwargs={'submission_id': submit.submission_id}))
     else:
         form = UploadForm()
     return render_to_response('submit/submit_index.html', 
@@ -19,4 +25,28 @@ def submit_index(request):
 
 
 def submit_status(request):
-    pass
+    error = None
+    filename = None
+    if request.method == 'POST':
+        filename = request.POST.get('filename', '')
+        detail = IdSubmissionDetail.objects.filter(filename=filename)
+        if detail:
+            return HttpResponseRedirect(reverse(draft_status, None, kwargs={'submission_id': detail[0].submission_id}))
+        error = 'No valid history found for %s' % filename
+    return render_to_response('submit/submit_status.html', 
+                              {'selected': 'status',
+                               'error': error,
+                               'filename': filename},
+                              context_instance=RequestContext(request))
+    
+
+
+def draft_status(request, submission_id):
+    detail = get_object_or_404(IdSubmissionDetail, submission_id=submission_id)
+    idnits_success = check_idnits_success(detail.idnits_message)
+    return render_to_response('submit/draft_status.html', 
+                              {'selected': 'status',
+                               'detail': detail,
+                               'idnits_success': idnits_success,
+                              },
+                              context_instance=RequestContext(request))

ietf/templates/submit/draft_status.html

@@ -0,0 +1,70 @@
+{% extends "submit/submit_base.html" %}
+{% block title %}Submission status{% endblock %}
+
+{% block pagehead %}
+<script type="text/javascript" src="/js/lib/jquery-1.4.2.min.js"></script>
+<script type="text/javascript">
+(function ($) {
+
+    $.fn.IdnitsInfo = function() {
+        return this.each(function () {
+            var title = $(this).attr('title');
+
+            var handleClose = function() {
+                idnitsDialog.hide();
+            };
+
+            var buttons = [{text:"Close", handler:handleClose, isDefault:true}];
+            var kl = [new YAHOO.util.KeyListener(document, {keys:27}, handleClose)]
+            var idnitsDialog = new YAHOO.widget.Dialog("idnits_results", {
+                visible:false, draggable:false, close:true, modal:true,
+                width:"860px", fixedcenter:true, constraintoviewport:true,
+                buttons: buttons, keylisteners:kl});
+            idnitsDialog.render();
+
+            var showIdnits = function() {
+                idnitsDialog.show();
+            }
+
+            $(this).click(function() {
+                showIdnits();
+                return false;
+            });
+        });
+    };
+
+    $(document).ready(function () {
+        $('a.idnits_trigger').IdnitsInfo();
+    });
+
+})(jQuery);
+</script>
+{% endblock %}
+
+{% block submit_content %}
+<h2>Check Page</h2>
+<p>
+{% if idnits_success %}
+Your draft has been verified to meet IDNITS requirements.
+{% else %}
+Your draft has NOT been verified to meet IDNITS requirements.
+{% endif %}
+<a class="idnits_trigger" href="#">(View IDNITS Results)</a>
+</p>
+<div id="idnits_results" style="visibility:hidden;">
+  <div class="hd"><span id="idnits_title">Idnits results</span></div>
+  <div class="bd"> 
+    <div id="stream_dialog_body" style="padding: 0em 5em; height: 400px; overflow: auto;">
+      <pre>{{ detail.idnits_message }}</pre>
+    </div>
+  </div>
+</div>
+
+
+<h2>Meta-Data from the Draft</h2>
+
+<p>
+The IETF is an organized activity of the <a href="http://www.isoc.org">Internet Society</a>
+<br>Please send problem reports to <a href="mailto:ietf-action@ietf.org">ietf-action@ietf.org</a>.
+</p>
+{% endblock %}

ietf/templates/submit/submit_status.html

@@ -0,0 +1,27 @@
+{% extends "submit/submit_base.html" %}
+{% block title %}Submission status{% endblock %}
+
+{% block pagehead %}
+{{ form.media }}
+{% endblock %}
+
+{% block submit_content %}
+<p>
+Please enter the filename of the Internet-Draft who's status you wish to view:
+</p>
+
+<form method="post" action="">
+{% if error %}<div style="color: red;">{{ error }}</div>{% endif %}
+<input type="text" name="filename" value="{{ filename|default:"draft-" }}" />
+<input type="submit" value="Find status" />
+</form>
+
+<p>
+<strong>
+Please note that the Status page only displays the status of an Internet-Draft whose posting is still in progress or an Internet-Draft that has been successfully posted. </strong>
+</p>
+<p>
+The IETF is an organized activity of the <a href="http://www.isoc.org">Internet Society</a>
+<br>Please send problem reports to <a href="mailto:ietf-action@ietf.org">ietf-action@ietf.org</a>.
+</p>
+{% endblock %}

ietf/templates/submit/submitform.html

@@ -13,6 +13,7 @@
    <div class="formErrors">
      Please correct the errors below.
    </div>
+   {{ form.non_field_errors }}
   {% endif %}
 {% for fieldset in form.get_fieldsets %}
   {% if fieldset.name %}