import hashlib import random import os import subprocess import datetime from django import forms from django.core.validators import email_re from django.conf import settings from django.contrib.sites.models import Site from django.template.loader import render_to_string from django.utils.html import mark_safe from django.core.urlresolvers import reverse as urlreverse from ietf.idtracker.models import InternetDraft, IETFWG from ietf.proceedings.models import Meeting from ietf.submit.models import IdSubmissionDetail, TempIdAuthors, Preapproval from ietf.submit.utils import MANUAL_POST_REQUESTED, NONE_WG, UPLOADED, WAITING_AUTHENTICATION, POSTED, POSTED_BY_SECRETARIAT from ietf.submit.parsers.pdf_parser import PDFParser from ietf.submit.parsers.plain_parser import PlainParser from ietf.submit.parsers.ps_parser import PSParser from ietf.submit.parsers.xml_parser import XMLParser from ietf.utils.mail import send_mail from ietf.utils.draft import Draft class UploadForm(forms.Form): txt = forms.FileField(label=u'.txt format', required=True) xml = forms.FileField(label=u'.xml format', required=False) pdf = forms.FileField(label=u'.pdf format', required=False) ps = forms.FileField(label=u'.ps format', required=False) fieldsets = [('Upload a draft', ('txt', 'xml', 'pdf', 'ps'))] class Media: css = {'all': ("/css/liaisons.css", )} def __init__(self, *args, **kwargs): self.request=kwargs.pop('request', None) self.remote_ip=self.request.META.get('REMOTE_ADDR', None) super(UploadForm, self).__init__(*args, **kwargs) self.in_first_cut_off = False self.idnits_message = None self.shutdown = False self.draft = None self.filesize = None self.group = None self.file_type = [] self.read_dates() def read_dates(self): now = datetime.datetime.utcnow() first_cut_off = Meeting.get_first_cut_off() second_cut_off = Meeting.get_second_cut_off() ietf_monday = Meeting.get_ietf_monday() if now.date() >= first_cut_off and now.date() < second_cut_off: # We are in the first_cut_off if now.date() == first_cut_off and now.hour < settings.CUTOFF_HOUR: self.cutoff_warning = 'The pre-meeting cut-off date for new documents (i.e., version -00 Internet-Drafts) is %s, at %02sh UTC. After that, you will not be able to submit a new document until %s, at %sh UTC' % (first_cut_off, settings.CUTOFF_HOUR, ietf_monday, settings.CUTOFF_HOUR, ) else: # No 00 version allowed self.cutoff_warning = 'The pre-meeting cut-off date for new documents (i.e., version -00 Internet-Drafts) was %s at %sh UTC. You will not be able to submit a new document until %s, at %sh UTC.
You can still submit a version -01 or higher Internet-Draft until %sh UTC, %s' % (first_cut_off, settings.CUTOFF_HOUR, ietf_monday, settings.CUTOFF_HOUR, settings.CUTOFF_HOUR, second_cut_off, ) self.in_first_cut_off = True elif now.date() >= second_cut_off and now.date() < ietf_monday: if now.date() == second_cut_off and now.hour < settings.CUTOFF_HOUR: # We are in the first_cut_off yet self.cutoff_warning = 'The pre-meeting cut-off date for new documents (i.e., version -00 Internet-Drafts) was %s at %02sh UTC. You will not be able to submit a new document until %s, at %02sh UTC.
The I-D submission tool will be shut down at %02sh UTC today, and reopened at %02sh UTC on %s' % (first_cut_off, settings.CUTOFF_HOUR, ietf_monday, settings.CUTOFF_HOUR, settings.CUTOFF_HOUR, settings.CUTOFF_HOUR, ietf_monday) self.in_first_cut_off = True else: # Completely shut down of the tool self.cutoff_warning = 'The cut-off time for the I-D submission was %02dh UTC, %s.
The I-D submission tool will be reopened at %02dh local time at the IETF meeting location, %s.' % (settings.CUTOFF_HOUR, second_cut_off, settings.CUTOFF_HOUR, ietf_monday) self.shutdown = True def __unicode__(self): return self.as_div() def as_div(self): return render_to_string('submit/submitform.html', {'form': self}) def get_fieldsets(self): if not self.fieldsets: yield dict(name=None, fields=self) else: for fieldset, fields in self.fieldsets: fieldset_dict = dict(name=fieldset, fields=[]) for field_name in fields: if field_name in self.fields.keyOrder: fieldset_dict['fields'].append(self[field_name]) if not fieldset_dict['fields']: # if there is no fields in this fieldset, we continue to next fieldset continue yield fieldset_dict def clean_txt(self): txt_file = self.cleaned_data['txt'] if not txt_file: return txt_file parsed_info = PlainParser(txt_file).critical_parse() if parsed_info.errors: raise forms.ValidationError(parsed_info.errors) self.filesize=txt_file.size return txt_file def clean_pdf(self): pdf_file = self.cleaned_data['pdf'] if not pdf_file: return pdf_file parsed_info = PDFParser(pdf_file).critical_parse() if parsed_info.errors: raise forms.ValidationError(parsed_info.errors) return pdf_file def clean_ps(self): ps_file = self.cleaned_data['ps'] if not ps_file: return ps_file parsed_info = PSParser(ps_file).critical_parse() if parsed_info.errors: raise forms.ValidationError(parsed_info.errors) return ps_file def clean_xml(self): xml_file = self.cleaned_data['xml'] if not xml_file: return xml_file parsed_info = XMLParser(xml_file).critical_parse() if parsed_info.errors: raise forms.ValidationError(parsed_info.errors) return xml_file def clean(self): if self.shutdown: raise forms.ValidationError('The tool is shut down') self.check_paths() if self.cleaned_data.get('txt', None): self.get_draft() self.group=self.get_working_group() self.check_previous_submission() if self.draft.revision == '00' and self.in_first_cut_off: raise forms.ValidationError(mark_safe(self.cutoff_warning)) self.check_tresholds() return super(UploadForm, self).clean() def check_tresholds(self): filename = self.draft.filename revision = self.draft.revision remote_ip = self.remote_ip today = datetime.date.today() # Same draft by name same_name = IdSubmissionDetail.objects.filter(filename=filename, revision=revision, submission_date=today) if same_name.count() > settings.MAX_SAME_DRAFT_NAME: raise forms.ValidationError('The same I-D cannot be submitted more than %s times a day' % settings.MAX_SAME_DRAFT_NAME) if sum([i.filesize for i in same_name]) > (settings.MAX_SAME_DRAFT_NAME_SIZE * 1048576): raise forms.ValidationError('The same I-D submission cannot exceed more than %s MByte a day' % settings.MAX_SAME_DRAFT_NAME_SIZE) # Total from same ip same_ip = IdSubmissionDetail.objects.filter(remote_ip=remote_ip, submission_date=today) if same_ip.count() > settings.MAX_SAME_SUBMITTER: raise forms.ValidationError('The same submitter cannot submit more than %s I-Ds a day' % settings.MAX_SAME_SUBMITTER) if sum([i.filesize for i in same_ip]) > (settings.MAX_SAME_SUBMITTER_SIZE * 1048576): raise forms.ValidationError('The same submitter cannot exceed more than %s MByte a day' % settings.MAX_SAME_SUBMITTER_SIZE) # Total in same group if self.group: same_group = IdSubmissionDetail.objects.filter(group_acronym=self.group, submission_date=today) if same_group.count() > settings.MAX_SAME_WG_DRAFT: raise forms.ValidationError('The same working group I-Ds cannot be submitted more than %s times a day' % settings.MAX_SAME_WG_DRAFT) if sum([i.filesize for i in same_group]) > (settings.MAX_SAME_WG_DRAFT_SIZE * 1048576): raise forms.ValidationError('Total size of same working group I-Ds cannot exceed %s MByte a day' % settings.MAX_SAME_WG_DRAFT_SIZE) # Total drafts for today total_today = IdSubmissionDetail.objects.filter(submission_date=today) if total_today.count() > settings.MAX_DAILY_SUBMISSION: raise forms.ValidationError('The total number of today\'s submission has reached the maximum number of submission per day') if sum([i.filesize for i in total_today]) > (settings.MAX_DAILY_SUBMISSION_SIZE * 1048576): raise forms.ValidationError('The total size of today\'s submission has reached the maximum size of submission per day') def check_paths(self): self.staging_path = getattr(settings, 'IDSUBMIT_STAGING_PATH', None) self.idnits = getattr(settings, 'IDSUBMIT_IDNITS_BINARY', None) if not self.staging_path: raise forms.ValidationError('IDSUBMIT_STAGING_PATH not defined in settings.py') if not os.path.exists(self.staging_path): raise forms.ValidationError('IDSUBMIT_STAGING_PATH defined in settings.py does not exist') if not self.idnits: raise forms.ValidationError('IDSUBMIT_IDNITS_BINARY not defined in settings.py') if not os.path.exists(self.idnits): raise forms.ValidationError('IDSUBMIT_IDNITS_BINARY defined in settings.py does not exist') def check_previous_submission(self): filename = self.draft.filename revision = self.draft.revision existing = IdSubmissionDetail.objects.filter(filename=filename, revision=revision, status__pk__gte=0, status__pk__lt=100) if existing: raise forms.ValidationError(mark_safe('Duplicate Internet-Draft submission is currently in process. Check it here' % existing[0].pk)) def get_draft(self): if self.draft: return self.draft txt_file = self.cleaned_data['txt'] txt_file.seek(0) self.draft = Draft(txt_file.read(), txt_file.name) txt_file.seek(0) return self.draft def save(self): for ext in ['txt', 'pdf', 'xml', 'ps']: fd = self.cleaned_data[ext] if not fd: continue self.file_type.append('.%s' % ext) filename = os.path.join(self.staging_path, '%s-%s.%s' % (self.draft.filename, self.draft.revision, ext)) destination = open(filename, 'wb+') for chunk in fd.chunks(): destination.write(chunk) destination.close() self.check_idnits() return self.save_draft_info(self.draft) def check_idnits(self): filepath = os.path.join(self.staging_path, '%s-%s.txt' % (self.draft.filename, self.draft.revision)) p = subprocess.Popen([self.idnits, '--submitcheck', '--nitcount', filepath], stdout=subprocess.PIPE) self.idnits_message = p.stdout.read() def get_working_group(self): filename = self.draft.filename existing_draft = InternetDraft.objects.filter(filename=filename) if existing_draft: group = existing_draft[0].group and existing_draft[0].group.ietfwg or None if group and group.pk != NONE_WG: if settings.USE_DB_REDESIGN_PROXY_CLASSES and group.type_id == "area": return None return group else: return None else: if filename.startswith('draft-ietf-'): # Extra check for WG that contains dashes for group in IETFWG.objects.filter(group_acronym__acronym__contains='-'): if filename.startswith('draft-ietf-%s-' % group.group_acronym.acronym): return group group_acronym = filename.split('-')[2] try: return IETFWG.objects.get(group_acronym__acronym=group_acronym) except IETFWG.DoesNotExist: raise forms.ValidationError('There is no active group with acronym \'%s\', please rename your draft' % group_acronym) else: return None def save_draft_info(self, draft): document_id = 0 existing_draft = InternetDraft.objects.filter(filename=draft.filename) if existing_draft: if settings.USE_DB_REDESIGN_PROXY_CLASSES: document_id = -1 else: document_id = existing_draft[0].id_document_tag detail = IdSubmissionDetail.objects.create( id_document_name=draft.get_title(), filename=draft.filename, revision=draft.revision, txt_page_count=draft.get_pagecount(), filesize=self.filesize, creation_date=draft.get_creation_date(), submission_date=datetime.date.today(), idnits_message=self.idnits_message, temp_id_document_tag=document_id, group_acronym=self.group, remote_ip=self.remote_ip, first_two_pages=''.join(draft.pages[:2]), status_id=UPLOADED, abstract=draft.get_abstract(), file_type=','.join(self.file_type), ) order = 0 for author in draft.get_author_list(): full_name, first_name, middle_initial, last_name, name_suffix, email, company = author order += 1 if settings.USE_DB_REDESIGN_PROXY_CLASSES: # save full name TempIdAuthors.objects.create( id_document_tag=document_id, first_name=full_name.strip(), email_address=(email or "").strip(), author_order=order, submission=detail) else: TempIdAuthors.objects.create( id_document_tag=document_id, first_name=first_name, middle_initial=middle_initial, last_name=last_name, name_suffix=name_suffix, email_address=email, author_order=order, submission=detail) return detail class AutoPostForm(forms.Form): if settings.USE_DB_REDESIGN_PROXY_CLASSES: name = forms.CharField(required=True) else: first_name = forms.CharField(label=u'Given name', required=True) last_name = forms.CharField(label=u'Last name', required=True) email = forms.EmailField(label=u'Email address', required=True) def __init__(self, *args, **kwargs): self.draft = kwargs.pop('draft', None) self.validation = kwargs.pop('validation', None) super(AutoPostForm, self).__init__(*args, **kwargs) def get_author_buttons(self): if settings.USE_DB_REDESIGN_PROXY_CLASSES: buttons = [] for i in self.validation.authors: buttons.append('' % dict(name=i.get_full_name(), email=i.email()[1] or '')) return "".join(buttons) # this should be moved to a Javascript file and attributes like data-first-name ... button_template = '' buttons = [] for i in self.validation.authors: full_name = u'%s. %s' % (i.first_name[0], i.last_name) buttons.append(button_template % {'first_name': i.first_name, 'last_name': i.last_name, 'email': i.email()[1] or '', 'full_name': full_name}) return ''.join(buttons) def save(self, request): self.save_submitter_info() self.save_new_draft_info() self.send_confirmation_mail(request) def send_confirmation_mail(self, request): subject = 'Confirmation for Auto-Post of I-D %s' % self.draft.filename from_email = settings.IDSUBMIT_FROM_EMAIL to_email = self.cleaned_data['email'] confirm_url = settings.IDTRACKER_BASE_URL + urlreverse('draft_confirm', kwargs=dict(submission_id=self.draft.submission_id, auth_key=self.draft.auth_key)) status_url = settings.IDTRACKER_BASE_URL + urlreverse('draft_status_by_hash', kwargs=dict(submission_id=self.draft.submission_id, submission_hash=self.draft.get_hash())) send_mail(request, to_email, from_email, subject, 'submit/confirm_autopost.txt', { 'draft': self.draft, 'confirm_url': confirm_url, 'status_url': status_url }) def save_submitter_info(self): if settings.USE_DB_REDESIGN_PROXY_CLASSES: return TempIdAuthors.objects.create( id_document_tag=self.draft.temp_id_document_tag, first_name=self.cleaned_data['name'], email_address=self.cleaned_data['email'], author_order=0, submission=self.draft) return TempIdAuthors.objects.create( id_document_tag=self.draft.temp_id_document_tag, first_name=self.cleaned_data['first_name'], last_name=self.cleaned_data['last_name'], email_address=self.cleaned_data['email'], author_order=0, submission=self.draft) def save_new_draft_info(self): salt = hashlib.sha1(str(random.random())).hexdigest()[:5] self.draft.auth_key = hashlib.sha1(salt+self.cleaned_data['email']).hexdigest() self.draft.status_id = WAITING_AUTHENTICATION self.draft.save() class MetaDataForm(AutoPostForm): title = forms.CharField(label=u'Title', required=True) version = forms.CharField(label=u'Version', required=True) creation_date = forms.DateField(label=u'Creation date', required=True) pages = forms.IntegerField(label=u'Pages', required=True) abstract = forms.CharField(label=u'Abstract', widget=forms.Textarea, required=True) if settings.USE_DB_REDESIGN_PROXY_CLASSES: name = forms.CharField(required=True) else: first_name = forms.CharField(label=u'Given name', required=True) last_name = forms.CharField(label=u'Last name', required=True) email = forms.EmailField(label=u'Email address', required=True) comments = forms.CharField(label=u'Comments to the secretariat', widget=forms.Textarea, required=False) if settings.USE_DB_REDESIGN_PROXY_CLASSES: fields = ['title', 'version', 'creation_date', 'pages', 'abstract', 'name', 'email', 'comments'] else: fields = ['title', 'version', 'creation_date', 'pages', 'abstract', 'first_name', 'last_name', 'email', 'comments'] def __init__(self, *args, **kwargs): super(MetaDataForm, self).__init__(*args, **kwargs) self.set_initials() self.authors = self.get_initial_authors() def get_initial_authors(self): authors=[] if self.is_bound: for key, value in self.data.items(): if settings.USE_DB_REDESIGN_PROXY_CLASSES: if key.startswith('name_'): author = {'errors': {}} index = key.replace('name_', '') name = value.strip() if not name: author['errors']['name'] = 'This field is required' email = self.data.get('email_%s' % index, '').strip() if email and not email_re.search(email): author['errors']['email'] = 'Enter a valid e-mail address' if name or email: author.update({'get_full_name': name, 'email': (name, email), 'index': index, }) authors.append(author) else: if key.startswith('first_name_'): author = {'errors': {}} index = key.replace('first_name_', '') first_name = value.strip() if not first_name: author['errors']['first_name'] = 'This field is required' last_name = self.data.get('last_name_%s' % index, '').strip() if not last_name: author['errors']['last_name'] = 'This field is required' email = self.data.get('email_%s' % index, '').strip() if email and not email_re.search(email): author['errors']['email'] = 'Enter a valid e-mail address' if first_name or last_name or email: author.update({'first_name': first_name, 'last_name': last_name, 'email': ('%s %s' % (first_name, last_name), email), 'index': index, }) authors.append(author) authors.sort(key=lambda x: x['index']) return authors def set_initials(self): self.fields['pages'].initial=self.draft.txt_page_count self.fields['creation_date'].initial=self.draft.creation_date self.fields['version'].initial=self.draft.revision self.fields['abstract'].initial=self.draft.abstract self.fields['title'].initial=self.draft.id_document_name def clean_creation_date(self): creation_date = self.cleaned_data.get('creation_date', None) if not creation_date: return None submit_date = self.draft.submission_date if (creation_date + datetime.timedelta(days=3) < submit_date or creation_date - datetime.timedelta(days=3) > submit_date): raise forms.ValidationError('Creation Date must be within 3 days of submission date') return creation_date def clean_version(self): version = self.cleaned_data.get('version', None) if not version: return None if len(version) > 2: raise forms.ValidationError('Version field is not in NN format') try: version_int = int(version) except ValueError: raise forms.ValidationError('Version field is not in NN format') if version_int > 99 or version_int < 0: raise forms.ValidationError('Version must be set between 00 and 99') existing_revisions = [int(i.revision_display()) for i in InternetDraft.objects.filter(filename=self.draft.filename)] expected = 0 if existing_revisions: expected = max(existing_revisions) + 1 if version_int != expected: raise forms.ValidationError('Invalid Version Number (Version %02d is expected)' % expected) return version def clean(self): if bool([i for i in self.authors if i['errors']]): raise forms.ValidationError('Please fix errors in author list') return super(MetaDataForm, self).clean() def get_authors(self): if not self.is_bound: return self.validation.get_authors() else: return self.authors def move_docs(self, draft, revision): old_revision = draft.revision for ext in draft.file_type.split(','): source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (draft.filename, old_revision, ext)) dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (draft.filename, revision, ext)) os.rename(source, dest) def save_new_draft_info(self): draft = self.draft draft.id_document_name = self.cleaned_data['title'] if draft.revision != self.cleaned_data['version']: self.move_docs(draft, self.cleaned_data['version']) draft.revision = self.cleaned_data['version'] draft.creation_date = self.cleaned_data['creation_date'] draft.txt_page_count = self.cleaned_data['pages'] draft.abstract = self.cleaned_data['abstract'] draft.comment_to_sec = self.cleaned_data['comments'] draft.status_id = MANUAL_POST_REQUESTED draft.save() # sync authors draft.tempidauthors_set.all().delete() self.save_submitter_info() # submitter is author 0 for i, author in enumerate(self.authors): if settings.USE_DB_REDESIGN_PROXY_CLASSES: # save full name TempIdAuthors.objects.create( id_document_tag=draft.temp_id_document_tag, first_name=author["get_full_name"], email_address=author["email"][1], author_order=i + 1, submission=draft) def save(self, request): self.save_new_draft_info() self.send_mail_to_secretariat(request) def send_mail_to_secretariat(self, request): subject = 'Manual Post Requested for %s' % self.draft.filename from_email = settings.IDSUBMIT_FROM_EMAIL to_email = settings.IDSUBMIT_TO_EMAIL cc = [self.cleaned_data['email']] cc += [i['email'][1] for i in self.authors] if self.draft.group_acronym: cc += [i.person.email()[1] for i in self.draft.group_acronym.wgchair_set.all()] cc = list(set(cc)) submitter = self.draft.tempidauthors_set.get(author_order=0) send_mail(request, to_email, from_email, subject, 'submit/manual_post_mail.txt', { 'form': self, 'draft': self.draft, 'url': settings.IDTRACKER_BASE_URL + urlreverse('draft_status', kwargs=dict(submission_id=self.draft.submission_id)), 'submitter': submitter }, cc=cc) class PreapprovalForm(forms.Form): name = forms.CharField(max_length=255, required=True, label="Pre-approved name", initial="draft-ietf-") def clean_name(self): n = self.cleaned_data['name'].strip().lower() if not n.startswith("draft-"): raise forms.ValidationError("Name doesn't start with \"draft-\".") if len(n.split(".")) > 1 and len(n.split(".")[-1]) == 3: raise forms.ValidationError("Name appears to end with a file extension .%s - do not include an extension." % n.split(".")[-1]) components = n.split("-") if components[-1] == "00": raise forms.ValidationError("Name appears to end with a revision number -00 - do not include the revision.") if len(components) < 4: raise forms.ValidationError("Name has less than four dash-delimited components - can't form a valid WG draft name.") if not components[-1]: raise forms.ValidationError("Name ends with a dash.") acronym = components[2] if acronym not in self.groups.values_list('acronym', flat=True): raise forms.ValidationError("WG acronym not recognized as one you can approve drafts for.") if Preapproval.objects.filter(name=n): raise forms.ValidationError("Pre-approval for this name already exists.") if IdSubmissionDetail.objects.filter(status__in=[POSTED, POSTED_BY_SECRETARIAT ], filename=n): raise forms.ValidationError("A draft with this name has already been submitted and accepted. A pre-approval would not make any difference.") return n