e1f0917659
The new API requires at least one event and will automatically save a snapshot of the document and related state. Document.save() will now throw an exception if called directly, as the new API is intended to ensure that documents are saved with both an appropriate snapsnot and relevant history log, both of which are easily defeated by just calling .save() directly. To simplify things, the snapshot is generated after the changes to a document have been made (in anticipation of coming changes), instead of before as was usual. While revising the existing code to work with this API, a couple of missing events was discovered: - In draft expiry, a "Document has expired" event was only generated in case an IESG process had started on the document - now it's always generated, as the document changes its state in any case - Synchronization updates like title and abstract amendmends from the RFC Editor were silently (except for RFC publication) applied and not accompanied by a descriptive event - they now are - do_replace in the Secretariat tools now adds an event - Proceedings post_process in the Secretariat tools now adds an event - do_withdraw in the Secretariat tools now adds an event A migration is needed for snapshotting all documents, takes a while to run. It turns out that a single document had a bad foreign key so the migration fixes that too. - Legacy-Id: 10101
442 lines
16 KiB
Python
442 lines
16 KiB
Python
import os
|
|
import re
|
|
import datetime
|
|
|
|
from django.conf import settings
|
|
|
|
from ietf.doc.models import Document, State, DocAlias, DocEvent, DocumentAuthor
|
|
from ietf.doc.models import NewRevisionDocEvent
|
|
from ietf.doc.models import RelatedDocument, DocRelationshipName
|
|
from ietf.doc.utils import add_state_change_event, rebuild_reference_relations
|
|
from ietf.doc.utils import set_replaces_for_document
|
|
from ietf.doc.mails import send_review_possibly_replaces_request
|
|
from ietf.group.models import Group
|
|
from ietf.ietfauth.utils import has_role
|
|
from ietf.name.models import StreamName
|
|
from ietf.person.models import Person, Email
|
|
from ietf.submit.mail import announce_to_lists, announce_new_version, announce_to_authors
|
|
from ietf.submit.models import Submission, SubmissionEvent, Preapproval, DraftSubmissionStateName
|
|
from ietf.utils import unaccent
|
|
from ietf.utils.log import log
|
|
from ietf.utils.pipe import pipe
|
|
|
|
def check_idnits(path):
|
|
#p = subprocess.Popen([self.idnits, '--submitcheck', '--nitcount', path], stdout=subprocess.PIPE)
|
|
cmd = "%s --submitcheck --nitcount %s" % (settings.IDSUBMIT_IDNITS_BINARY, path)
|
|
code, out, err = pipe(cmd)
|
|
if code != 0:
|
|
log("idnits error: %s:\n Error %s: %s" %( cmd, code, err))
|
|
return out
|
|
|
|
def found_idnits(idnits_message):
|
|
if not idnits_message:
|
|
return False
|
|
success_re = re.compile('\s+Summary:\s+0\s+|No nits found')
|
|
if success_re.search(idnits_message):
|
|
return True
|
|
return False
|
|
|
|
def validate_submission(submission):
|
|
errors = {}
|
|
|
|
if submission.state_id not in ("cancel", "posted"):
|
|
for ext in submission.file_types.split(','):
|
|
source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, submission.rev, ext))
|
|
if not os.path.exists(source):
|
|
errors['files'] = '"%s" was not found in the staging area. We recommend you that you cancel this submission and upload your files again.' % os.path.basename(source)
|
|
break
|
|
|
|
if not submission.title:
|
|
errors['title'] = 'Title is empty or was not found'
|
|
|
|
if submission.group and submission.group.state_id != "active":
|
|
errors['group'] = 'Group exists but is not an active group'
|
|
|
|
if not submission.abstract:
|
|
errors['abstract'] = 'Abstract is empty or was not found'
|
|
|
|
if not submission.authors_parsed():
|
|
errors['authors'] = 'No authors found'
|
|
|
|
# revision
|
|
if submission.state_id != "posted":
|
|
error = validate_submission_rev(submission.name, submission.rev)
|
|
if error:
|
|
errors['rev'] = error
|
|
|
|
# draft date
|
|
error = validate_submission_document_date(submission.submission_date, submission.document_date)
|
|
if error:
|
|
errors['document_date'] = error
|
|
|
|
return errors
|
|
|
|
def has_been_replaced_by(name):
|
|
docs=Document.objects.filter(name=name)
|
|
|
|
if docs:
|
|
doc=docs[0]
|
|
return doc.related_that("replaces")
|
|
|
|
return None
|
|
|
|
def validate_submission_rev(name, rev):
|
|
if not rev:
|
|
return 'Revision not found'
|
|
|
|
try:
|
|
rev = int(rev)
|
|
except ValueError:
|
|
return 'Revision must be a number'
|
|
else:
|
|
if not (0 <= rev <= 99):
|
|
return 'Revision must be between 00 and 99'
|
|
|
|
expected = 0
|
|
existing_revs = [int(i.rev) for i in Document.objects.filter(name=name)]
|
|
if existing_revs:
|
|
expected = max(existing_revs) + 1
|
|
|
|
if rev != expected:
|
|
return 'Invalid revision (revision %02d is expected)' % expected
|
|
|
|
replaced_by=has_been_replaced_by(name)
|
|
if replaced_by:
|
|
return 'This document has been replaced by %s' % ",".join(rd.name for rd in replaced_by)
|
|
|
|
return None
|
|
|
|
def validate_submission_document_date(submission_date, document_date):
|
|
if not document_date:
|
|
return 'Document date is empty or not in a proper format'
|
|
elif abs(submission_date - document_date) > datetime.timedelta(days=3):
|
|
return 'Document date must be within 3 days of submission date'
|
|
|
|
return None
|
|
|
|
def create_submission_event(request, submission, desc):
|
|
by = None
|
|
if request and request.user.is_authenticated():
|
|
try:
|
|
by = request.user.person
|
|
except Person.DoesNotExist:
|
|
pass
|
|
|
|
SubmissionEvent.objects.create(submission=submission, by=by, desc=desc)
|
|
|
|
|
|
def post_submission(request, submission):
|
|
# find out who did it
|
|
system = Person.objects.get(name="(System)")
|
|
submitter_parsed = submission.submitter_parsed()
|
|
if submitter_parsed["name"] and submitter_parsed["email"]:
|
|
submitter = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"]).person
|
|
else:
|
|
submitter = system
|
|
|
|
# update draft attributes
|
|
try:
|
|
draft = Document.objects.get(name=submission.name)
|
|
except Document.DoesNotExist:
|
|
draft = Document.objects.create(name=submission.name, type_id="draft")
|
|
|
|
prev_rev = draft.rev
|
|
|
|
draft.type_id = "draft"
|
|
draft.title = submission.title
|
|
group = submission.group or Group.objects.get(type="individ")
|
|
if not (group.type_id == "individ" and draft.group and draft.group.type_id == "area"):
|
|
# don't overwrite an assigned area if it's still an individual
|
|
# submission
|
|
draft.group_id = group.pk
|
|
draft.rev = submission.rev
|
|
draft.pages = submission.pages
|
|
draft.abstract = submission.abstract
|
|
was_rfc = draft.get_state_slug() == "rfc"
|
|
|
|
if not draft.stream:
|
|
stream_slug = None
|
|
if draft.name.startswith("draft-iab-"):
|
|
stream_slug = "iab"
|
|
elif draft.name.startswith("draft-irtf-"):
|
|
stream_slug = "irtf"
|
|
elif draft.name.startswith("draft-ietf-") and (draft.group.type_id != "individ" or was_rfc):
|
|
stream_slug = "ietf"
|
|
|
|
if stream_slug:
|
|
draft.stream = StreamName.objects.get(slug=stream_slug)
|
|
|
|
draft.expires = datetime.datetime.now() + datetime.timedelta(settings.INTERNET_DRAFT_DAYS_TO_EXPIRE)
|
|
|
|
events = []
|
|
|
|
# new revision event
|
|
e = NewRevisionDocEvent.objects.create(
|
|
type="new_revision",
|
|
doc=draft,
|
|
rev=draft.rev,
|
|
by=submitter,
|
|
desc="New version available: <b>%s-%s.txt</b>" % (draft.name, draft.rev),
|
|
)
|
|
events.append(e)
|
|
|
|
# update related objects
|
|
DocAlias.objects.get_or_create(name=submission.name, document=draft)
|
|
|
|
draft.set_state(State.objects.get(used=True, type="draft", slug="active"))
|
|
|
|
update_authors(draft, submission)
|
|
|
|
trouble = rebuild_reference_relations(draft, filename=os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (submission.name, submission.rev)))
|
|
if trouble:
|
|
log('Rebuild_reference_relations trouble: %s'%trouble)
|
|
|
|
# automatic state changes
|
|
if draft.get_state_slug("draft-iana-review") in ("ok-act", "ok-noact", "not-ok"):
|
|
prev_state = draft.get_state("draft-iana-review")
|
|
next_state = State.objects.get(used=True, type="draft-iana-review", slug="changed")
|
|
draft.set_state(next_state)
|
|
e = add_state_change_event(draft, submitter, prev_state, next_state)
|
|
if e:
|
|
events.append(e)
|
|
|
|
state_change_msg = ""
|
|
|
|
if not was_rfc and draft.tags.filter(slug="need-rev"):
|
|
draft.tags.remove("need-rev")
|
|
draft.tags.add("ad-f-up")
|
|
|
|
e = DocEvent(type="changed_document", doc=draft)
|
|
e.desc = "Sub state has been changed to <b>AD Followup</b> from <b>Revised ID Needed</b>"
|
|
e.by = system
|
|
e.save()
|
|
events.append(e)
|
|
|
|
state_change_msg = e.desc
|
|
|
|
if draft.stream_id == "ietf" and draft.group.type_id == "wg" and draft.rev == "00":
|
|
# automatically set state "WG Document"
|
|
draft.set_state(State.objects.get(used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-doc"))
|
|
|
|
# save history now that we're done with changes to the draft itself
|
|
draft.save_with_history(events)
|
|
|
|
# clean up old files
|
|
if prev_rev != draft.rev:
|
|
from ietf.doc.expire import move_draft_files_to_archive
|
|
move_draft_files_to_archive(draft, prev_rev)
|
|
|
|
move_files_to_repository(submission)
|
|
submission.state = DraftSubmissionStateName.objects.get(slug="posted")
|
|
|
|
new_replaces, new_possibly_replaces = update_replaces_from_submission(request, submission, draft)
|
|
|
|
announce_to_lists(request, submission)
|
|
announce_new_version(request, submission, draft, state_change_msg)
|
|
announce_to_authors(request, submission)
|
|
|
|
if new_possibly_replaces:
|
|
send_review_possibly_replaces_request(request, draft)
|
|
|
|
submission.save()
|
|
|
|
def update_replaces_from_submission(request, submission, draft):
|
|
if not submission.replaces:
|
|
return [], []
|
|
|
|
is_secretariat = has_role(request.user, "Secretariat")
|
|
is_chair_of = []
|
|
if request.user.is_authenticated():
|
|
is_chair_of = list(Group.objects.filter(role__person__user=request.user, role__name="chair"))
|
|
|
|
replaces = DocAlias.objects.filter(name__in=submission.replaces.split(",")).select_related("document", "document__group")
|
|
existing_replaces = list(draft.related_that_doc("replaces"))
|
|
existing_suggested = set(draft.related_that_doc("possibly-replaces"))
|
|
|
|
submitter_email = submission.submitter_parsed()["email"]
|
|
|
|
approved = []
|
|
suggested = []
|
|
for r in replaces:
|
|
if r in existing_replaces:
|
|
continue
|
|
|
|
rdoc = r.document
|
|
|
|
if rdoc == draft:
|
|
continue
|
|
|
|
# TODO - I think the .exists() is in the wrong place below....
|
|
if (is_secretariat
|
|
or (draft.group in is_chair_of and (rdoc.group.type_id == "individ" or rdoc.group in is_chair_of))
|
|
or (submitter_email and rdoc.authors.filter(address__iexact=submitter_email)).exists()):
|
|
approved.append(r)
|
|
else:
|
|
if r not in existing_suggested:
|
|
suggested.append(r)
|
|
|
|
by = request.user.person if request.user.is_authenticated() else Person.objects.get(name="(System)")
|
|
set_replaces_for_document(request, draft, existing_replaces + approved, by,
|
|
email_subject="%s replacement status set during submit by %s" % (draft.name, submission.submitter_parsed()["name"]))
|
|
|
|
|
|
if suggested:
|
|
possibly_replaces = DocRelationshipName.objects.get(slug="possibly-replaces")
|
|
for r in suggested:
|
|
RelatedDocument.objects.create(source=draft, target=r, relationship=possibly_replaces)
|
|
|
|
DocEvent.objects.create(doc=draft, by=by, type="added_suggested_replaces",
|
|
desc="Added suggested replacement relationships: %s" % ", ".join(d.name for d in suggested))
|
|
|
|
return approved, suggested
|
|
|
|
def get_person_from_name_email(name, email):
|
|
# try email
|
|
if email:
|
|
persons = Person.objects.filter(email__address=email).distinct()
|
|
if len(persons) == 1:
|
|
return persons[0]
|
|
else:
|
|
persons = Person.objects.none()
|
|
|
|
if not persons:
|
|
persons = Person.objects.all()
|
|
|
|
# try full name
|
|
p = persons.filter(alias__name=name).distinct()
|
|
if p:
|
|
return p[0]
|
|
|
|
return None
|
|
|
|
def ensure_person_email_info_exists(name, email):
|
|
person = get_person_from_name_email(name, email)
|
|
|
|
# make sure we have a person
|
|
if not person:
|
|
person = Person()
|
|
person.name = name
|
|
person.ascii = unaccent.asciify(person.name)
|
|
person.save()
|
|
|
|
# make sure we have an email address
|
|
if email:
|
|
addr = email.lower()
|
|
else:
|
|
# we're in trouble, use a fake one
|
|
addr = u"unknown-email-%s" % person.name.replace(" ", "-")
|
|
|
|
try:
|
|
email = person.email_set.get(address=addr)
|
|
except Email.DoesNotExist:
|
|
try:
|
|
# maybe it's pointing to someone else
|
|
email = Email.objects.get(address=addr)
|
|
except Email.DoesNotExist:
|
|
# most likely we just need to create it
|
|
email = Email(address=addr)
|
|
email.active = True
|
|
|
|
email.person = person
|
|
email.save()
|
|
|
|
return email
|
|
|
|
def update_authors(draft, submission):
|
|
authors = []
|
|
for order, author in enumerate(submission.authors_parsed()):
|
|
email = ensure_person_email_info_exists(author["name"], author["email"])
|
|
|
|
a = DocumentAuthor.objects.filter(document=draft, author=email)
|
|
if a:
|
|
a = a[0]
|
|
else:
|
|
a = DocumentAuthor(document=draft, author=email)
|
|
|
|
a.order = order
|
|
a.save()
|
|
|
|
authors.append(email)
|
|
|
|
draft.documentauthor_set.exclude(author__in=authors).delete()
|
|
|
|
def cancel_submission(submission):
|
|
submission.state = DraftSubmissionStateName.objects.get(slug="cancel")
|
|
submission.save()
|
|
|
|
remove_submission_files(submission)
|
|
|
|
def rename_submission_files(submission, prev_rev, new_rev):
|
|
from ietf.submit.forms import SubmissionUploadForm
|
|
for ext in SubmissionUploadForm.base_fields.keys():
|
|
source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, prev_rev, ext))
|
|
dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, new_rev, ext))
|
|
if os.path.exists(source):
|
|
os.rename(source, dest)
|
|
|
|
def move_files_to_repository(submission):
|
|
from ietf.submit.forms import SubmissionUploadForm
|
|
for ext in SubmissionUploadForm.base_fields.keys():
|
|
source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext))
|
|
dest = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext))
|
|
if os.path.exists(source):
|
|
os.rename(source, dest)
|
|
else:
|
|
if os.path.exists(dest):
|
|
log("Intended to move '%s' to '%s', but found source missing while destination exists.")
|
|
elif ext in submission.file_types.split(','):
|
|
raise ValueError("Intended to move '%s' to '%s', but found source and destination missing.")
|
|
|
|
def remove_submission_files(submission):
|
|
for ext in submission.file_types.split(','):
|
|
source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (submission.name, submission.rev, ext))
|
|
if os.path.exists(source):
|
|
os.unlink(source)
|
|
|
|
def approvable_submissions_for_user(user):
|
|
if not user.is_authenticated():
|
|
return []
|
|
|
|
res = Submission.objects.filter(state="grp-appr").order_by('-submission_date')
|
|
if has_role(user, "Secretariat"):
|
|
return res
|
|
|
|
# those we can reach as chair
|
|
return res.filter(group__role__name="chair", group__role__person__user=user)
|
|
|
|
def preapprovals_for_user(user):
|
|
if not user.is_authenticated():
|
|
return []
|
|
|
|
posted = Submission.objects.distinct().filter(state="posted").values_list('name', flat=True)
|
|
res = Preapproval.objects.exclude(name__in=posted).order_by("-time").select_related('by')
|
|
if has_role(user, "Secretariat"):
|
|
return res
|
|
|
|
acronyms = [g.acronym for g in Group.objects.filter(role__person__user=user, type__in=("wg", "rg"))]
|
|
|
|
res = res.filter(name__regex="draft-[^-]+-(%s)-.*" % "|".join(acronyms))
|
|
|
|
return res
|
|
|
|
def recently_approved_by_user(user, since):
|
|
if not user.is_authenticated():
|
|
return []
|
|
|
|
res = Submission.objects.distinct().filter(state="posted", submission_date__gte=since, rev="00").order_by('-submission_date')
|
|
if has_role(user, "Secretariat"):
|
|
return res
|
|
|
|
# those we can reach as chair
|
|
return res.filter(group__role__name="chair", group__role__person__user=user)
|
|
|
|
def expirable_submissions(older_than_days):
|
|
cutoff = datetime.date.today() - datetime.timedelta(days=older_than_days)
|
|
return Submission.objects.exclude(state__in=("cancel", "posted")).filter(submission_date__lt=cutoff)
|
|
|
|
def expire_submission(submission, by):
|
|
submission.state_id = "cancel"
|
|
submission.save()
|
|
|
|
SubmissionEvent.objects.create(submission=submission, by=by, desc="Canceled expired submission")
|