datatracker/ietf/idrfc/views_search.py
Henrik Levkowetz 38b0a59ba3 Merged personal/rjs/v4.70@6042:
* Made reporting IPR consistant across document searches, IPR searches, AD document queues, last calls, and agendas. All these points now report on the document(s) being directly queried, and the documents those transitively replace or obsolete.

  * Brought search results against WGs and document titles into consistency with results from searching for individual documents

  * Added the IPR count to the link on the documents main page (when not zero)

  * Built on Henrik's reimplementation of all_related_*, making the *_related_* functions return DocAlias? lists consistently, and added corresponding _relations_ functions to get lists of actual RelatedDocument? objects.

  * Added getting the DocAlias? with the same name to Document

  * Added getting related IPR disclosures (as described in the first bullet) to Document

  * Simplified ipr/related.py

  * Removed the use of DraftLikeDocAlias? and IETFWG from ipr/search.py. Retooled the various search functions and templates to use DocAlias? and IprDocAlias? directly.

  * Removed dead code from ipr/search.py

  * Removed the special handling of WG 2000 from ipr/search.py 

  This fixes bug #1071
 - Legacy-Id: 6083
2013-09-02 18:34:54 +00:00

623 lines
26 KiB
Python

# Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.
#
# * Neither the name of the Nokia Corporation and/or its
# subsidiary(-ies) nor the names of its contributors may be used
# to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import re, datetime
from django import forms
from django.shortcuts import render_to_response
from django.db.models import Q
from django.template import RequestContext
from django.http import Http404, HttpResponse, HttpResponseBadRequest, HttpResponseRedirect
from ietf.idrfc.expire import expirable_draft
from ietf.utils import normalize_draftname
from ietf.doc.models import *
from ietf.person.models import *
from ietf.group.models import *
from ietf.ipr.models import IprDocAlias
from ietf.ipr.search import related_docs
from ietf.idindex.index import active_drafts_index_by_group
class SearchForm(forms.Form):
name = forms.CharField(required=False)
rfcs = forms.BooleanField(required=False, initial=True)
activedrafts = forms.BooleanField(required=False, initial=True)
olddrafts = forms.BooleanField(required=False, initial=False)
by = forms.ChoiceField(choices=[(x,x) for x in ('author','group','area','ad','state')], required=False, initial='wg', label='Foobar')
author = forms.CharField(required=False)
group = forms.CharField(required=False)
area = forms.ModelChoiceField(Group.objects.filter(type="area", state="active").order_by('name'), empty_label="any area", required=False)
ad = forms.ChoiceField(choices=(), required=False)
state = forms.ModelChoiceField(State.objects.filter(type="draft-iesg"), empty_label="any state", required=False)
substate = forms.ChoiceField(choices=(), required=False)
sort = forms.ChoiceField(choices=(("document", "Document"), ("title", "Title"), ("date", "Date"), ("status", "Status"), ("ipr", "Ipr"), ("ad", "AD")), required=False, widget=forms.HiddenInput)
doctypes = DocTypeName.objects.exclude(slug='draft').order_by('name');
def __init__(self, *args, **kwargs):
super(SearchForm, self).__init__(*args, **kwargs)
responsible = Document.objects.values_list('ad', flat=True).distinct()
active_ads = list(Person.objects.filter(role__name="ad",
role__group__type="area",
role__group__state="active").distinct())
inactive_ads = list(((Person.objects.filter(pk__in=responsible) | Person.objects.filter(role__name="pre-ad",
role__group__type="area",
role__group__state="active")).distinct())
.exclude(pk__in=[x.pk for x in active_ads]))
extract_last_name = lambda x: x.name_parts()[3]
active_ads.sort(key=extract_last_name)
inactive_ads.sort(key=extract_last_name)
self.fields['ad'].choices = c = [('', 'any AD')] + [(ad.pk, ad.plain_name()) for ad in active_ads] + [('', '------------------')] + [(ad.pk, ad.name) for ad in inactive_ads]
self.fields['substate'].choices = [('', 'any substate'), ('0', 'no substate')] + [(n.slug, n.name) for n in DocTagName.objects.filter(slug__in=IESG_SUBSTATE_TAGS)]
def clean_name(self):
value = self.cleaned_data.get('name','')
return normalize_draftname(value)
def clean(self):
q = self.cleaned_data
# Reset query['by'] if needed
if 'by' in q:
for k in ('author', 'group', 'area', 'ad'):
if q['by'] == k and not q.get(k):
q['by'] = None
if q['by'] == 'state' and not (q.get("state") or q.get('substate')):
q['by'] = None
else:
q['by'] = None
# Reset other fields
for k in ('author','group', 'area', 'ad'):
if k != q['by']:
q[k] = ""
if q['by'] != 'state':
q['state'] = q['substate'] = None
return q
def wrap_value(v):
return lambda: v
def fill_in_search_attributes(docs):
# fill in some attributes for the search results to save some
# hairy template code and avoid repeated SQL queries - remaining
# queries we don't handle here are mostly implicit many-to-many
# relations for which there is poor support in Django 1.2
docs_dict = dict((d.pk, d) for d in docs)
doc_ids = docs_dict.keys()
rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc", document__in=doc_ids).values_list("document_id", "name"))
# latest event cache
event_types = ("published_rfc",
"changed_ballot_position",
"started_iesg_process",
"new_revision")
for d in docs:
d.latest_event_cache = dict()
for e in event_types:
d.latest_event_cache[e] = None
for e in DocEvent.objects.filter(doc__in=doc_ids, type__in=event_types).order_by('time'):
docs_dict[e.doc_id].latest_event_cache[e.type] = e
# IPR
for d in docs:
d.iprs = []
# Consider reworking the following block using all_relations_that_doc? That might simplify the final assembly
# down to the code at "if a not in docs_dict"...
rel_docs = []
rel_id_camefrom = {}
for d in docs:
if isinstance(d,DocAlias):
d = d.document
rel_this_doc = d.all_related_that_doc(['replaces','obs'])
for rel in rel_this_doc:
rel_id_camefrom.setdefault(rel.document.pk,[]).append(d.pk)
rel_docs += [x.document for x in rel_this_doc]
ipr_docaliases = IprDocAlias.objects.filter(doc_alias__document__in=doc_ids).select_related('doc_alias')
for a in ipr_docaliases:
docs_dict[a.doc_alias.document_id].iprs.append(a)
rel_docs_dict = dict((d.pk, d) for d in rel_docs)
rel_doc_ids = rel_docs_dict.keys()
rel_ipr_docaliases = IprDocAlias.objects.filter(doc_alias__document__in=rel_doc_ids).select_related('doc_alias')
for a in rel_ipr_docaliases:
if a.doc_alias.document_id in rel_id_camefrom:
for k in rel_id_camefrom[a.doc_alias.document_id]:
if a not in docs_dict[k].iprs:
docs_dict[k].iprs.append(a)
# telechat date, can't do this with above query as we need to get TelechatDocEvents out
seen = set()
for e in TelechatDocEvent.objects.filter(doc__in=doc_ids, type="scheduled_for_telechat").order_by('-time'):
if e.doc_id not in seen:
d = docs_dict[e.doc_id]
d.telechat_date = wrap_value(d.telechat_date(e))
seen.add(e.doc_id)
# misc
for d in docs:
# emulate canonical name which is used by a lot of the utils
d.canonical_name = wrap_value(rfc_aliases[d.pk] if d.pk in rfc_aliases else d.name)
if d.rfc_number() != None and d.latest_event_cache["published_rfc"]:
d.latest_revision_date = d.latest_event_cache["published_rfc"].time
elif d.latest_event_cache["new_revision"]:
d.latest_revision_date = d.latest_event_cache["new_revision"].time
else:
d.latest_revision_date = d.time
if d.type_id == "draft":
if d.get_state_slug() == "rfc":
d.search_heading = "RFC"
elif d.get_state_slug() in ("ietf-rm", "auth-rm"):
d.search_heading = "Withdrawn Internet-Draft"
else:
d.search_heading = "%s Internet-Draft" % d.get_state()
else:
d.search_heading = "%s" % (d.type,);
d.expirable = expirable_draft(d)
if d.get_state_slug() != "rfc":
d.milestones = d.groupmilestone_set.filter(state="active").order_by("time").select_related("group")
# RFCs
# errata
erratas = set(Document.objects.filter(tags="errata", name__in=rfc_aliases.keys()).distinct().values_list("name", flat=True))
for d in docs:
d.has_errata = d.name in erratas
# obsoleted/updated by
for a in rfc_aliases:
d = docs_dict[a]
d.obsoleted_by_list = []
d.updated_by_list = []
xed_by = RelatedDocument.objects.filter(target__name__in=rfc_aliases.values(),
relationship__in=("obs", "updates")).select_related('target__document_id')
rel_rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc",
document__in=[rel.source_id for rel in xed_by]).values_list('document_id', 'name'))
for rel in xed_by:
d = docs_dict[rel.target.document_id]
if rel.relationship_id == "obs":
l = d.obsoleted_by_list
elif rel.relationship_id == "updates":
l = d.updated_by_list
l.append(rel_rfc_aliases[rel.source_id].upper())
l.sort()
def retrieve_search_results(form, all_types=False):
"""Takes a validated SearchForm and return the results."""
if not form.is_valid():
raise ValueError("SearchForm doesn't validate: %s" % form.errors)
query = form.cleaned_data
types=[];
meta = {}
if (query['activedrafts'] or query['olddrafts'] or query['rfcs']):
types.append('draft')
# Advanced document types are data-driven, so we need to read them from the
# raw form.data field (and track their checked/unchecked state ourselves)
meta['checked'] = {}
alltypes = DocTypeName.objects.exclude(slug='draft').order_by('name');
for doctype in alltypes:
if form.data.__contains__('include-' + doctype.slug):
types.append(doctype.slug)
meta['checked'][doctype.slug] = True
if len(types) == 0 and not all_types:
return ([], {})
MAX = 500
if all_types:
docs = Document.objects.all()
else:
docs = Document.objects.filter(type__in=types)
# name
if query["name"]:
docs = docs.filter(Q(docalias__name__icontains=query["name"]) |
Q(title__icontains=query["name"])).distinct()
# rfc/active/old check buttons
allowed_draft_states = []
if query["rfcs"]:
allowed_draft_states.append("rfc")
if query["activedrafts"]:
allowed_draft_states.append("active")
if query["olddrafts"]:
allowed_draft_states.extend(['repl', 'expired', 'auth-rm', 'ietf-rm'])
docs = docs.filter(Q(states__slug__in=allowed_draft_states) |
~Q(type__slug='draft')).distinct()
# radio choices
by = query["by"]
if by == "author":
docs = docs.filter(authors__person__name__icontains=query["author"])
elif by == "group":
docs = docs.filter(group__acronym=query["group"])
elif by == "area":
docs = docs.filter(Q(group__type="wg", group__parent=query["area"]) |
Q(group=query["area"])).distinct()
elif by == "ad":
docs = docs.filter(ad=query["ad"])
elif by == "state":
if query["state"]:
docs = docs.filter(states=query["state"])
if query["substate"]:
docs = docs.filter(tags=query["substate"])
# evaluate and fill in attribute results immediately to cut down
# the number of queries
results = list(docs.select_related("states", "ad", "ad__person", "std_level", "intended_std_level", "group", "stream")[:MAX])
fill_in_search_attributes(results)
# sort
def sort_key(d):
res = []
rfc_num = d.rfc_number()
if d.type_id == "draft":
res.append(["Active", "Expired", "Replaced", "Withdrawn", "RFC"].index(d.search_heading.split()[0] ))
else:
res.append(d.type_id);
res.append("-");
res.append(d.get_state_slug());
res.append("-");
if query["sort"] == "title":
res.append(d.title)
elif query["sort"] == "date":
res.append(str(d.latest_revision_date))
elif query["sort"] == "status":
if rfc_num != None:
res.append(int(rfc_num))
else:
res.append(d.get_state().order if d.get_state() else None)
elif query["sort"] == "ipr":
res.append(len(d.iprs))
elif query["sort"] == "ad":
if rfc_num != None:
res.append(int(rfc_num))
elif d.get_state_slug() == "active":
if d.get_state("draft-iesg"):
res.append(d.get_state("draft-iesg").order)
else:
res.append(0)
else:
if rfc_num != None:
res.append(int(rfc_num))
else:
res.append(d.canonical_name())
return res
results.sort(key=sort_key)
# fill in a meta dict with some information for rendering the result table
if len(results) == MAX:
meta['max'] = MAX
meta['by'] = query['by']
meta['advanced'] = bool(query['by'] or len(meta['checked']))
meta['headers'] = [{'title': 'Document', 'key':'document'},
{'title': 'Title', 'key':'title'},
{'title': 'Date', 'key':'date'},
{'title': 'Status', 'key':'status', 'colspan':'2'},
{'title': 'IPR', 'key':'ipr'},
{'title': 'AD / Shepherd', 'key':'ad'}]
if hasattr(form.data, "urlencode"): # form was fed a Django QueryDict, not local plain dict
d = form.data.copy()
for h in meta['headers']:
d["sort"] = h["key"]
h["sort_url"] = "?" + d.urlencode()
if h['key'] == query.get('sort'):
h['sorted'] = True
return (results, meta)
def search(request):
if request.GET:
# backwards compatibility
get_params = request.GET.copy()
if 'activeDrafts' in request.GET:
get_params['activedrafts'] = request.GET['activeDrafts']
if 'oldDrafts' in request.GET:
get_params['olddrafts'] = request.GET['oldDrafts']
if 'subState' in request.GET:
get_params['substate'] = request.GET['subState']
form = SearchForm(get_params)
if not form.is_valid():
return HttpResponseBadRequest("form not valid: %s" % form.errors)
results, meta = retrieve_search_results(form)
meta['searching'] = True
else:
form = SearchForm()
results = []
meta = { 'by': None, 'advanced': False, 'searching': False }
return render_to_response('doc/search.html',
{'form':form, 'docs':results, 'meta':meta, 'show_add_to_list': True },
context_instance=RequestContext(request))
def ad_dashboard_group(doc):
if doc.type.slug=='draft':
if doc.get_state_slug('draft') == 'rfc':
return 'RFC'
elif doc.get_state_slug('draft') == 'active' and doc.get_state_slug('draft-iesg'):
return '%s Internet-Draft' % doc.get_state('draft-iesg').name
else:
return '%s Internet-Draft' % doc.get_state('draft').name
elif doc.type.slug=='conflrev':
if doc.get_state_slug('conflrev') in ('appr-reqnopub-sent','appr-noprob-sent'):
return 'Approved Conflict Review'
elif doc.get_state_slug('conflrev') in ('appr-reqnopub-pend','appr-noprob-pend','appr-reqnopub-pr','appr-noprob-pr'):
return "%s Conflict Review" % State.objects.get(type__slug='draft-iesg',slug='approved')
else:
return '%s Conflict Review' % doc.get_state('conflrev')
elif doc.type.slug=='statchg':
if doc.get_state_slug('statchg') in ('appr-sent',):
return 'Approved Status Change'
if doc.get_state_slug('statchg') in ('appr-pend','appr-pr'):
return '%s Status Change' % State.objects.get(type__slug='draft-iesg',slug='approved')
else:
return '%s Status Change' % doc.get_state('statchg')
elif doc.type.slug=='charter':
if doc.get_state_slug('charter') == 'approved':
return "Approved Charter"
else:
return '%s Charter' % doc.get_state('charter')
else:
return "Document"
def ad_dashboard_sort_key(doc):
if doc.type.slug=='draft' and doc.get_state_slug('draft') == 'rfc':
return "21%04d" % int(doc.rfc_number())
if doc.type.slug=='statchg' and doc.get_state_slug('statchg') == 'appr-sent':
return "22%d" % 0 # TODO - get the date of the transition into this state here
if doc.type.slug=='conflrev' and doc.get_state_slug('conflrev') in ('appr-reqnopub-sent','appr-noprob-sent'):
return "23%d" % 0 # TODO - get the date of the transition into this state here
if doc.type.slug=='charter' and doc.get_state_slug('charter') == 'approved':
return "24%d" % 0 # TODO - get the date of the transition into this state here
seed = ad_dashboard_group(doc)
if doc.type.slug=='conflrev' and doc.get_state_slug('conflrev') == 'adrev':
state = State.objects.get(type__slug='draft-iesg',slug='ad-eval')
return "1%d%s" % (state.order,seed)
if doc.type.slug=='charter':
if doc.get_state_slug('charter') in ('notrev','infrev'):
return "100%s" % seed
elif doc.get_state_slug('charter') == 'intrev':
state = State.objects.get(type__slug='draft-iesg',slug='ad-eval')
return "1%d%s" % (state.order,seed)
elif doc.get_state_slug('charter') == 'extrev':
state = State.objects.get(type__slug='draft-iesg',slug='lc')
return "1%d%s" % (state.order,seed)
elif doc.get_state_slug('charter') == 'iesgrev':
state = State.objects.get(type__slug='draft-iesg',slug='iesg-eva')
return "1%d%s" % (state.order,seed)
if doc.type.slug=='statchg' and doc.get_state_slug('statchg') == 'adrev':
state = State.objects.get(type__slug='draft-iesg',slug='ad-eval')
return "1%d%s" % (state.order,seed)
if seed.startswith('Needs Shepherd'):
return "100%s" % seed
if seed.endswith(' Document'):
seed = seed[:-9]
elif seed.endswith(' Internet-Draft'):
seed = seed[:-15]
elif seed.endswith(' Conflict Review'):
seed = seed[:-16]
elif seed.endswith(' Status Change'):
seed = seed[:-14]
state = State.objects.filter(type__slug='draft-iesg',name=seed)
if state:
ageseconds = 0
changetime= doc.latest_event(type='changed_document')
if changetime:
ad = (datetime.datetime.now()-doc.latest_event(type='changed_document').time)
ageseconds = (ad.microseconds + (ad.seconds + ad.days * 24 * 3600) * 10**6) / 10**6
return "1%d%s%s%010d" % (state[0].order,seed,doc.type.slug,ageseconds)
return "3%s" % seed
def drafts_for_ad(request, name):
ad = None
responsible = Document.objects.values_list('ad', flat=True).distinct()
for p in Person.objects.filter(Q(role__name__in=("pre-ad", "ad"),
role__group__type="area",
role__group__state="active")
| Q(pk__in=responsible)).distinct():
if name == p.full_name_as_key():
ad = p
break
if not ad:
raise Http404
form = SearchForm({'by':'ad','ad': ad.id,
'rfcs':'on', 'activedrafts':'on', 'olddrafts':'on',
'sort': 'status'})
results, meta = retrieve_search_results(form)
del meta["headers"][-1]
#
for d in results:
if d.get_state_slug() == "active":
iesg_state = d.get_state("draft-iesg")
if iesg_state:
if iesg_state.slug == "dead":
d.search_heading = "IESG Dead Internet-Drafts"
else:
d.search_heading = "%s Internet-Drafts" % iesg_state.name
return render_to_response('doc/drafts_for_ad.html',
{ 'form':form, 'docs':results, 'meta':meta, 'ad_name': ad.plain_name() },
context_instance=RequestContext(request))
def docs_for_ad(request, name):
ad = None
responsible = Document.objects.values_list('ad', flat=True).distinct()
for p in Person.objects.filter(Q(role__name__in=("pre-ad", "ad"),
role__group__type="area",
role__group__state="active")
| Q(pk__in=responsible)).distinct():
if name == p.full_name_as_key():
ad = p
break
if not ad:
raise Http404
form = SearchForm({'by':'ad','ad': ad.id,
'rfcs':'on', 'activedrafts':'on', 'olddrafts':'on',
'sort': 'status'})
results, meta = retrieve_search_results(form, all_types=True)
results.sort(key=ad_dashboard_sort_key)
del meta["headers"][-1]
#
for d in results:
d.search_heading = ad_dashboard_group(d)
#
return render_to_response('doc/drafts_for_ad.html',
{ 'form':form, 'docs':results, 'meta':meta, 'ad_name': ad.plain_name() },
context_instance=RequestContext(request))
def drafts_in_last_call(request):
lc_state = State.objects.get(type="draft-iesg", slug="lc").pk
form = SearchForm({'by':'state','state': lc_state, 'rfcs':'on', 'activedrafts':'on'})
results, meta = retrieve_search_results(form)
return render_to_response('doc/drafts_in_last_call.html',
{ 'form':form, 'docs':results, 'meta':meta },
context_instance=RequestContext(request))
def drafts_in_iesg_process(request, last_call_only=None):
if last_call_only:
states = State.objects.filter(type="draft-iesg", slug__in=("lc", "writeupw", "goaheadw"))
title = "Documents in Last Call"
else:
states = State.objects.filter(type="draft-iesg").exclude(slug__in=('pub', 'dead', 'watching', 'rfcqueue'))
title = "Documents in IESG process"
grouped_docs = []
for s in states.order_by("order"):
docs = Document.objects.filter(type="draft", states=s).distinct().order_by("time").select_related("ad", "group", "group__parent")
if docs:
if s.slug == "lc":
for d in docs:
e = d.latest_event(LastCallDocEvent, type="sent_last_call")
d.lc_expires = e.expires if e else datetime.datetime.min
docs = list(docs)
docs.sort(key=lambda d: d.lc_expires)
grouped_docs.append((s, docs))
return render_to_response('doc/drafts_in_iesg_process.html', {
"grouped_docs": grouped_docs,
"title": title,
"last_call_only": last_call_only,
}, context_instance=RequestContext(request))
def index_all_drafts(request):
# try to be efficient since this view returns a lot of data
categories = []
for s in ("active", "rfc", "expired", "repl", "auth-rm", "ietf-rm"):
state = State.objects.get(type="draft", slug=s)
if state.slug == "rfc":
heading = "RFCs"
elif state.slug in ("ietf-rm", "auth-rm"):
heading = "Internet-Drafts %s" % state.name
else:
heading = "%s Internet-Drafts" % state.name
draft_names = DocAlias.objects.filter(document__states=state).values_list("name", "document")
names = []
names_to_skip = set()
for name, doc in draft_names:
sort_key = name
if name != doc:
if not name.startswith("rfc"):
name, doc = doc, name
names_to_skip.add(doc)
if name.startswith("rfc"):
name = name.upper()
sort_key = -int(name[3:])
names.append((name, sort_key))
names.sort(key=lambda t: t[1])
names = ['<a href="/doc/' + name + '/">' + name +'</a>'
for name, _ in names if name not in names_to_skip]
categories.append((state,
heading,
len(names),
"<br>".join(names)
))
return render_to_response('doc/index_all_drafts.html', { "categories": categories },
context_instance=RequestContext(request))
def index_active_drafts(request):
groups = active_drafts_index_by_group()
return render_to_response("doc/index_active_drafts.html", { 'groups': groups }, context_instance=RequestContext(request))