Refactor search-related utilities slightly to make them more

independent of the search form, to enable them to be reused in the
community lists.

Also clean up the related code a bit, use SearchForm to handle other
document types, and fix missing ids in the HTML so that one can click
the "by" labels to hit the corresponding radio button.
 - Legacy-Id: 10729
This commit is contained in:
Ole Laursen 2016-01-25 18:11:49 +00:00
parent d67a96b4f0
commit 197bc07771
6 changed files with 225 additions and 218 deletions

180
ietf/doc/utils_search.py Normal file
View file

@ -0,0 +1,180 @@
from ietf.doc.models import Document, DocAlias, RelatedDocument, DocEvent, TelechatDocEvent
from ietf.doc.expire import expirable_draft
from ietf.community.utils import augment_docs_with_tracking_info
def wrap_value(v):
return lambda: v
def fill_in_document_table_attributes(docs):
# fill in some attributes for the document table results to save
# some hairy template code and avoid repeated SQL queries
docs_dict = dict((d.pk, d) for d in docs)
doc_ids = docs_dict.keys()
rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc", document__in=doc_ids).values_list("document_id", "name"))
# latest event cache
event_types = ("published_rfc",
"changed_ballot_position",
"started_iesg_process",
"new_revision")
for d in docs:
d.latest_event_cache = dict()
for e in event_types:
d.latest_event_cache[e] = None
for e in DocEvent.objects.filter(doc__in=doc_ids, type__in=event_types).order_by('time'):
docs_dict[e.doc_id].latest_event_cache[e.type] = e
# telechat date, can't do this with above query as we need to get TelechatDocEvents out
seen = set()
for e in TelechatDocEvent.objects.filter(doc__in=doc_ids, type="scheduled_for_telechat").order_by('-time'):
if e.doc_id not in seen:
d = docs_dict[e.doc_id]
d.telechat_date = wrap_value(d.telechat_date(e))
seen.add(e.doc_id)
# misc
for d in docs:
# emulate canonical name which is used by a lot of the utils
d.canonical_name = wrap_value(rfc_aliases[d.pk] if d.pk in rfc_aliases else d.name)
if d.rfc_number() != None and d.latest_event_cache["published_rfc"]:
d.latest_revision_date = d.latest_event_cache["published_rfc"].time
elif d.latest_event_cache["new_revision"]:
d.latest_revision_date = d.latest_event_cache["new_revision"].time
else:
d.latest_revision_date = d.time
if d.type_id == "draft":
if d.get_state_slug() == "rfc":
d.search_heading = "RFC"
elif d.get_state_slug() in ("ietf-rm", "auth-rm"):
d.search_heading = "Withdrawn Internet-Draft"
else:
d.search_heading = "%s Internet-Draft" % d.get_state()
else:
d.search_heading = "%s" % (d.type,);
d.expirable = expirable_draft(d)
if d.get_state_slug() != "rfc":
d.milestones = d.groupmilestone_set.filter(state="active").order_by("time").select_related("group")
# RFCs
# errata
erratas = set(Document.objects.filter(tags="errata", name__in=rfc_aliases.keys()).distinct().values_list("name", flat=True))
for d in docs:
d.has_errata = d.name in erratas
# obsoleted/updated by
for a in rfc_aliases:
d = docs_dict[a]
d.obsoleted_by_list = []
d.updated_by_list = []
xed_by = RelatedDocument.objects.filter(target__name__in=rfc_aliases.values(),
relationship__in=("obs", "updates")).select_related('target__document_id')
rel_rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc",
document__in=[rel.source_id for rel in xed_by]).values_list('document_id', 'name'))
for rel in xed_by:
d = docs_dict[rel.target.document_id]
if rel.relationship_id == "obs":
l = d.obsoleted_by_list
elif rel.relationship_id == "updates":
l = d.updated_by_list
l.append(rel_rfc_aliases[rel.source_id].upper())
l.sort()
def prepare_document_table(request, docs, query=None, max_results=500):
"""Take a queryset of documents and a QueryDict with sorting info
and return list of documents with attributes filled in for
displaying a full table of information about the documents, plus
dict with information about the columns."""
if not isinstance(docs, list):
# evaluate and fill in attribute results immediately to decrease
# the number of queries
docs = docs.select_related("ad", "ad__person", "std_level", "intended_std_level", "group", "stream")
docs = docs.prefetch_related("states__type", "tags")
docs = list(docs[:max_results])
fill_in_document_table_attributes(docs)
augment_docs_with_tracking_info(docs, request.user)
meta = {}
sort_key = query and query.get('sort')
# sort
def sort_key(d):
res = []
rfc_num = d.rfc_number()
if d.type_id == "draft":
res.append(["Active", "Expired", "Replaced", "Withdrawn", "RFC"].index(d.search_heading.split()[0]))
else:
res.append(d.type_id);
res.append("-");
res.append(d.get_state_slug());
res.append("-");
if sort_key == "title":
res.append(d.title)
elif sort_key == "date":
res.append(str(d.latest_revision_date))
elif sort_key == "status":
if rfc_num != None:
res.append(int(rfc_num))
else:
res.append(d.get_state().order if d.get_state() else None)
elif sort_key == "ipr":
res.append(len(d.ipr()))
elif sort_key == "ad":
if rfc_num != None:
res.append(int(rfc_num))
elif d.get_state_slug() == "active":
if d.get_state("draft-iesg"):
res.append(d.get_state("draft-iesg").order)
else:
res.append(0)
else:
if rfc_num != None:
res.append(int(rfc_num))
else:
res.append(d.canonical_name())
return res
docs.sort(key=sort_key)
# fill in a meta dict with some information for rendering the table
if len(docs) == max_results:
meta['max'] = max_results
meta['headers'] = [{'title': 'Document', 'key':'document'},
{'title': 'Title', 'key':'title'},
{'title': 'Date', 'key':'date'},
{'title': 'Status', 'key':'status'},
{'title': 'IPR', 'key':'ipr'},
{'title': 'AD / Shepherd', 'key':'ad'}]
if query and hasattr(query, "urlencode"): # fed a Django QueryDict
d = query.copy()
for h in meta['headers']:
d["sort"] = h["key"]
h["sort_url"] = "?" + d.urlencode()
if h['key'] == sort_key:
h['sorted'] = True
return (docs, meta)

View file

@ -40,16 +40,15 @@ from django.http import Http404, HttpResponseBadRequest, HttpResponse, HttpRespo
import debug # pyflakes:ignore
from ietf.doc.models import ( Document, DocHistory, DocAlias, State, RelatedDocument,
DocEvent, LastCallDocEvent, TelechatDocEvent, IESG_SUBSTATE_TAGS )
from ietf.doc.expire import expirable_draft
from ietf.doc.models import ( Document, DocHistory, DocAlias, State,
LastCallDocEvent, IESG_SUBSTATE_TAGS )
from ietf.doc.fields import select2_id_doc_name_json
from ietf.group.models import Group
from ietf.idindex.index import active_drafts_index_by_group
from ietf.name.models import DocTagName, DocTypeName, StreamName
from ietf.person.models import Person
from ietf.utils.draft_search import normalize_draftname
from ietf.community.utils import augment_docs_with_tracking_info
from ietf.doc.utils_search import prepare_document_table
class SearchForm(forms.Form):
@ -58,7 +57,7 @@ class SearchForm(forms.Form):
activedrafts = forms.BooleanField(required=False, initial=True)
olddrafts = forms.BooleanField(required=False, initial=False)
by = forms.ChoiceField(choices=[(x,x) for x in ('author','group','area','ad','state','stream')], required=False, initial='wg')
by = forms.ChoiceField(choices=[(x,x) for x in ('author','group','area','ad','state','stream')], required=False, initial='group')
author = forms.CharField(required=False)
group = forms.CharField(required=False)
stream = forms.ModelChoiceField(StreamName.objects.all().order_by('name'), empty_label="any stream", required=False)
@ -69,7 +68,7 @@ class SearchForm(forms.Form):
sort = forms.ChoiceField(choices=(("document", "Document"), ("title", "Title"), ("date", "Date"), ("status", "Status"), ("ipr", "Ipr"), ("ad", "AD")), required=False, widget=forms.HiddenInput)
doctypes = DocTypeName.objects.exclude(slug='draft').order_by('name');
doctypes = forms.ModelMultipleChoiceField(queryset=DocTypeName.objects.exclude(slug='draft').order_by('name'), required=False)
def __init__(self, *args, **kwargs):
super(SearchForm, self).__init__(*args, **kwargs)
@ -111,96 +110,7 @@ class SearchForm(forms.Form):
q['state'] = q['substate'] = None
return q
def wrap_value(v):
return lambda: v
def fill_in_search_attributes(docs):
# fill in some attributes for the search results to save some
# hairy template code and avoid repeated SQL queries
docs_dict = dict((d.pk, d) for d in docs)
doc_ids = docs_dict.keys()
rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc", document__in=doc_ids).values_list("document_id", "name"))
# latest event cache
event_types = ("published_rfc",
"changed_ballot_position",
"started_iesg_process",
"new_revision")
for d in docs:
d.latest_event_cache = dict()
for e in event_types:
d.latest_event_cache[e] = None
for e in DocEvent.objects.filter(doc__in=doc_ids, type__in=event_types).order_by('time'):
docs_dict[e.doc_id].latest_event_cache[e.type] = e
# telechat date, can't do this with above query as we need to get TelechatDocEvents out
seen = set()
for e in TelechatDocEvent.objects.filter(doc__in=doc_ids, type="scheduled_for_telechat").order_by('-time'):
if e.doc_id not in seen:
d = docs_dict[e.doc_id]
d.telechat_date = wrap_value(d.telechat_date(e))
seen.add(e.doc_id)
# misc
for d in docs:
# emulate canonical name which is used by a lot of the utils
d.canonical_name = wrap_value(rfc_aliases[d.pk] if d.pk in rfc_aliases else d.name)
if d.rfc_number() != None and d.latest_event_cache["published_rfc"]:
d.latest_revision_date = d.latest_event_cache["published_rfc"].time
elif d.latest_event_cache["new_revision"]:
d.latest_revision_date = d.latest_event_cache["new_revision"].time
else:
d.latest_revision_date = d.time
if d.type_id == "draft":
if d.get_state_slug() == "rfc":
d.search_heading = "RFC"
elif d.get_state_slug() in ("ietf-rm", "auth-rm"):
d.search_heading = "Withdrawn Internet-Draft"
else:
d.search_heading = "%s Internet-Draft" % d.get_state()
else:
d.search_heading = "%s" % (d.type,);
d.expirable = expirable_draft(d)
if d.get_state_slug() != "rfc":
d.milestones = d.groupmilestone_set.filter(state="active").order_by("time").select_related("group")
# RFCs
# errata
erratas = set(Document.objects.filter(tags="errata", name__in=rfc_aliases.keys()).distinct().values_list("name", flat=True))
for d in docs:
d.has_errata = d.name in erratas
# obsoleted/updated by
for a in rfc_aliases:
d = docs_dict[a]
d.obsoleted_by_list = []
d.updated_by_list = []
xed_by = RelatedDocument.objects.filter(target__name__in=rfc_aliases.values(),
relationship__in=("obs", "updates")).select_related('target__document_id')
rel_rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc",
document__in=[rel.source_id for rel in xed_by]).values_list('document_id', 'name'))
for rel in xed_by:
d = docs_dict[rel.target.document_id]
if rel.relationship_id == "obs":
l = d.obsoleted_by_list
elif rel.relationship_id == "updates":
l = d.updated_by_list
l.append(rel_rfc_aliases[rel.source_id].upper())
l.sort()
def retrieve_search_results(form, request, all_types=False):
def retrieve_search_results(form, all_types=False):
"""Takes a validated SearchForm and return the results."""
if not form.is_valid():
@ -208,29 +118,19 @@ def retrieve_search_results(form, request, all_types=False):
query = form.cleaned_data
types=[];
meta = {}
if (query['activedrafts'] or query['olddrafts'] or query['rfcs']):
types.append('draft')
# Advanced document types are data-driven, so we need to read them from the
# raw form.data field (and track their checked/unchecked state ourselves)
meta['checked'] = {}
alltypes = DocTypeName.objects.exclude(slug='draft').order_by('name');
for doctype in alltypes:
if form.data.__contains__('include-' + doctype.slug):
types.append(doctype.slug)
meta['checked'][doctype.slug] = True
if len(types) == 0 and not all_types:
return ([], {})
MAX = 500
if all_types:
docs = Document.objects.all()
else:
types = []
if query['activedrafts'] or query['olddrafts'] or query['rfcs']:
types.append('draft')
types.extend(query["doctypes"])
if not types:
return []
docs = Document.objects.filter(type__in=types)
# name
@ -269,82 +169,7 @@ def retrieve_search_results(form, request, all_types=False):
elif by == "stream":
docs = docs.filter(stream=query["stream"])
# evaluate and fill in attribute results immediately to cut down
# the number of queries
docs = docs.select_related("ad", "ad__person", "std_level", "intended_std_level", "group", "stream")
docs = docs.prefetch_related("states__type", "tags")
results = list(docs[:MAX])
fill_in_search_attributes(results)
# sort
def sort_key(d):
res = []
rfc_num = d.rfc_number()
if d.type_id == "draft":
res.append(["Active", "Expired", "Replaced", "Withdrawn", "RFC"].index(d.search_heading.split()[0] ))
else:
res.append(d.type_id);
res.append("-");
res.append(d.get_state_slug());
res.append("-");
if query["sort"] == "title":
res.append(d.title)
elif query["sort"] == "date":
res.append(str(d.latest_revision_date))
elif query["sort"] == "status":
if rfc_num != None:
res.append(int(rfc_num))
else:
res.append(d.get_state().order if d.get_state() else None)
elif query["sort"] == "ipr":
res.append(len(d.ipr()))
elif query["sort"] == "ad":
if rfc_num != None:
res.append(int(rfc_num))
elif d.get_state_slug() == "active":
if d.get_state("draft-iesg"):
res.append(d.get_state("draft-iesg").order)
else:
res.append(0)
else:
if rfc_num != None:
res.append(int(rfc_num))
else:
res.append(d.canonical_name())
return res
results.sort(key=sort_key)
augment_docs_with_tracking_info(results, request.user)
# fill in a meta dict with some information for rendering the result table
if len(results) == MAX:
meta['max'] = MAX
meta['by'] = query['by']
meta['advanced'] = bool(query['by'] or len(meta['checked']))
meta['headers'] = [{'title': 'Document', 'key':'document'},
{'title': 'Title', 'key':'title'},
{'title': 'Date', 'key':'date'},
{'title': 'Status', 'key':'status'},
{'title': 'IPR', 'key':'ipr'},
{'title': 'AD / Shepherd', 'key':'ad'}]
if hasattr(form.data, "urlencode"): # form was fed a Django QueryDict, not local plain dict
d = form.data.copy()
for h in meta['headers']:
d["sort"] = h["key"]
h["sort_url"] = "?" + d.urlencode()
if h['key'] == query.get('sort'):
h['sorted'] = True
return (results, meta)
return docs
def search(request):
if request.GET:
@ -361,12 +186,13 @@ def search(request):
if not form.is_valid():
return HttpResponseBadRequest("form not valid: %s" % form.errors)
results, meta = retrieve_search_results(form, request)
results = retrieve_search_results(form)
results, meta = prepare_document_table(request, results, get_params)
meta['searching'] = True
else:
form = SearchForm()
results = []
meta = { 'by': None, 'advanced': False, 'searching': False }
meta = { 'by': None, 'searching': False }
return render(request, 'doc/search/search.html', {
'form':form, 'docs':results, 'meta':meta, },
@ -426,7 +252,7 @@ def search_for_name(request, name):
else:
for t in doctypenames:
if n.startswith(t.prefix):
search_args += "&include-%s=on" % t.slug
search_args += "&doctypes=%s" % t.slug
break
else:
search_args += "&rfcs=on&activedrafts=on&olddrafts=on"
@ -533,8 +359,8 @@ def docs_for_ad(request, name):
raise Http404
form = SearchForm({'by':'ad','ad': ad.id,
'rfcs':'on', 'activedrafts':'on', 'olddrafts':'on',
'sort': 'status'})
results, meta = retrieve_search_results(form, request, all_types=True)
'sort': 'status', 'doctypes': list(DocTypeName.objects.exclude(slug='draft').values_list("pk", flat=True))})
results, meta = prepare_document_table(request, retrieve_search_results(form), form.data)
results.sort(key=ad_dashboard_sort_key)
del meta["headers"][-1]
#
@ -548,7 +374,7 @@ def docs_for_ad(request, name):
def drafts_in_last_call(request):
lc_state = State.objects.get(type="draft-iesg", slug="lc").pk
form = SearchForm({'by':'state','state': lc_state, 'rfcs':'on', 'activedrafts':'on'})
results, meta = retrieve_search_results(form, request)
results, meta = prepare_document_table(request, retrieve_search_results(form), form.data)
return render(request, 'doc/drafts_in_last_call.html', {
'form':form, 'docs':results, 'meta':meta

View file

@ -47,10 +47,10 @@ from django.views.decorators.cache import cache_page
from django.db.models import Q
from django.utils.safestring import mark_safe
from ietf.doc.views_search import SearchForm, retrieve_search_results
from ietf.doc.models import Document, State, DocAlias, RelatedDocument
from ietf.doc.utils import get_chartering_type
from ietf.doc.templatetags.ietf_filters import clean_whitespace
from ietf.doc.utils_search import prepare_document_table
from ietf.group.models import Group, Role, ChangeStateGroupEvent
from ietf.name.models import GroupTypeName
from ietf.group.utils import get_charter_text, can_manage_group_type, milestone_reviewer_for_group_type
@ -379,12 +379,12 @@ def construct_group_menu_context(request, group, selected, group_type, others):
return d
def search_for_group_documents(group, request):
form = SearchForm({ 'by':'group', 'group': group.acronym or "", 'rfcs':'on', 'activedrafts': 'on' })
docs, meta = retrieve_search_results(form, request)
qs = Document.objects.filter(states__type="draft", states__slug__in=["active", "rfc"], group=group)
docs, meta = prepare_document_table(request, qs)
# get the related docs
form_related = SearchForm({ 'by':'group', 'name': u'-%s-' % group.acronym, 'activedrafts': 'on' })
raw_docs_related, meta_related = retrieve_search_results(form_related, request)
qs_related = Document.objects.filter(states__type="draft", states__slug="active", name__contains="-%s-" % group.acronym)
raw_docs_related, meta_related = prepare_document_table(request, qs_related)
docs_related = []
for d in raw_docs_related:

View file

@ -5,7 +5,8 @@ from django.template import RequestContext
from django.http import Http404, HttpResponseForbidden
from django import forms
from ietf.doc.views_search import SearchForm, retrieve_search_results
from ietf.doc.models import Document
from ietf.doc.utils_search import prepare_document_table
from ietf.group.models import Group, GroupEvent, Role
from ietf.group.utils import save_group_in_history
from ietf.ietfauth.utils import has_role
@ -27,9 +28,9 @@ def stream_documents(request, acronym):
group = get_object_or_404(Group, acronym=acronym)
editable = has_role(request.user, "Secretariat") or group.has_role(request.user, "chair")
stream = StreamName.objects.get(slug=acronym)
form = SearchForm({'by':'stream', 'stream':acronym,
'rfcs':'on', 'activedrafts':'on'})
docs, meta = retrieve_search_results(form, request)
qs = Document.objects.filter(states__type="draft", states__slug__in=["active", "rfc"], stream=acronym)
docs, meta = prepare_document_table(request, qs)
return render_to_response('group/stream_documents.html', {'stream':stream, 'docs':docs, 'meta':meta, 'editable':editable }, context_instance=RequestContext(request))
class StreamEditForm(forms.Form):

View file

@ -57,7 +57,7 @@ from ietf.iesg.agenda import agenda_data, agenda_sections, fill_in_agenda_docs,
from ietf.iesg.models import TelechatDate
from ietf.ietfauth.utils import has_role, role_required, user_is_person
from ietf.person.models import Person
from ietf.doc.views_search import fill_in_search_attributes
from ietf.doc.utils_search import fill_in_document_table_attributes
def review_decisions(request, year=None):
events = DocEvent.objects.filter(type__in=("iesg_disapproved", "iesg_approved"))
@ -363,7 +363,7 @@ def agenda_documents(request):
sections = agenda_sections()
# augment the docs with the search attributes, since we're using
# the search_result_row view to display them (which expects them)
fill_in_search_attributes(docs_by_date[date])
fill_in_document_table_attributes(docs_by_date[date])
fill_in_agenda_docs(date, sections, docs_by_date[date])
telechats.append({

View file

@ -42,10 +42,10 @@
<label class="control-label" for="id_olddrafts">{{ form.olddrafts }} Internet-Draft (expired, replaced or withdrawn)</label>
</div>
{% for doc_type in form.doctypes %}
{% for value, label in form.fields.doctypes.choices %}
<div class="checkbox">
<label class="control-label" for="id_{{doc_type.slug}}">
<input type="checkbox" class="advdoctype" {% if doc_type.slug in meta.checked %}checked{% endif %} name="include-{{doc_type.slug}}" id="id_{{doc_type.slug}}"/>{{doc_type|safe|capfirst_allcaps}}
<label class="control-label" for="id_doctypes_{{ value }}">
<input type="checkbox" class="advdoctype" {% if value in form.doctypes.value %}checked{% endif %} name="doctypes" value="{{ value }}" id="id_doctypes_{{ value }}"/>{{ label|safe|capfirst_allcaps}}
</label>
</div>
{% endfor %}
@ -54,7 +54,7 @@
<div class="form-group search_field">
<div class="col-sm-4">
<input type="radio" name="by" value="author" {% if meta.by == "author" %}checked{% endif %}/>
<input type="radio" name="by" value="author" {% if form.by.value == "author" %}checked{% endif %} id="id_author"/>
<label for="id_author" class="control-label">Author</label>
</div>
<div class="col-sm-8">
@ -64,7 +64,7 @@
<div class="form-group search_field">
<div class="col-sm-4">
<input type="radio" name="by" value="group" {% if meta.by == "group" %}checked{% endif %}/>
<input type="radio" name="by" value="group" {% if form.by.value == "group" %}checked{% endif %} id="id_group"/>
<label for="id_group" class="control-label">WG</label>
</div>
<div class="col-sm-8">
@ -75,7 +75,7 @@
<div class="form-group search_field">
<div class="col-sm-4">
<input type="radio" name="by" value="area" {% if meta.by == "area" %}checked{% endif %}/>
<input type="radio" name="by" value="area" {% if form.by.value == "area" %}checked{% endif %} id="id_area"/>
<label for="id_area" class="control-label">Area</label>
</div>
<div class="col-sm-8">
@ -85,7 +85,7 @@
<div class="form-group search_field">
<div class="col-sm-4">
<input type="radio" name="by" value="ad" {% if meta.by == "ad" %}checked{% endif %}/>
<input type="radio" name="by" value="ad" {% if form.by.value == "ad" %}checked{% endif %} id="id_ad"/>
<label for="id_ad" class="control-label">AD</label>
</div>
<div class="col-sm-8">
@ -95,7 +95,7 @@
<div class="form-group search_field">
<div class="col-sm-4">
<input type="radio" name="by" value="state" {% if meta.by == "state" %}checked{% endif %}/>
<input type="radio" name="by" value="state" {% if form.by.value == "state" %}checked{% endif %} id="id_state"/>
<label for="id_state" class="control-label">IESG State</label>
</div>
<div class="col-sm-4">
@ -108,7 +108,7 @@
<div class="form-group search_field">
<div class="col-sm-4">
<input type="radio" name="by" value="stream" {% if meta.by == "stream" %}checked{% endif %}/>
<input type="radio" name="by" value="stream" {% if form.by.value == "stream" %}checked{% endif %} id="id_stream"/>
<label for="id_stream" class="control-label">Stream</label>
</div>
<div class="col-sm-4">