Rewrite search query building to support new schema, extend proxy support to search pages, fix a couple of bugs

- Legacy-Id: 2780
This commit is contained in:
Ole Laursen 2011-02-01 10:59:20 +00:00
parent 19b572b285
commit 9f1bd35ca9
9 changed files with 322 additions and 34 deletions

View file

@ -262,9 +262,9 @@ class RfcWrapper:
if not self._idinternal:
if settings.USE_DB_REDESIGN_PROXY_CLASSES:
pub = rfcindex.latest_event(type="published_rfc")
started = rfcindex.latest_event(type="started_iesg_process")
if pub and started and pub.time < started.time:
pub = rfcindex.rfc_published_date
started = rfcindex.started_iesg_process if hasattr(rfcindex, 'started_iesg_process') else rfcindex.latest_event(type="started_iesg_process")
if pub and started and pub < started.time.date():
self._idinternal = rfcindex
else:
try:
@ -282,7 +282,9 @@ class RfcWrapper:
if not self.maturity_level:
self.maturity_level = "Unknown"
if settings.USE_DB_REDESIGN_PROXY_CLASSES and rfcindex.filename.startswith('rfc'):
if settings.USE_DB_REDESIGN_PROXY_CLASSES:
if not rfcindex.name.startswith('rfc'):
self.draft_name = rfcindex.name
return # we've already done the lookup while importing so skip the rest
ids = InternetDraft.objects.filter(rfc_number=self.rfc_number)
@ -779,7 +781,7 @@ def position_to_string(position):
return "No Record"
p = None
for k,v in positions.iteritems():
if position.__dict__[k] > 0:
if getattr(position, k) > 0:
p = v
if not p:
p = "No Record"

View file

@ -40,6 +40,11 @@ register = template.Library()
def get_user_adid(context):
if 'user' in context and in_group(context['user'], "Area_Director"):
if settings.USE_DB_REDESIGN_PROXY_CLASSES:
# with the new design, we need the email address (FIXME:
# we shouldn't go through the old classes though,
# IESGLogin needs revamping)
return context['user'].get_profile().person().email()[1]
return context['user'].get_profile().iesg_login_id()
else:
return None

View file

@ -10,8 +10,8 @@ from django.template.loader import render_to_string
from django.template import RequestContext
from django import forms
from django.utils.html import strip_tags
from django.conf import settings
from ietf import settings
from ietf.utils.mail import send_mail_text, send_mail_preformatted
from ietf.ietfauth.decorators import group_required
from ietf.idtracker.templatetags.ietf_filters import in_group

View file

@ -42,8 +42,8 @@ from django.utils import simplejson as json
from django.utils.decorators import decorator_from_middleware
from django.middleware.gzip import GZipMiddleware
from django.core.urlresolvers import reverse as urlreverse
from ietf import settings
from django.conf import settings
from ietf.idtracker.models import InternetDraft, IDInternal, BallotInfo, DocumentComment
from ietf.idtracker.templatetags.ietf_filters import format_textarea, fill
from ietf.idrfc import markup_txt

View file

@ -11,8 +11,8 @@ from django.template import RequestContext
from django import forms
from django.utils.html import strip_tags
from django.db.models import Max
from django.conf import settings
from ietf import settings
from ietf.utils.mail import send_mail_text
from ietf.ietfauth.decorators import group_required
from ietf.idtracker.templatetags.ietf_filters import in_group

View file

@ -41,6 +41,7 @@ from ietf.idrfc.models import RfcIndex
from django.http import Http404, HttpResponse, HttpResponsePermanentRedirect
from ietf.idrfc.idrfc_wrapper import IdWrapper,RfcWrapper,IdRfcWrapper
from ietf.utils import normalize_draftname
from django.conf import settings
class SearchForm(forms.Form):
name = forms.CharField(required=False)
@ -252,6 +253,206 @@ def search_query(query_original):
meta['advanced'] = True
return (results,meta)
if settings.USE_DB_REDESIGN_PROXY_CLASSES:
from doc.models import *
from person.models import *
from group.models import *
class SearchForm(forms.Form):
name = forms.CharField(required=False)
rfcs = forms.BooleanField(required=False,initial=True)
activeDrafts = forms.BooleanField(required=False,initial=True)
oldDrafts = forms.BooleanField(required=False,initial=False)
lucky = forms.BooleanField(required=False,initial=False)
by = forms.ChoiceField(choices=[(x,x) for x in ('author','group','area','ad','state')], required=False, initial='wg', label='Foobar')
author = forms.CharField(required=False)
group = forms.CharField(required=False)
area = forms.ModelChoiceField(Group.objects.filter(type="area", state="active").order_by('name'), empty_label="any area", required=False)
ad = forms.ChoiceField(choices=(), required=False)
# FIXME: state needs a sort
state = forms.ModelChoiceField(IesgDocStateName.objects.all(), empty_label="any state", required=False)
subState = forms.ChoiceField(choices=(), required=False)
def __init__(self, *args, **kwargs):
super(SearchForm, self).__init__(*args, **kwargs)
responsible = Document.objects.values_list('ad', flat=True).distinct()
active_ads = list(Email.objects.filter(role__name="ad",
role__group__type="area",
role__group__state="active")
.select_related('person'))
inactive_ads = list(Email.objects.filter(pk__in=responsible)
.exclude(pk__in=[x.pk for x in active_ads])
.select_related('person'))
extract_last_name = lambda x: x.get_name().split(' ')[-1]
active_ads.sort(key=extract_last_name)
inactive_ads.sort(key=extract_last_name)
# FIXME: -99
self.fields['ad'].choices = c = [('', 'any AD')] + [(ad.pk, ad.get_name()) for ad in active_ads] + [('-99', '------------------')] + [(ad.pk, ad.get_name()) for ad in inactive_ads]
self.fields['subState'].choices = [('', 'any substate'), ('0', 'no substate')] + [(n.slug, n.name) for n in DocInfoTagName.objects.filter(slug__in=('point', 'ad-f-up', 'need-rev', 'extpty'))]
def clean_name(self):
value = self.cleaned_data.get('name','')
return normalize_draftname(value)
def clean(self):
q = self.cleaned_data
# Reset query['by'] if needed
for k in ('author','group','area','ad'):
if (q['by'] == k) and not q[k]:
q['by'] = None
if (q['by'] == 'state') and not (q['state'] or q['subState']):
q['by'] = None
# Reset other fields
for k in ('author','group','area','ad'):
if q['by'] != k:
self.data[k] = ""
q[k] = ""
if q['by'] != 'state':
self.data['state'] = ""
self.data['subState'] = ""
q['state'] = ""
q['subState'] = ""
return q
def search_query(query_original):
query = dict(query_original.items())
drafts = query['activeDrafts'] or query['oldDrafts']
if (not drafts) and (not query['rfcs']):
return ([], {})
# Non-ASCII strings don't match anything; this check
# is currently needed to avoid complaints from MySQL.
# FIXME: this should be fixed if it's still a problem
for k in ['name','author','group']:
try:
tmp = str(query.get(k, ''))
except:
query[k] = '*NOSUCH*'
# Start by search InternetDrafts
idresults = []
rfcresults = []
MAX = 500
docs = InternetDraft.objects.all()
# name
if query["name"]:
docs = docs.filter(Q(docalias__name__icontains=query["name"]) |
Q(title__icontains=query["name"])).distinct()
# rfc/active/old check buttons
allowed = []
disallowed = []
def add(allow, states):
l = allowed if allow else disallowed
l.extend(states)
add(query["rfcs"], ['rfc'])
add(query["activeDrafts"], ['active'])
add(query["oldDrafts"], ['repl', 'expired', 'auth-rm', 'ietf-rm'])
docs = docs.filter(state__in=allowed).exclude(state__in=disallowed)
# radio choices
by = query["by"]
if by == "author":
# FIXME: this is full name, not last name as hinted in the HTML
docs = docs.filter(authors__person__name__icontains=query["author"])
elif by == "group":
docs = docs.filter(group__acronym=query["group"])
elif by == "area":
docs = docs.filter(Q(group__parent=query["area"]) |
Q(ad__role__name="ad",
ad__role__group=query["area"]))
elif by == "ad":
docs = docs.filter(ad=query["ad"])
elif by == "state":
if query["state"]:
docs = docs.filter(iesg_state=query["state"])
if query["subState"]:
docs = docs.filter(tags=query["subState"])
# evaluate and fill in values with aggregate queries to avoid
# too many individual queries
results = list(docs.select_related("state", "iesg_state", "ad", "ad__person", "std_level", "intended_std_level", "group")[:MAX])
rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc", document__in=[r.pk for r in results]).values_list("document_id", "name"))
# canonical name
for r in results:
if r.pk in rfc_aliases:
r.canonical_name = rfc_aliases[r.pk]
else:
r.canonical_name = r.name
result_map = dict((r.pk, r) for r in results)
# events
event_types = ("published_rfc",
"changed_ballot_position",
"started_iesg_process",
"new_revision")
for d in rfc_aliases.keys():
for e in event_types:
setattr(result_map[d], e, None)
for e in Event.objects.filter(doc__in=rfc_aliases.keys(), type__in=event_types).order_by('-time'):
r = result_map[e.doc_id]
if not getattr(r, e.type):
# sets e.g. r.published_date = e for use in proxy wrapper
setattr(r, e.type, e)
# obsoleted/updated by
for d in rfc_aliases:
r = result_map[d]
r.obsoleted_by_list = []
r.updated_by_list = []
xed_by = RelatedDocument.objects.filter(doc_alias__name__in=rfc_aliases.values(), relationship__in=("obs", "updates")).select_related('doc_alias__document_id')
rel_rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc", document__in=[rel.document_id for rel in xed_by]).values_list('document_id', 'name'))
for rel in xed_by:
r = result_map[rel.doc_alias.document_id]
if rel.relationship_id == "obs":
attr = "obsoleted_by_list"
else:
attr = "updated_by_list"
getattr(r, attr).append(int(rel_rfc_aliases[rel.document_id][3:]))
# sort
def sort_key(d):
if d.canonical_name.startswith('rfc'):
return (2, "%06d" % int(d.canonical_name[3:]))
elif d.state_id == "active":
return (1, d.canonical_name)
else:
return (3, d.canonical_name)
results.sort(key=sort_key)
meta = {}
if len(docs) == MAX:
meta['max'] = MAX
if query['by']:
meta['advanced'] = True
# finally wrap in old wrappers
wrapped_results = []
for r in results:
draft = None
rfc = None
if not r.name.startswith('rfc'):
draft = IdWrapper(r)
if r.name.startswith('rfc') or r.pk in rfc_aliases:
rfc = RfcWrapper(r)
wrapped_results.append(IdRfcWrapper(draft, rfc))
return (wrapped_results, meta)
def search_results(request):
if len(request.REQUEST.items()) == 0:
return search_main(request)
@ -288,6 +489,8 @@ def by_ad(request, name):
break
if not ad_id:
raise Http404
if settings.USE_DB_REDESIGN_PROXY_CLASSES:
ad_id = i.person.email()[1]
form = SearchForm({'by':'ad','ad':ad_id,
'rfcs':'on', 'activeDrafts':'on', 'oldDrafts':'on'})
if not form.is_valid():
@ -298,11 +501,17 @@ def by_ad(request, name):
@cache_page(15*60) # 15 minutes
def all(request):
active = InternetDraft.objects.all().filter(status=1).order_by("filename").values('filename')
rfc1 = InternetDraft.objects.all().filter(status=3).order_by("filename").values('filename','rfc_number')
rfc_numbers1 = InternetDraft.objects.all().filter(status=3).values_list('rfc_number', flat=True)
rfc2 = RfcIndex.objects.all().exclude(rfc_number__in=rfc_numbers1).order_by('rfc_number').values('rfc_number','draft')
dead = InternetDraft.objects.all().exclude(status__in=[1,3]).order_by("filename").select_related('status__status')
if settings.USE_DB_REDESIGN_PROXY_CLASSES:
active = (dict(filename=n) for n in InternetDraft.objects.filter(state="active").order_by("name").values_list('name', flat=True))
rfc1 = (dict(filename=d, rfc_number=int(n[3:])) for d, n in DocAlias.objects.filter(document__state="rfc", name__startswith="rfc").exclude(document__name__startswith="rfc").order_by("document__name").values_list('document__name','name').distinct())
rfc2 = (dict(rfc_number=r, draft=None) for r in sorted(int(n[3:]) for n in Document.objects.filter(name__startswith="rfc").values_list('name', flat=True)))
dead = InternetDraft.objects.exclude(state__in=("active", "rfc")).select_related("state").order_by("name")
else:
active = InternetDraft.objects.all().filter(status=1).order_by("filename").values('filename')
rfc1 = InternetDraft.objects.all().filter(status=3).order_by("filename").values('filename','rfc_number')
rfc_numbers1 = InternetDraft.objects.all().filter(status=3).values_list('rfc_number', flat=True)
rfc2 = RfcIndex.objects.all().exclude(rfc_number__in=rfc_numbers1).order_by('rfc_number').values('rfc_number','draft')
dead = InternetDraft.objects.all().exclude(status__in=[1,3]).order_by("filename").select_related('status__status')
return render_to_response('idrfc/all.html', {'active':active, 'rfc1':rfc1, 'rfc2':rfc2, 'dead':dead}, context_instance=RequestContext(request))
@cache_page(15*60) # 15 minutes

View file

@ -10,7 +10,7 @@ import glob, os
class InternetDraft(Document):
objects = TranslatingManager(dict(filename="name",
id_document_tag="id",
status=lambda v: ("state", { 1: 'active', 2: 'expired', 3: 'rfc', 4: 'auth-rm', 5: 'repl', 6: 'ietf-rm'}),
status=lambda v: ("state", { 1: 'active', 2: 'expired', 3: 'rfc', 4: 'auth-rm', 5: 'repl', 6: 'ietf-rm'}[v]),
job_owner="ad",
rfc_number=lambda v: ("docalias__name", "rfc%s" % v),
))
@ -32,7 +32,8 @@ class InternetDraft(Document):
@property
def group(self):
from group.proxy import Acronym as AcronymProxy
return AcronymProxy(super(self.__class__, self).group)
g = super(self.__class__, self).group
return AcronymProxy(g) if g else None
#filename = models.CharField(max_length=255, unique=True)
@property
def filename(self):
@ -44,7 +45,10 @@ class InternetDraft(Document):
#revision_date = models.DateField()
@property
def revision_date(self):
e = self.latest_event(type="new_revision")
if hasattr(self, "new_revision"):
e = self.new_revision
else:
e = self.latest_event(type="new_revision")
return e.time.date() if e else None
# helper function
def get_file_type_matches_from(self, glob_path):
@ -80,11 +84,10 @@ class InternetDraft(Document):
@property
def status(self):
from redesign.name.proxy import IDStatus
return IDStatus(self.state)
return IDStatus(self.state) if self.state else None
@property
def status_id(self):
from redesign.name.proxy import IDStatus
return { 'active': 1, 'repl': 5, 'expired': 2, 'rfc': 3, 'auth-rm': 4, 'ietf-rm': 6 }[self.state_id]
#intended_status = models.ForeignKey(IDIntendedStatus)
@ -125,6 +128,10 @@ class InternetDraft(Document):
#rfc_number = models.IntegerField(null=True, blank=True, db_index=True)
@property
def rfc_number(self):
# simple optimization for search results
if hasattr(self, "canonical_name"):
return int(self.canonical_name[3:]) if self.canonical_name.startswith('rfc') else None
aliases = self.docalias_set.filter(name__startswith="rfc")
return int(aliases[0].name[3:]) if aliases else None
@ -192,7 +199,11 @@ class InternetDraft(Document):
def idinternal(self):
# since IDInternal is now merged into the document, we try to
# guess here
return self if self.iesg_state or self.latest_event(type="changed_ballot_position") else None
if hasattr(self, "changed_ballot_position"):
e = self.changed_ballot_position
else:
e = self.latest_event(type="changed_ballot_position")
return self if self.iesg_state or e else None
# reverse relationship
@property
@ -216,7 +227,7 @@ class InternetDraft(Document):
return "%02d" % r
def expiration(self):
e = self.latest_event(type__in=("completed_resurrect", "new_revision"))
return e.time.date() + datetime.timedelta(self.DAYS_TO_EXPIRE)
return e.time.date() + datetime.timedelta(days=self.DAYS_TO_EXPIRE)
def can_expire(self):
# Copying the logic from expire-ids-1 without thinking
# much about it.
@ -315,6 +326,11 @@ class InternetDraft(Document):
def cur_state(self):
return self.iesg_state
@property
def cur_state_id(self):
# FIXME: results in wrong sort order
return abs(hash(self.iesg_state.slug))
#prev_state = models.ForeignKey(IDState, db_column='prev_state', related_name='docs_prev')
@property
def prev_state(self):
@ -334,7 +350,7 @@ class InternetDraft(Document):
@property
def job_owner(self):
from person.proxy import IESGLogin as IESGLoginProxy
return IESGLoginProxy(self.ad)
return IESGLoginProxy(self.ad) if self.ad else None
#event_date = models.DateField(null=True)
@property
@ -528,14 +544,14 @@ class InternetDraft(Document):
res = []
def add(ad, pos):
# FIXME: ad and pos don't emulate old interface
res.append(dict(ad=ad, pos=pos))
from person.proxy import IESGLogin as IESGLoginProxy
res.append(dict(ad=IESGLoginProxy(ad), pos=Position(pos) if pos else None))
found = set()
for pos in self.event_set.filter(type="changed_ballot_position", ballotposition__ad__in=active_ads).select_related('ad').order_by("-time"):
if not pos.ballotposition.ad in found:
found.add(pos.ballotposition.ad)
add(pos.ballotposition.ad, pos)
for pos in BallotPosition.objects.filter(doc=self, type="changed_ballot_position", ad__in=active_ads).select_related('ad').order_by("-time", "-id"):
if pos.ad not in found:
found.add(pos.ad)
add(pos.ad, pos)
for ad in active_ads:
if ad not in found:
@ -553,7 +569,7 @@ class InternetDraft(Document):
yes = noobj = discuss = recuse = 0
for position in positions:
p = position.ballotposition.pos_id
p = position.pos_id
if p == "yes":
yes += 1
if p == "noobj":
@ -601,7 +617,10 @@ class InternetDraft(Document):
#rfc_published_date = models.DateField()
@property
def rfc_published_date(self):
e = self.latest_event(type="published_rfc")
if hasattr(self, 'published_rfc'):
e = self.published_rfc
else:
e = self.latest_event(type="published_rfc")
return e.time.date() if e else None
#current_status = models.CharField(max_length=50,null=True)
@ -617,7 +636,9 @@ class InternetDraft(Document):
#updated_by = models.CharField(max_length=200,blank=True,null=True)
@property
def updated_by(self):
return ",".join("RFC%s" % n for n in sorted(d.rfc_number for d in InternetDraft.objects.filter(relateddocument__doc_alias__document=self, relateddocument__relationship="updates")))
if not hasattr(self, "updated_by_list"):
self.updated_by_list = [d.rfc_number for d in InternetDraft.objects.filter(relateddocument__doc_alias__document=self, relateddocument__relationship="updates")]
return ",".join("RFC%s" % n for n in sorted(self.updated_by_list))
#obsoletes = models.CharField(max_length=200,blank=True,null=True)
@property
@ -627,7 +648,9 @@ class InternetDraft(Document):
#obsoleted_by = models.CharField(max_length=200,blank=True,null=True)
@property
def obsoleted_by(self):
return ",".join("RFC%s" % n for n in sorted(d.rfc_number for d in InternetDraft.objects.filter(relateddocument__doc_alias__document=self, relateddocument__relationship="obs")))
if not hasattr(self, "obsoleted_by_list"):
self.obsoleted_by_list = [d.rfc_number for d in InternetDraft.objects.filter(relateddocument__doc_alias__document=self, relateddocument__relationship="obs")]
return ",".join("RFC%s" % n for n in sorted(self.obsoleted_by_list))
#also = models.CharField(max_length=50,blank=True,null=True)
@property
@ -729,3 +752,48 @@ class DocumentComment(Event):
class Meta:
proxy = True
class Position(BallotPosition):
def __init__(self, base):
for f in base._meta.fields:
if not f.name in ('discuss',): # don't overwrite properties
setattr(self, f.name, getattr(base, f.name))
#ballot = models.ForeignKey(BallotInfo, related_name='positions')
@property
def ballot(self):
return self.doc # FIXME: doesn't emulate old interface
# ad = models.ForeignKey(IESGLogin) # same name
#yes = models.IntegerField(db_column='yes_col')
@property
def yes(self):
return self.pos_id == "yes"
#noobj = models.IntegerField(db_column='no_col')
@property
def noobj(self):
return self.pos_id == "noobj"
#abstain = models.IntegerField()
@property
def abstain(self):
return self.pos_id == "abstain"
#approve = models.IntegerField(default=0) # unused
#discuss = models.IntegerField()
@property
def discuss(self):
return self.pos_id == "discuss"
#recuse = models.IntegerField()
@property
def recuse(self):
return self.pos_id == "recuse"
def __str__(self):
return "Position for %s on %s" % ( self.ad, self.ballot )
def abstain_ind(self):
if self.recuse:
return 'R'
if self.abstain:
return 'X'
else:
return ' '
class Meta:
proxy = True

View file

@ -48,11 +48,11 @@ class Area(Group):
#def additional_urls(self):
# return AreaWGURL.objects.filter(name=self.area_acronym.name)
def active_wgs(self):
return IETFWG.objects.filter(type="wg", state="active", parent=self).order_by("acronym")
return IETFWG.objects.filter(type="wg", state="active", parent=self).select_related('type', 'state', 'parent').order_by("acronym")
@staticmethod
def active_areas():
return Area.objects.filter(type="area", state="active").order_by('acronym')
return Area.objects.filter(type="area", state="active").select_related('type', 'state', 'parent').order_by('acronym')
class Meta:
proxy = True

View file

@ -4,10 +4,14 @@ class IESGLogin(Email):
def __init__(self, base):
for f in base._meta.fields:
setattr(self, f.name, getattr(base, f.name))
SECRETARIAT_LEVEL = 0
AD_LEVEL = 1
INACTIVE_AD_LEVEL = 2
@property
def id(self):
return self.pk # this is not really backwards-compatible
#login_name = models.CharField(blank=True, max_length=255)
@property
def login_name(self): raise NotImplemented