altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Speed up the community refactor migration by building the index

Browse source 
of the group name search rules in Python
 - Legacy-Id: 11188
This commit is contained in:
Ole Laursen 2016-05-13 13:30:37 +00:00
parent 12079a5612
commit 9ad8c983a2
1 changed files with 18 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
ietf/community/migrations/0004_cleanup_data.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import unicode_literals from __future__ import unicode_literals
from tqdm import tqdm import re
from django.db import migrations, models from django.db import migrations, models
def port_rules_to_typed_system(apps, schema_editor): def port_rules_to_typed_system(apps, schema_editor):
@ -22,11 +22,7 @@ def port_rules_to_typed_system(apps, schema_editor):
return person_qs return person_qs
print("") for rule in SearchRule.objects.all():
print(" * Port Rules to Typed System")
print(" Expect an initial rule count of around 550 here, as of May 2016:")
print(" rule count: %s" % SearchRule.objects.all().count())
for rule in tqdm(SearchRule.objects.all()):
handled = False handled = False
if rule.rule_type in ['wg_asociated', 'area_asociated', 'wg_asociated_rfc', 'area_asociated_rfc']: if rule.rule_type in ['wg_asociated', 'area_asociated', 'wg_asociated_rfc', 'area_asociated_rfc']:
@ -117,14 +113,10 @@ def port_rules_to_typed_system(apps, schema_editor):
else: else:
rule.delete() rule.delete()
#print "NOT HANDLED", rule.pk, rule.rule_type, rule.value #print "NOT HANDLED", rule.pk, rule.rule_type, rule.value
print(" rule count: %s" % SearchRule.objects.all().count())
def delete_extra_person_rules(apps, schema_editor): def delete_extra_person_rules(apps, schema_editor):
SearchRule = apps.get_model("community", "SearchRule") SearchRule = apps.get_model("community", "SearchRule")
print("")
print(" * Delete Extra Person Rules")
SearchRule.objects.exclude(person=None).filter(value="").delete() SearchRule.objects.exclude(person=None).filter(value="").delete()
print(" rule count: %s" % SearchRule.objects.all().count())
RENAMED_RULES = [ RENAMED_RULES = [
('wg_asociated', 'group'), ('wg_asociated', 'group'),
@ -150,44 +142,33 @@ def rename_rule_type_forwards(apps, schema_editor):
renamings = dict(RENAMED_RULES) renamings = dict(RENAMED_RULES)
print("") for r in SearchRule.objects.all():
print(" * Rename Rule Type Forwards")
for r in tqdm(SearchRule.objects.all()):
if r.rule_type in renamings: if r.rule_type in renamings:
r.rule_type = renamings[r.rule_type] r.rule_type = renamings[r.rule_type]
r.save() r.save()
print(" rule count: %s" % SearchRule.objects.all().count())
def rename_rule_type_backwards(apps, schema_editor): def rename_rule_type_backwards(apps, schema_editor):
SearchRule = apps.get_model("community", "SearchRule") SearchRule = apps.get_model("community", "SearchRule")
renamings = dict((to, fro) for fro, to in RENAMED_RULES) renamings = dict((to, fro) for fro, to in RENAMED_RULES)
print("") for r in SearchRule.objects.all():
print(" * Rename Rule Type Backwards")
for r in tqdm(SearchRule.objects.all()):
if r.rule_type in renamings: if r.rule_type in renamings:
r.rule_type = renamings[r.rule_type] r.rule_type = renamings[r.rule_type]
r.save() r.save()
print(" rule count: %s" % SearchRule.objects.all().count())
def get_rid_of_empty_lists(apps, schema_editor): def get_rid_of_empty_lists(apps, schema_editor):
CommunityList = apps.get_model("community", "CommunityList") CommunityList = apps.get_model("community", "CommunityList")
print("") for cl in CommunityList.objects.filter(added_docs=None, searchrule=None, emailsubscription=None):
print(" * Get Rid of Empty Lists") cl.delete()
for cl in tqdm(CommunityList.objects.all()):
if not cl.added_docs.exists() and not cl.searchrule_set.exists() and not cl.emailsubscription_set.exists():
cl.delete()
def move_email_subscriptions_to_preregistered_email(apps, schema_editor): def move_email_subscriptions_to_preregistered_email(apps, schema_editor):
EmailSubscription = apps.get_model("community", "EmailSubscription") EmailSubscription = apps.get_model("community", "EmailSubscription")
Email = apps.get_model("person", "Email") Email = apps.get_model("person", "Email")
Person = apps.get_model("person", "Person") Person = apps.get_model("person", "Person")
print("") for e in EmailSubscription.objects.all():
print(" * Move Email Subscriptions to Preregistered Email")
for e in tqdm(EmailSubscription.objects.all()):
email_obj = None email_obj = None
try: try:
email_obj = Email.objects.get(address=e.email) email_obj = Email.objects.get(address=e.email)
@ -210,14 +191,12 @@ def move_email_subscriptions_to_preregistered_email(apps, schema_editor):
def fill_in_notify_on(apps, schema_editor): def fill_in_notify_on(apps, schema_editor):
EmailSubscription = apps.get_model("community", "EmailSubscription") EmailSubscription = apps.get_model("community", "EmailSubscription")
# print("")
# print(" * Fill In Notify On")
EmailSubscription.objects.filter(significant=False).update(notify_on="all") EmailSubscription.objects.filter(significant=False).update(notify_on="all")
EmailSubscription.objects.filter(significant=True).update(notify_on="significant") EmailSubscription.objects.filter(significant=True).update(notify_on="significant")
def add_group_community_lists(apps, schema_editor): def add_group_community_lists(apps, schema_editor):
Group = apps.get_model("group", "Group") Group = apps.get_model("group", "Group")
Document = apps.get_model("doc", "Document") DocAlias = apps.get_model("doc", "DocAlias")
State = apps.get_model("doc", "State") State = apps.get_model("doc", "State")
CommunityList = apps.get_model("community", "CommunityList") CommunityList = apps.get_model("community", "CommunityList")
SearchRule = apps.get_model("community", "SearchRule") SearchRule = apps.get_model("community", "SearchRule")
@ -225,23 +204,27 @@ def add_group_community_lists(apps, schema_editor):
active_state = State.objects.get(slug="active", type="draft") active_state = State.objects.get(slug="active", type="draft")
rfc_state = State.objects.get(slug="rfc", type="draft") rfc_state = State.objects.get(slug="rfc", type="draft")
print("") draft_aliases = DocAlias.objects.filter(name__startswith="draft")
print(" * Add Group Community Lists")
for g in tqdm(Group.objects.filter(type__in=("rg", "wg"))): for g in Group.objects.filter(type__in=("rg", "wg")):
clist = CommunityList.objects.filter(group=g).first() clist = CommunityList.objects.filter(group=g).first()
if clist: if clist:
SearchRule.objects.get_or_create(community_list=clist, rule_type="group", group=g, state=active_state) SearchRule.objects.get_or_create(community_list=clist, rule_type="group", group=g, state=active_state)
SearchRule.objects.get_or_create(community_list=clist, rule_type="group_rfc", group=g, state=rfc_state) SearchRule.objects.get_or_create(community_list=clist, rule_type="group_rfc", group=g, state=rfc_state)
r, _ = SearchRule.objects.get_or_create(community_list=clist, rule_type="name_contains", text=r"^draft-[^-]+-%s-" % g.acronym, state=active_state) r, _ = SearchRule.objects.get_or_create(community_list=clist, rule_type="name_contains", text=r"^draft-[^-]+-%s-" % g.acronym, state=active_state)
r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text)
# do the matching manually in Python to speed up the operation
#r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text)
name_re = re.compile(r.text)
r.name_contains_index = [ a.document_id for a in draft_aliases if name_re.match(a.name) ]
else: else:
clist = CommunityList.objects.create(group=g) clist = CommunityList.objects.create(group=g)
SearchRule.objects.create(community_list=clist, rule_type="group", group=g, state=active_state) SearchRule.objects.create(community_list=clist, rule_type="group", group=g, state=active_state)
SearchRule.objects.create(community_list=clist, rule_type="group_rfc", group=g, state=rfc_state) SearchRule.objects.create(community_list=clist, rule_type="group_rfc", group=g, state=rfc_state)
r = SearchRule.objects.create(community_list=clist, rule_type="name_contains", text=r"^draft-[^-]+-%s-" % g.acronym, state=active_state) r = SearchRule.objects.create(community_list=clist, rule_type="name_contains", text=r"^draft-[^-]+-%s-" % g.acronym, state=active_state)
r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text) name_re = re.compile(r.text)
print(" rule count: %s" % SearchRule.objects.all().count()) r.name_contains_index = [ a.document_id for a in draft_aliases if name_re.match(a.name) ]
def noop(apps, schema_editor): def noop(apps, schema_editor):
pass pass