diff --git a/ietf/community/migrations/0004_cleanup_data.py b/ietf/community/migrations/0004_cleanup_data.py index d8b3b5e5d..6ee14c951 100644 --- a/ietf/community/migrations/0004_cleanup_data.py +++ b/ietf/community/migrations/0004_cleanup_data.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from tqdm import tqdm +import re from django.db import migrations, models def port_rules_to_typed_system(apps, schema_editor): @@ -22,11 +22,7 @@ def port_rules_to_typed_system(apps, schema_editor): return person_qs - print("") - print(" * Port Rules to Typed System") - print(" Expect an initial rule count of around 550 here, as of May 2016:") - print(" rule count: %s" % SearchRule.objects.all().count()) - for rule in tqdm(SearchRule.objects.all()): + for rule in SearchRule.objects.all(): handled = False if rule.rule_type in ['wg_asociated', 'area_asociated', 'wg_asociated_rfc', 'area_asociated_rfc']: @@ -117,14 +113,10 @@ def port_rules_to_typed_system(apps, schema_editor): else: rule.delete() #print "NOT HANDLED", rule.pk, rule.rule_type, rule.value - print(" rule count: %s" % SearchRule.objects.all().count()) def delete_extra_person_rules(apps, schema_editor): SearchRule = apps.get_model("community", "SearchRule") - print("") - print(" * Delete Extra Person Rules") SearchRule.objects.exclude(person=None).filter(value="").delete() - print(" rule count: %s" % SearchRule.objects.all().count()) RENAMED_RULES = [ ('wg_asociated', 'group'), @@ -150,44 +142,33 @@ def rename_rule_type_forwards(apps, schema_editor): renamings = dict(RENAMED_RULES) - print("") - print(" * Rename Rule Type Forwards") - for r in tqdm(SearchRule.objects.all()): + for r in SearchRule.objects.all(): if r.rule_type in renamings: r.rule_type = renamings[r.rule_type] r.save() - print(" rule count: %s" % SearchRule.objects.all().count()) def rename_rule_type_backwards(apps, schema_editor): SearchRule = apps.get_model("community", "SearchRule") renamings = dict((to, fro) for fro, to in RENAMED_RULES) - print("") - print(" * Rename Rule Type Backwards") - for r in tqdm(SearchRule.objects.all()): + for r in SearchRule.objects.all(): if r.rule_type in renamings: r.rule_type = renamings[r.rule_type] r.save() - print(" rule count: %s" % SearchRule.objects.all().count()) def get_rid_of_empty_lists(apps, schema_editor): CommunityList = apps.get_model("community", "CommunityList") - print("") - print(" * Get Rid of Empty Lists") - for cl in tqdm(CommunityList.objects.all()): - if not cl.added_docs.exists() and not cl.searchrule_set.exists() and not cl.emailsubscription_set.exists(): - cl.delete() + for cl in CommunityList.objects.filter(added_docs=None, searchrule=None, emailsubscription=None): + cl.delete() def move_email_subscriptions_to_preregistered_email(apps, schema_editor): EmailSubscription = apps.get_model("community", "EmailSubscription") Email = apps.get_model("person", "Email") Person = apps.get_model("person", "Person") - print("") - print(" * Move Email Subscriptions to Preregistered Email") - for e in tqdm(EmailSubscription.objects.all()): + for e in EmailSubscription.objects.all(): email_obj = None try: email_obj = Email.objects.get(address=e.email) @@ -210,14 +191,12 @@ def move_email_subscriptions_to_preregistered_email(apps, schema_editor): def fill_in_notify_on(apps, schema_editor): EmailSubscription = apps.get_model("community", "EmailSubscription") -# print("") -# print(" * Fill In Notify On") EmailSubscription.objects.filter(significant=False).update(notify_on="all") EmailSubscription.objects.filter(significant=True).update(notify_on="significant") def add_group_community_lists(apps, schema_editor): Group = apps.get_model("group", "Group") - Document = apps.get_model("doc", "Document") + DocAlias = apps.get_model("doc", "DocAlias") State = apps.get_model("doc", "State") CommunityList = apps.get_model("community", "CommunityList") SearchRule = apps.get_model("community", "SearchRule") @@ -225,23 +204,27 @@ def add_group_community_lists(apps, schema_editor): active_state = State.objects.get(slug="active", type="draft") rfc_state = State.objects.get(slug="rfc", type="draft") - print("") - print(" * Add Group Community Lists") - for g in tqdm(Group.objects.filter(type__in=("rg", "wg"))): + draft_aliases = DocAlias.objects.filter(name__startswith="draft") + + for g in Group.objects.filter(type__in=("rg", "wg")): clist = CommunityList.objects.filter(group=g).first() if clist: SearchRule.objects.get_or_create(community_list=clist, rule_type="group", group=g, state=active_state) SearchRule.objects.get_or_create(community_list=clist, rule_type="group_rfc", group=g, state=rfc_state) r, _ = SearchRule.objects.get_or_create(community_list=clist, rule_type="name_contains", text=r"^draft-[^-]+-%s-" % g.acronym, state=active_state) - r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text) + + # do the matching manually in Python to speed up the operation + #r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text) + name_re = re.compile(r.text) + r.name_contains_index = [ a.document_id for a in draft_aliases if name_re.match(a.name) ] else: clist = CommunityList.objects.create(group=g) SearchRule.objects.create(community_list=clist, rule_type="group", group=g, state=active_state) SearchRule.objects.create(community_list=clist, rule_type="group_rfc", group=g, state=rfc_state) r = SearchRule.objects.create(community_list=clist, rule_type="name_contains", text=r"^draft-[^-]+-%s-" % g.acronym, state=active_state) - r.name_contains_index = Document.objects.filter(docalias__name__regex=r.text) - print(" rule count: %s" % SearchRule.objects.all().count()) + name_re = re.compile(r.text) + r.name_contains_index = [ a.document_id for a in draft_aliases if name_re.match(a.name) ] def noop(apps, schema_editor): pass