From 3d1eb07afe0f9461c432255ac5045168fab97729 Mon Sep 17 00:00:00 2001 From: Ole Laursen Date: Wed, 10 Apr 2013 11:48:07 +0000 Subject: [PATCH] Port idindex to new schema, speed them up, add tests, refactor index page in views_search to share code with the text index file, get rid of some special-case idindex filters from ietf_filters, move "/drafts/" redirects to a file in /doc/ - Legacy-Id: 5634 --- ietf/doc/redirect_drafts_urls.py | 23 ++ ietf/doc/tests.py | 1 - ietf/idindex/generate_all_id2_txt.py | 2 +- ietf/idindex/generate_all_id_txt.py | 4 +- ietf/idindex/generate_id_abstracts_txt.py | 8 +- ietf/idindex/generate_id_index_txt.py | 8 +- ietf/idindex/index.py | 270 ++++++++++++++++++++ ietf/idindex/models.py | 1 - ietf/idindex/tests.py | 210 ++++++++++----- ietf/idindex/testurl.list | 20 -- ietf/idindex/urls.py | 27 -- ietf/idindex/views.py | 199 --------------- ietf/idrfc/views_search.py | 45 +--- ietf/idtracker/templatetags/ietf_filters.py | 37 ++- ietf/templates/idindex/all_id2.txt | 3 +- ietf/templates/idindex/all_ids.txt | 17 -- ietf/templates/idindex/id_abstracts.txt | 9 - ietf/templates/idindex/id_index.txt | 25 +- ietf/urls.py | 2 +- ietf/utils/test_data.py | 1 + 20 files changed, 478 insertions(+), 434 deletions(-) create mode 100644 ietf/doc/redirect_drafts_urls.py create mode 100644 ietf/idindex/index.py delete mode 100644 ietf/idindex/testurl.list delete mode 100644 ietf/idindex/urls.py delete mode 100644 ietf/idindex/views.py delete mode 100644 ietf/templates/idindex/all_ids.txt delete mode 100644 ietf/templates/idindex/id_abstracts.txt diff --git a/ietf/doc/redirect_drafts_urls.py b/ietf/doc/redirect_drafts_urls.py new file mode 100644 index 000000000..fa920a461 --- /dev/null +++ b/ietf/doc/redirect_drafts_urls.py @@ -0,0 +1,23 @@ +# Copyright The IETF Trust 2007, All Rights Reserved + +from django.conf import settings +from django.conf.urls.defaults import patterns + + +from django.http import HttpResponsePermanentRedirect +from django.shortcuts import get_object_or_404 + +from ietf.group.models import Group + +urlpatterns = patterns('', + (r'^$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/'}), + (r'^all/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/all/'}), + (r'^rfc/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/all/#rfc'}), + (r'^dead/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/all/#expired'}), + (r'^current/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/active/'}), + (r'^(?P\d+)/(related/)?$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/' }), + (r'^(?P[^/]+)/(related/)?$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/%(name)s/' }), + (r'^wgid/(?P\d+)/$', lambda request, id: HttpResponsePermanentRedirect("/wg/%s/" % get_object_or_404(Group, id=id).acronym)), + (r'^wg/(?P[^/]+)/$', 'django.views.generic.simple.redirect_to', { 'url': '/wg/%(acronym)s/' }), + (r'^all_id(?:_txt)?.html$', 'django.views.generic.simple.redirect_to', { 'url': 'http://www.ietf.org/id/all_id.txt' }), +) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 2f11b5b3b..ee3c2c505 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -2,7 +2,6 @@ import os, shutil, datetime import django.test from django.core.urlresolvers import reverse as urlreverse -from django.conf import settings from pyquery import PyQuery diff --git a/ietf/idindex/generate_all_id2_txt.py b/ietf/idindex/generate_all_id2_txt.py index bbebdcb08..f2ca9c229 100644 --- a/ietf/idindex/generate_all_id2_txt.py +++ b/ietf/idindex/generate_all_id2_txt.py @@ -34,5 +34,5 @@ from ietf import settings from django.core import management management.setup_environ(settings) -from ietf.idindex.views import all_id2_txt +from ietf.idindex.index import all_id2_txt print all_id2_txt().encode('utf-8'), diff --git a/ietf/idindex/generate_all_id_txt.py b/ietf/idindex/generate_all_id_txt.py index f427bb185..1f72bc428 100644 --- a/ietf/idindex/generate_all_id_txt.py +++ b/ietf/idindex/generate_all_id_txt.py @@ -34,5 +34,5 @@ from ietf import settings from django.core import management management.setup_environ(settings) -from ietf.idindex.views import all_id_txt -print all_id_txt(), +from ietf.idindex.index import all_id_txt +print all_id_txt().encode("utf-8"), diff --git a/ietf/idindex/generate_id_abstracts_txt.py b/ietf/idindex/generate_id_abstracts_txt.py index 6ea4bd40a..9a35b8b21 100644 --- a/ietf/idindex/generate_id_abstracts_txt.py +++ b/ietf/idindex/generate_id_abstracts_txt.py @@ -34,9 +34,5 @@ from ietf import settings from django.core import management management.setup_environ(settings) -from ietf.idindex.views import id_abstracts_txt -x = id_abstracts_txt() -if isinstance(x, unicode): - print x.encode('utf-8'), -else: - print x, +from ietf.idindex.index import id_index_txt +print id_index_txt(with_abstracts=True).encode('utf-8'), diff --git a/ietf/idindex/generate_id_index_txt.py b/ietf/idindex/generate_id_index_txt.py index 1b53117d5..ad25d5f4c 100644 --- a/ietf/idindex/generate_id_index_txt.py +++ b/ietf/idindex/generate_id_index_txt.py @@ -34,9 +34,5 @@ from ietf import settings from django.core import management management.setup_environ(settings) -from ietf.idindex.views import id_index_txt -x = id_index_txt() -if isinstance(x, unicode): - print x.encode('utf-8'), -else: - print x, +from ietf.idindex.index import id_index_txt +print id_index_txt().encode('utf-8'), diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py new file mode 100644 index 000000000..b3ac0984e --- /dev/null +++ b/ietf/idindex/index.py @@ -0,0 +1,270 @@ +import datetime, os + +import pytz + +from django.conf import settings +from django.template.loader import render_to_string + +from ietf.idtracker.templatetags.ietf_filters import clean_whitespace +from ietf.doc.models import * + +def all_id_txt(): + # this returns a lot of data so try to be efficient + + # precalculations + revision_time = dict(NewRevisionDocEvent.objects.filter(type="new_revision", doc__name__startswith="draft-").order_by('time').values_list("doc_id", "time")) + + def formatted_rev_date(name): + t = revision_time.get(name) + return t.strftime("%Y-%m-%d") if t else "" + + rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc", + document__states=State.objects.get(type="draft", slug="rfc")).values_list("document_id", "name")) + + replacements = dict(RelatedDocument.objects.filter(target__document__states=State.objects.get(type="draft", slug="repl"), + relationship="replaces").values_list("target__document_id", "source")) + + + # we need a distinct to prevent the queries below from multiplying the result + all_ids = Document.objects.filter(type="draft").order_by('name').exclude(name__startswith="rfc").distinct() + + res = ["\nInternet-Drafts Status Summary\n"] + + def add_line(f1, f2, f3, f4): + # each line must have exactly 4 tab-separated fields + res.append(f1 + "\t" + f2 + "\t" + f3 + "\t" + f4) + + + inactive_states = ["pub", "watching", "dead"] + + in_iesg_process = all_ids.exclude(states=State.objects.get(type="draft", slug="rfc")).filter(states__in=list(State.objects.filter(type="draft-iesg").exclude(slug__in=inactive_states))).only("name", "rev") + + # handle those actively in the IESG process + for d in in_iesg_process: + state = d.get_state("draft-iesg").name + tags = d.tags.filter(slug__in=IESG_SUBSTATE_TAGS).values_list("name", flat=True) + if tags: + state += "::" + "::".join(tags) + add_line(d.name + "-" + d.rev, + formatted_rev_date(d.name), + "In IESG processing - ID Tracker state <" + state + ">", + "", + ) + + + # handle the rest + + not_in_process = all_ids.exclude(pk__in=[d.name for d in in_iesg_process]) + + for s in State.objects.filter(type="draft").order_by("order"): + for name, rev in not_in_process.filter(states=s).values_list("name", "rev"): + state = s.name + last_field = "" + + if s.slug == "rfc": + a = rfc_aliases.get(name) + if a: + last_field = a[3:] + elif s.slug == "repl": + state += " replaced by " + replacements.get(name, "0") + + add_line(name + "-" + rev, + formatted_rev_date(name), + state, + last_field, + ) + + return u"\n".join(res) + "\n" + +def file_types_for_drafts(): + """Look in the draft directory and return file types found as dict (name + rev -> [t1, t2, ...]).""" + file_types = {} + for filename in os.listdir(settings.INTERNET_DRAFT_PATH): + if filename.startswith("draft-"): + base, ext = os.path.splitext(filename) + if ext: + if base not in file_types: + file_types[base] = [ext] + else: + file_types[base].append(ext) + + return file_types + +def all_id2_txt(): + # this returns a lot of data so try to be efficient + + drafts = Document.objects.filter(type="draft").exclude(name__startswith="rfc").order_by('name').select_related('group', 'group__parent', 'ad', 'ad__email', 'intended_std_level', 'shepherd', 'shepherd__email') + + rfc_aliases = dict(DocAlias.objects.filter(name__startswith="rfc", + document__states=State.objects.get(type="draft", slug="rfc")).values_list("document_id", "name")) + + replacements = dict(RelatedDocument.objects.filter(target__document__states=State.objects.get(type="draft", slug="repl"), + relationship="replaces").values_list("target__document_id", "source")) + + revision_time = dict(DocEvent.objects.filter(type="new_revision", doc__name__startswith="draft-").order_by('time').values_list("doc_id", "time")) + + file_types = file_types_for_drafts() + + authors = {} + for a in DocumentAuthor.objects.filter(document__name__startswith="draft-").order_by("order").select_related("author", "author__person").iterator(): + if a.document_id not in authors: + l = authors[a.document_id] = [] + else: + l = authors[a.document_id] + if "@" in a.author.address: + l.append(u'%s <%s>' % (a.author.person.plain_name().replace("@", ""), a.author.address.replace(",", ""))) + else: + l.append(a.author.person.plain_name()) + + res = [] + for d in drafts: + state = d.get_state_slug() + iesg_state = d.get_state("draft-iesg") + + fields = [] + # 0 + fields.append(d.name + "-" + d.rev) + # 1 + fields.append("-1") # used to be internal numeric identifier, we don't have that anymore + # 2 + fields.append(d.get_state().name if state else "") + # 3 + if state == "active": + s = "I-D Exists" + if iesg_state: + s = iesg_state.name + tags = d.tags.filter(slug__in=IESG_SUBSTATE_TAGS).values_list("name", flat=True) + if tags: + s += "::" + "::".join(tags) + fields.append(s) + else: + fields.append("") + # 4 + rfc_number = "" + if state == "rfc": + a = rfc_aliases.get(d.name) + if a: + rfc_number = a[3:] + fields.append(rfc_number) + # 5 + repl = "" + if state == "repl": + repl = replacements.get(d.name, "") + fields.append(repl) + # 6 + t = revision_time.get(d.name) + fields.append(t.strftime("%Y-%m-%d") if t else "") + # 7 + group_acronym = "" + if d.group and d.group.type_id != "area" and d.group.acronym != "none": + group_acronym = d.group.acronym + fields.append(group_acronym) + # 8 + area = "" + if d.group: + if d.group.type_id == "area": + area = d.group.acronym + elif d.group.type_id == "wg" and d.group.parent and d.group.parent.type_id == "area": + area = d.group.parent.acronym + fields.append(area) + # 9 responsible AD name + fields.append(unicode(d.ad) if d.ad else "") + # 10 + fields.append(d.intended_std_level.name if d.intended_std_level else "") + # 11 + lc_expires = "" + if iesg_state and iesg_state.slug == "lc": + e = d.latest_event(LastCallDocEvent, type="sent_last_call") + if e: + lc_expires = e.expires.strftime("%Y-%m-%d") + fields.append(lc_expires) + # 12 + fields.append(",".join(file_types.get(d.name + "-" + d.rev, "")) if state == "active" else "") + # 13 + fields.append(clean_whitespace(d.title)) # FIXME: we should make sure this is okay in the database and in submit + # 14 + fields.append(u", ".join(authors.get(d.name, []))) + # 15 + fields.append(d.shepherd.formatted_email().replace('"', '') if d.shepherd else "") + # 16 Responsible AD name and email + fields.append(d.ad.formatted_email().replace('"', '') if d.ad else "") + + # + res.append(u"\t".join(fields)) + + return render_to_string("idindex/all_id2.txt", {'data': u"\n".join(res) }) + +def active_drafts_index_by_group(extra_values=()): + """Return active drafts grouped into their corresponding + associated group, for spitting out draft index.""" + + # this returns a lot of data so try to be efficient + + active_state = State.objects.get(type="draft", slug="active") + + groups_dict = dict((g.id, g) for g in Group.objects.all()) + + extracted_values = ("name", "rev", "title", "group_id") + extra_values + + docs_dict = dict((d["name"], d) + for d in Document.objects.filter(states=active_state).values(*extracted_values)) + + # add initial and latest revision time + for time, doc_id in NewRevisionDocEvent.objects.filter(type="new_revision", doc__states=active_state).order_by('-time').values_list("time", "doc_id"): + d = docs_dict.get(doc_id) + if d: + if "rev_time" not in d: + d["rev_time"] = time + d["initial_rev_time"] = time + + # add authors + for a in DocumentAuthor.objects.filter(document__states=active_state).order_by("order").select_related("author__person"): + d = docs_dict.get(a.document_id) + if d: + if "authors" not in d: + d["authors"] = [] + d["authors"].append(unicode(a.author.person)) + + # put docs into groups + for d in docs_dict.itervalues(): + g = groups_dict.get(d["group_id"]) + if not g: + continue + + if not hasattr(g, "active_drafts"): + g.active_drafts = [] + + g.active_drafts.append(d) + + groups = [g for g in groups_dict.itervalues() if hasattr(g, "active_drafts")] + groups.sort(key=lambda g: g.acronym) + + fallback_time = datetime.datetime(1950, 1, 1) + for g in groups: + g.active_drafts.sort(key=lambda d: d.get("initial_rev_time", fallback_time)) + + return groups + +def id_index_txt(with_abstracts=False): + extra_values = () + if with_abstracts: + extra_values = ("abstract",) + groups = active_drafts_index_by_group(extra_values) + + file_types = file_types_for_drafts() + for g in groups: + for d in g.active_drafts: + # we need to output a multiple extension thing + types = file_types.get(d["name"] + "-" + d["rev"], "") + exts = ".txt" + if ".ps" in types: + exts += ",.ps" + if ".pdf" in types: + exts += ",.pdf" + d["exts"] = exts + + return render_to_string("idindex/id_index.txt", { + 'groups': groups, + 'time': datetime.datetime.now(pytz.UTC).strftime("%Y-%m-%d %H:%M:%S %Z"), + 'with_abstracts': with_abstracts, + }) diff --git a/ietf/idindex/models.py b/ietf/idindex/models.py index 7f1df9760..e69de29bb 100644 --- a/ietf/idindex/models.py +++ b/ietf/idindex/models.py @@ -1 +0,0 @@ -# Copyright The IETF Trust 2007, All Rights Reserved diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py index b21d781cc..198e08485 100644 --- a/ietf/idindex/tests.py +++ b/ietf/idindex/tests.py @@ -1,72 +1,146 @@ -# Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). -# All rights reserved. Contact: Pasi Eronen -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# * Neither the name of the Nokia Corporation and/or its -# subsidiary(-ies) nor the names of its contributors may be used -# to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import datetime, shutil -import unittest -import re -from django.test.client import Client -from ietf.utils.test_utils import SimpleUrlTestCase, RealDatabaseTest +import django.test +from django.core.urlresolvers import reverse as urlreverse -class IdIndexUrlTestCase(SimpleUrlTestCase): - def testUrls(self): - self.doTestUrls(__file__) +from ietf.utils.test_data import make_test_data -# class IndexTestCase(unittest.TestCase, RealDatabaseTest): -# def setUp(self): -# self.setUpRealDatabase() -# def tearDown(self): -# self.tearDownRealDatabase() +from ietf.doc.models import * +from ietf.idindex.index import * -# def testAllId(self): -# print " Testing all_id.txt generation" -# c = Client() -# response = c.get('/drafts/_test/all_id.txt') -# self.assertEquals(response.status_code, 200) -# content = response.content -# # Test that correct version number is shown for couple of old drafts -# self.assert_(content.find("draft-ietf-tls-psk-09") >= 0) -# self.assert_(content.find("draft-eronen-eap-sim-aka-80211-00") >= 0) -# # Since all_id.txt contains all old drafts, it should never shrink -# lines = content.split("\n") -# self.assert_(len(lines) > 18000) -# # Test that the lines look OK and have correct number of tabs -# r = re.compile(r'^(draft-\S*-\d\d)\t(\d\d\d\d-\d\d-\d\d)\t([^\t]+)\t([^\t]*)$') -# for line in lines: -# if ((line == "") or -# (line == "Internet-Drafts Status Summary") or -# (line == "Web version is available at") or -# (line == "https://datatracker.ietf.org/public/idindex.cgi")): -# pass -# elif r.match(line): -# pass -# else: -# self.fail("Unexpected line \""+line+"\"") -# print "OK (all_id.txt)" + +class IndexTestCase(django.test.TestCase): + fixtures = ['names'] + + def setUp(self): + self.id_dir = os.path.abspath("tmp-id-dir") + os.mkdir(self.id_dir) + settings.INTERNET_DRAFT_PATH = self.id_dir + + def tearDown(self): + shutil.rmtree(self.id_dir) + + def write_draft_file(self, name, size): + with open(os.path.join(self.id_dir, name), 'w') as f: + f.write("a" * size) + + def test_all_id_txt(self): + draft = make_test_data() + + # active in IESG process + draft.set_state(State.objects.get(type="draft", slug="active")) + draft.set_state(State.objects.get(type="draft-iesg", slug="lc")) + + txt = all_id_txt() + + self.assertTrue(draft.name + "-" + draft.rev in txt) + self.assertTrue(draft.get_state("draft-iesg").name in txt) + + # not active in IESG process + draft.unset_state("draft-iesg") + + txt = all_id_txt() + self.assertTrue(draft.name + "-" + draft.rev in txt) + self.assertTrue("Active" in txt) + + # published + draft.set_state(State.objects.get(type="draft", slug="rfc")) + DocAlias.objects.create(name="rfc1234", document=draft) + + txt = all_id_txt() + self.assertTrue(draft.name + "-" + draft.rev in txt) + self.assertTrue("RFC\t1234" in txt) + + # replaced + draft.set_state(State.objects.get(type="draft", slug="repl")) + + RelatedDocument.objects.create( + relationship=DocRelationshipName.objects.get(slug="replaces"), + source=Document.objects.create(type_id="draft", rev="00", name="draft-test-replacement"), + target=draft.docalias_set.get(name__startswith="draft")) + + txt = all_id_txt() + self.assertTrue(draft.name + "-" + draft.rev in txt) + self.assertTrue("Replaced replaced by draft-test-replacement" in txt) + + def test_all_id2_txt(self): + draft = make_test_data() + + def get_fields(content): + self.assertTrue(draft.name + "-" + draft.rev in content) + + for line in content.splitlines(): + if line.startswith(draft.name + "-" + draft.rev): + return line.split("\t") + # test Active + draft.set_state(State.objects.get(type="draft", slug="active")) + draft.set_state(State.objects.get(type="draft-iesg", slug="review-e")) + + NewRevisionDocEvent.objects.create(doc=draft, type="new_revision", rev=draft.rev, by=draft.ad) + + self.write_draft_file("%s-%s.txt" % (draft.name, draft.rev), 5000) + self.write_draft_file("%s-%s.pdf" % (draft.name, draft.rev), 5000) + + t = get_fields(all_id2_txt()) + self.assertEqual(t[0], draft.name + "-" + draft.rev) + self.assertEqual(t[1], "-1") + self.assertEqual(t[2], "Active") + self.assertEqual(t[3], "Expert Review") + self.assertEqual(t[4], "") + self.assertEqual(t[5], "") + self.assertEqual(t[6], draft.latest_event(type="new_revision").time.strftime("%Y-%m-%d")) + self.assertEqual(t[7], draft.group.acronym) + self.assertEqual(t[8], draft.group.parent.acronym) + self.assertEqual(t[9], unicode(draft.ad)) + self.assertEqual(t[10], draft.intended_std_level.name) + self.assertEqual(t[11], "") + self.assertEqual(t[12], ".txt,.pdf") + self.assertEqual(t[13], draft.title) + author = draft.documentauthor_set.order_by("order").get() + self.assertEqual(t[14], "%s <%s>" % (author.author.person.name, author.author.address)) + self.assertEqual(t[15], "%s <%s>" % (draft.shepherd, draft.shepherd.email_address())) + self.assertEqual(t[16], "%s <%s>" % (draft.ad, draft.ad.email_address())) + + + # test RFC + draft.set_state(State.objects.get(type="draft", slug="rfc")) + DocAlias.objects.create(name="rfc1234", document=draft) + t = get_fields(all_id2_txt()) + self.assertEqual(t[4], "1234") + + # test Replaced + draft.set_state(State.objects.get(type="draft", slug="repl")) + RelatedDocument.objects.create( + relationship=DocRelationshipName.objects.get(slug="replaces"), + source=Document.objects.create(type_id="draft", rev="00", name="draft-test-replacement"), + target=draft.docalias_set.get(name__startswith="draft")) + + t = get_fields(all_id2_txt()) + self.assertEqual(t[5], "draft-test-replacement") + + # test Last Call + draft.set_state(State.objects.get(type="draft", slug="active")) + draft.set_state(State.objects.get(type="draft-iesg", slug="lc")) + + e = LastCallDocEvent.objects.create(doc=draft, type="sent_last_call", expires=datetime.datetime.now() + datetime.timedelta(days=14), by=draft.ad) + + DocAlias.objects.create(name="rfc1234", document=draft) + t = get_fields(all_id2_txt()) + self.assertEqual(t[11], e.expires.strftime("%Y-%m-%d")) + + + def test_id_index_txt(self): + draft = make_test_data() + + draft.set_state(State.objects.get(type="draft", slug="active")) + + txt = id_index_txt() + + self.assertTrue(draft.name + "-" + draft.rev in txt) + self.assertTrue(draft.title in txt) + + self.assertTrue(draft.abstract[:20] not in txt) + + txt = id_index_txt(with_abstracts=True) + + self.assertTrue(draft.abstract[:20] in txt) diff --git a/ietf/idindex/testurl.list b/ietf/idindex/testurl.list deleted file mode 100644 index 5ba17e942..000000000 --- a/ietf/idindex/testurl.list +++ /dev/null @@ -1,20 +0,0 @@ -301 /drafts/wgid/1041/ -404 /drafts/wgid/987654/ -301 /drafts/wg/idr/ -301 /drafts/rfc/ -301 /drafts/current/ -301 /drafts/all/ -301 /drafts/dead/ -#301 /drafts/9574/related/ -#301 /drafts/9574/ -301 /drafts/draft-ietf-dnsext-dnssec-protocol/related/ -301 /drafts/draft-ietf-dnsext-dnssec-protocol/ -#404 /drafts/987654/ -301 /drafts/all_id_txt.html -301 /drafts/all_id.html -301 /drafts/ -#200,heavy /drafts/_test/all_id.txt -# this takes 3 minutes, so disabled by default -#200,heavy /drafts/_test/all_id2.txt -#200,heavy /drafts/_test/id_index.txt -#200,heavy /drafts/_test/id_abstracts.txt diff --git a/ietf/idindex/urls.py b/ietf/idindex/urls.py deleted file mode 100644 index 70f14777a..000000000 --- a/ietf/idindex/urls.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright The IETF Trust 2007, All Rights Reserved - -from django.conf import settings -from django.conf.urls.defaults import patterns -from ietf.idindex import views - -urlpatterns = patterns('', - (r'^$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/'}), - (r'^all/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/all/'}), - (r'^rfc/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/all/#rfc'}), - (r'^dead/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/all/#dead'}), - (r'^current/$', 'django.views.generic.simple.redirect_to', { 'url': '/doc/active/'}), - (r'^(?P\d+)/(related/)?$', views.redirect_id), - (r'^(?P[^/]+)/(related/)?$', views.redirect_filename), - (r'^wgid/(?P\d+)/$', views.wgdocs_redirect_id), - (r'^wg/(?P[^/]+)/$', views.wgdocs_redirect_acronym), - (r'^all_id(?:_txt)?.html$', 'django.views.generic.simple.redirect_to', { 'url': 'http://www.ietf.org/id/all_id.txt' }), -) - -if settings.SERVER_MODE != 'production': - # these haven't been ported - urlpatterns += patterns('', - (r'^_test/all_id.txt$', views.test_all_id_txt), - (r'^_test/all_id2.txt$', views.test_all_id2_txt), - (r'^_test/id_index.txt$', views.test_id_index_txt), - (r'^_test/id_abstracts.txt$', views.test_id_abstracts_txt) - ) diff --git a/ietf/idindex/views.py b/ietf/idindex/views.py deleted file mode 100644 index 584693733..000000000 --- a/ietf/idindex/views.py +++ /dev/null @@ -1,199 +0,0 @@ -# Copyright The IETF Trust 2007, All Rights Reserved - -# Portions Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies). -# All rights reserved. Contact: Pasi Eronen -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# * Neither the name of the Nokia Corporation and/or its -# subsidiary(-ies) nor the names of its contributors may be used -# to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from django.http import HttpResponse, HttpResponsePermanentRedirect -from django.template import loader -from django.shortcuts import get_object_or_404 -from django.conf import settings - -from ietf.idtracker.models import Acronym, IETFWG, InternetDraft, IDInternal,PersonOrOrgInfo, Area -from ietf.idtracker.templatetags.ietf_filters import clean_whitespace -import re -import sys -from datetime import datetime as Datetime -import pytz - -def all_id_txt(): - # we need a distinct to prevent the queries below from multiplying the result - all_ids = InternetDraft.objects.order_by('name').exclude(name__startswith="rfc").distinct() - - inactive_states = ["pub", "watching", "dead"] - - in_track_ids = all_ids.exclude(states__type="draft", states__slug="rfc").filter(states__type="draft-iesg").exclude(states__type="draft-iesg", states__slug__in=inactive_states) - not_in_track = all_ids.filter(states__type="draft", states__slug="rfc") | all_ids.exclude(states__type="draft-iesg") | all_ids.filter(states__type="draft-iesg", states__slug__in=inactive_states) - - active = not_in_track.filter(states__type="draft", states__slug="active") - published = not_in_track.filter(states__type="draft", states__slug="rfc") - expired = not_in_track.filter(states__type="draft", states__slug="expired") - withdrawn_submitter = not_in_track.filter(states__type="draft", states__slug="auth-rm") - withdrawn_ietf = not_in_track.filter(states__type="draft", states__slug="ietf-rm") - replaced = not_in_track.filter(states__type="draft", states__slug="repl") - - return loader.render_to_string("idindex/all_ids.txt", - { 'in_track_ids':in_track_ids, - 'active':active, - 'published':published, - 'expired':expired, - 'withdrawn_submitter':withdrawn_submitter, - 'withdrawn_ietf':withdrawn_ietf, - 'replaced':replaced}) - -def all_id2_entry(id): - fields = [] - # 0 - fields.append(id.filename+"-"+id.revision_display()) - # 1 - fields.append(-1) # this used to be id.id_document_tag, we don't have this identifier anymore - # 2 - status = str(id.get_state()) - fields.append(status) - # 3 - iesgstate = id.idstate() if status=="Active" else "" - fields.append(iesgstate) - # 4 - fields.append(id.rfc_number if status=="RFC" else "") - # 5 - try: - fields.append(id.replaced_by.filename) - except (AttributeError, InternetDraft.DoesNotExist): - fields.append("") - # 6 - fields.append(id.revision_date) - # 7 - group_acronym = "" if id.group.type_id == "area" else id.group.acronym - if group_acronym == "none": - group_acronym = "" - fields.append(group_acronym) - # 8 - area = "" - if id.group.type_id == "area": - area = id.group.acronym - elif id.group.type_id == "wg" and id.group.parent: - area = id.group.parent.acronym - fields.append(area) - # 9 responsible AD name - fields.append(id.idinternal.job_owner if id.idinternal else "") - # 10 - s = id.intended_status - if s and str(s) not in ("None","Request"): - fields.append(str(s)) - else: - fields.append("") - # 11 - if (iesgstate=="In Last Call") or iesgstate.startswith("In Last Call::"): - fields.append(id.lc_expiration_date) - else: - fields.append("") - # 12 - fields.append(id.file_type if status=="Active" else "") - # 13 - fields.append(clean_whitespace(id.title)) - # 14 - authors = [] - for author in sorted(id.authors.all(), key=lambda x: x.final_author_order()): - try: - realname = unicode(author.person) - email = author.email() or "" - name = re.sub(u"[<>@,]", u"", realname) + u" <"+re.sub(u"[<>,]", u"", email).strip()+u">" - authors.append(clean_whitespace(name)) - except PersonOrOrgInfo.DoesNotExist: - pass - fields.append(u", ".join(authors)) - # 15 - if id.shepherd: - shepherd = id.shepherd - realname = unicode(shepherd) - email = shepherd.email_address() - name = re.sub(u"[<>@,]", u"", realname) + u" <"+re.sub(u"[<>,]", u"", email).strip()+u">" - else: - name = u"" - fields.append(name) - # 16 Responsible AD name and email - if id.ad: - ad = id.ad - realname = unicode(ad) - email = ad.email_address() - name = re.sub(u"[<>@,]", u"", realname) + u" <"+re.sub(u"[<>,]", u"", email).strip()+u">" - else: - name = u"" - fields.append(name) - # - return "\t".join([unicode(x) for x in fields]) - -def all_id2_txt(): - all_ids = InternetDraft.objects.order_by('name').exclude(name__startswith="rfc").select_related('group', 'group__parent', 'ad') - data = "\n".join(all_id2_entry(id) for id in all_ids) - - return loader.render_to_string("idindex/all_id2.txt",{'data':data}) - -def id_index_txt(): - groups = IETFWG.objects.all() - return loader.render_to_string("idindex/id_index.txt", {'groups':groups}) - -def id_abstracts_txt(): - groups = IETFWG.objects.all() - return loader.render_to_string("idindex/id_abstracts.txt", {'groups':groups, 'time':Datetime.now(pytz.UTC).strftime("%Y-%m-%d %H:%M:%S %Z")}) - -def test_all_id_txt(request): - return HttpResponse(all_id_txt(), mimetype='text/plain') -def test_all_id2_txt(request): - return HttpResponse(all_id2_txt(), mimetype='text/plain') -def test_id_index_txt(request): - return HttpResponse(id_index_txt(), mimetype='text/plain') -def test_id_abstracts_txt(request): - return HttpResponse(id_abstracts_txt(), mimetype='text/plain') - -def redirect_id(request, object_id): - '''Redirect from historical document ID to preferred filename url.''' - if settings.USE_DB_REDESIGN_PROXY_CLASSES: - return HttpResponsePermanentRedirect("/doc/") - - doc = get_object_or_404(InternetDraft, id_document_tag=object_id) - return HttpResponsePermanentRedirect("/doc/"+doc.filename+"/") - -def redirect_filename(request, filename): - return HttpResponsePermanentRedirect("/doc/"+filename+"/") - -def wgdocs_redirect_id(request, id): - if settings.USE_DB_REDESIGN_PROXY_CLASSES: - from ietf.group.models import Group - group = get_object_or_404(Group, id=id) - return HttpResponsePermanentRedirect("/wg/%s/" % group.acronym) - - group = get_object_or_404(Acronym, acronym_id=id) - return HttpResponsePermanentRedirect("/wg/"+group.acronym+"/") - -def wgdocs_redirect_acronym(request, acronym): - return HttpResponsePermanentRedirect("/wg/"+acronym+"/") - diff --git a/ietf/idrfc/views_search.py b/ietf/idrfc/views_search.py index f2a021832..20b3a8711 100644 --- a/ietf/idrfc/views_search.py +++ b/ietf/idrfc/views_search.py @@ -46,6 +46,7 @@ from ietf.doc.models import * from ietf.person.models import * from ietf.group.models import * from ietf.ipr.models import IprDocAlias +from ietf.idindex.index import active_drafts_index_by_group class SearchForm(forms.Form): name = forms.CharField(required=False) @@ -416,7 +417,8 @@ def index_all_drafts(request): names.sort(key=lambda t: t[1]) - names = ['' + name +'' for name, _ in names if name not in names_to_skip] + names = ['' + name +'' + for name, _ in names if name not in names_to_skip] categories.append((state, heading, @@ -427,45 +429,6 @@ def index_all_drafts(request): context_instance=RequestContext(request)) def index_active_drafts(request): - # try to be efficient since this view returns a lot of data - - active_state = State.objects.get(type="draft", slug="active") - - groups_dict = dict((g.id, g) for g in Group.objects.all()) - - docs_dict = dict((d["name"], d) - for d in Document.objects.filter(states=active_state).values("name", "rev", "title", "group_id")) - - # add latest revision time - for time, doc_id in NewRevisionDocEvent.objects.filter(type="new_revision", doc__states=active_state).order_by('-time').values_list("time", "doc_id"): - d = docs_dict.get(doc_id) - if d and "rev_time" not in d: - d["rev_time"] = time - - # add authors - for a in DocumentAuthor.objects.filter(document__states=active_state).order_by("order").select_related("author__person"): - d = docs_dict.get(a.document_id) - if d: - if "authors" not in d: - d["authors"] = [] - d["authors"].append(unicode(a.author.person)) - - # put docs into groups - for d in docs_dict.itervalues(): - g = groups_dict.get(d["group_id"]) - if not g: - continue - - if not hasattr(g, "active_drafts"): - g.active_drafts = [] - - g.active_drafts.append(d) - - groups = [g for g in groups_dict.itervalues() if hasattr(g, "active_drafts")] - groups.sort(key=lambda g: g.acronym) - - fallback_time = datetime.datetime(1990, 1, 1) - for g in groups: - g.active_drafts.sort(key=lambda d: d.get("rev_time", fallback_time)) + groups = active_drafts_index_by_group() return render_to_response("doc/index_active_drafts.html", { 'groups': groups }, context_instance=RequestContext(request)) diff --git a/ietf/idtracker/templatetags/ietf_filters.py b/ietf/idtracker/templatetags/ietf_filters.py index 3601ea9e7..b788e702d 100644 --- a/ietf/idtracker/templatetags/ietf_filters.py +++ b/ietf/idtracker/templatetags/ietf_filters.py @@ -31,9 +31,18 @@ def expand_comma(value): def format_charter(value): return value.replace("\n\n", "

").replace("\n","
\n") -@register.filter(name='indent') -def indent(value,numspaces=2): - return value.replace("\n", "\n"+" "*int(numspaces)); +@register.filter +def indent(value, numspaces=2): + replacement = "\n" + " " * int(numspaces) + res = value.replace("\n", replacement) + if res.endswith(replacement): + res = res[:-int(numspaces)] # fix up superfluous spaces + return res + +@register.filter +def unindent(value): + """Remove indentation from string.""" + return re.sub("\n +", "\n", value) @register.filter(name='parse_email_list') def parse_email_list(value): @@ -240,6 +249,11 @@ def dashify(string): """ return re.sub('.', '-', string) +@register.filter +def underline(string): + """Return string with an extra line underneath of dashes, for plain text underlining.""" + return string + "\n" + ("-" * len(string)) + @register.filter(name='lstrip') def lstripw(string, chars): """Strip matching leading characters from words in string""" @@ -319,23 +333,6 @@ def wrap_text(text, width=72): prev_indent = indent return "\n".join(filled) -@register.filter(name="id_index_file_types") -def id_index_file_types(text): - r = ".txt" - if text.find("txt") < 0: - return r - if text.find("ps") >= 0: - r = r + ",.ps" - if text.find("pdf") >= 0: - r = r + ",.pdf" - return r - -@register.filter(name="id_index_wrap") -def id_index_wrap(text): - x = wordwrap(text, 72) - x = x.replace("\n", "\n ") - return " "+x.strip() - @register.filter(name="compress_empty_lines") def compress_empty_lines(text): text = re.sub("( *\n){3,}", "\n\n", text) diff --git a/ietf/templates/idindex/all_id2.txt b/ietf/templates/idindex/all_id2.txt index 3addadb74..5d810ab96 100644 --- a/ietf/templates/idindex/all_id2.txt +++ b/ietf/templates/idindex/all_id2.txt @@ -4,8 +4,7 @@ # # Description of fields: # 0 draft name and latest revision -# 1 id_document_tag (internal database identifier; avoid using -# unless you really need it) +# 1 always -1 (was internal numeric database id in earlier schema) # 2 one of "Active", "Expired", "RFC", "Withdrawn by Submitter", # "Replaced", or "Withdrawn by IETF" # 3 if #2 is "Active", the IESG state for the document (such as diff --git a/ietf/templates/idindex/all_ids.txt b/ietf/templates/idindex/all_ids.txt deleted file mode 100644 index 0daf8710c..000000000 --- a/ietf/templates/idindex/all_ids.txt +++ /dev/null @@ -1,17 +0,0 @@ - -Internet-Drafts Status Summary - -{% for item in in_track_ids %}{{ item.filename }}-{{ item.revision_display }} {{ item.revision_date|default_if_none:"" }} In IESG processing - ID Tracker state <{{ item.idstate }}> {# that last tab is on purpose #} -{% endfor %}{%comment%} -{%endcomment%}{% for item in active %}{{ item.filename }}-{{ item.revision_display }} {{ item.revision_date|default_if_none:"" }} {{ item.status.status }} {# keep that last tab #} -{% endfor %}{%comment%} -{%endcomment%}{% for item in published %}{{ item.filename }}-{{ item.revision_display }} {{ item.revision_date|default_if_none:"" }} {{ item.status.status }} {{ item.rfc_number }} -{% endfor %}{%comment%} -{%endcomment%}{% for item in expired %}{{ item.filename }}-{{ item.revision_display }} {{ item.revision_date|default_if_none:"" }} {{ item.status.status }} {# keep that last tab #} -{% endfor %}{%comment%} -{%endcomment%}{% for item in withdrawn_submitter %}{{ item.filename }}-{{ item.revision_display }} {{ item.revision_date|default_if_none:"" }} {{ item.status.status }} {# keep that last tab #} -{% endfor %}{%comment%} -{%endcomment%}{% for item in withdrawn_ietf %}{{ item.filename }}-{{ item.revision_display }} {{ item.revision_date|default_if_none:"" }} {{ item.status.status }} {# keep that last tab #} -{% endfor %}{%comment%} -{%endcomment%}{% for item in replaced %}{{ item.filename }}-{{ item.revision_display }} {{ item.revision_date|default_if_none:"" }} {{ item.status.status }} replaced by {% if item.replaced_by_id %}{{ item.replaced_by.filename }}{% else %}0{% endif %} {# and this one needs the trailing tab as well #} -{% endfor %} diff --git a/ietf/templates/idindex/id_abstracts.txt b/ietf/templates/idindex/id_abstracts.txt deleted file mode 100644 index d225369f5..000000000 --- a/ietf/templates/idindex/id_abstracts.txt +++ /dev/null @@ -1,9 +0,0 @@ -{% extends "idindex/id_index.txt" %}{% load ietf_filters %} {% block intro %} Current Internet-Drafts - - This summary sheet provides a short synopsis of each Internet-Draft -available within the "internet-drafts" directory at the shadow -sites directory. These drafts are listed alphabetically by working -group acronym and start date. Generated {{ time }} -{% endblock %}{% block abstract %} - - {{ draft.clean_abstract|indent|indent|safe }}{% endblock %} diff --git a/ietf/templates/idindex/id_index.txt b/ietf/templates/idindex/id_index.txt index eca586397..2c18708d9 100644 --- a/ietf/templates/idindex/id_index.txt +++ b/ietf/templates/idindex/id_index.txt @@ -1,13 +1,12 @@ -{% autoescape off %}{% load ietf_filters %}{% block intro %} Current Internet-Drafts - This summary sheet provides an index of each Internet-Draft -These drafts are listed alphabetically by Working Group acronym and -initial post date. -{% endblock %} -{% for group in groups|dictsort:"group_acronym.acronym" %}{% if group.active_drafts %} -{{ group.group_acronym.name }} ({{ group.group_acronym.acronym}}) -{% filter dashify %}{{ group.group_acronym.name }} ({{ group.group_acronym.acronym}}){% endfilter %} -{% for draft in group.active_drafts|stable_dictsort:"filename"|stable_dictsort:"start_date" %} -{% filter id_index_wrap %} -"{{draft.title.strip|clean_whitespace}}", {% for author in draft.authors.all|dictsort:"final_author_order" %}{{author.person}}, {% endfor %}{{draft.revision_date|date:"j-M-y"}}, <{{draft.filename}}-{{draft.revision}}{{draft.file_type|id_index_file_types}}> -{% endfilter %}{% block abstract %}{% endblock %} -{% endfor %}{%endif %}{% endfor %}{% endautoescape %} +{% autoescape off %}{% load ietf_filters %} Current Internet-Drafts +This summary sheet provides an index of each Internet-Draft. These +drafts are listed alphabetically by Working Group acronym and initial +post date. Generated {{ time }}. + +{% for group in groups %} +{% filter underline %}{{ group.name }} ({{ group.acronym }}){% endfilter %} +{% for d in group.active_drafts %} + {% filter wordwrap:72|indent:2 %}"{{ d.title|clean_whitespace }}", {% for a in d.authors %}{{ a }}, {% endfor %}{{ d.rev_time|date:"Y-m-d"}}, <{{ d.name }}-{{ d.rev }}{{ d.exts }}> +{% endfilter %}{% if with_abstracts %} + + {{ d.abstract.strip|unindent|fill:72|indent:6 }}{% endif %}{% endfor %}{% endfor %}{% endautoescape %} diff --git a/ietf/urls.py b/ietf/urls.py index a0f17452c..13e793a3b 100644 --- a/ietf/urls.py +++ b/ietf/urls.py @@ -51,7 +51,7 @@ urlpatterns = patterns('', (r'^community/', include('ietf.community.urls')), (r'^cookies/', include('ietf.cookies.urls')), (r'^doc/', include('ietf.idrfc.urls')), - (r'^drafts/', include('ietf.idindex.urls')), + (r'^drafts/', include('ietf.doc.redirect_drafts_urls')), (r'^feed/(?P.*)/$', 'django.contrib.syndication.views.feed', { 'feed_dict': feeds}), (r'^idtracker/', include('ietf.idtracker.urls')), (r'^iesg/', include('ietf.iesg.urls')), diff --git a/ietf/utils/test_data.py b/ietf/utils/test_data.py index 4ab4d8553..97bd42edf 100644 --- a/ietf/utils/test_data.py +++ b/ietf/utils/test_data.py @@ -399,6 +399,7 @@ def make_test_data(): docalias = DocAlias.objects.create(name=doc.name, document=doc) doc.stream = StreamName.objects.get(slug='irtf') doc.save() + doc.set_state(State.objects.get(type="draft", slug="active")) crdoc = Document.objects.create(name='conflict-review-imaginary-irtf-submission', type_id='conflrev', rev='00', notify="fsm@ietf.org") DocAlias.objects.create(name=crdoc.name, document=crdoc) crdoc.set_state(State.objects.get(name='Needs Shepherd', type__slug='conflrev'))