Merged in [8970] from olau@iola.dk:

Summary: Enable support for a custom settings file for the test crawl
- and add a simple settings file that enables caching to speed up the
crawl (shaves ~35% off most pages AFAICT)
 - Legacy-Id: 8984
Note: SVN reference [8970] has been migrated to Git commit 8d4e1a8528

bin/test-crawl

@@ -2,23 +2,6 @@
 
 import os, sys, re, datetime, argparse, traceback, tempfile
 
-# boilerplate
-basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
-sys.path = [ basedir ] + sys.path
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings")
-
-import django.test
-from django.conf import settings
-
-# prevent memory from leaking when settings.DEBUG=True
-from django.db import connection
-class DontSaveQueries(object):
-    def append(self, x):
-        pass
-connection.queries = DontSaveQueries()
-
-MAX_URL_LENGTH = 500
-
 # args
 parser = argparse.ArgumentParser(
     description="""Perform a test crawl of the project. For each found URL, the HTTP
@@ -30,9 +13,29 @@ parser.add_argument('--urls', '-u', dest='url_file',
                     help='file with URLs to start the crawl from')
 parser.add_argument('--slow', dest='slow_threshold', type=float, default=1.0,
                     help='responses taking longer than this (in seconds) results in SLOW being printed')
+parser.add_argument('--settings', dest='settings', help='custom settings file')
 
 args = parser.parse_args()
 
+# boilerplate
+basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+sys.path = [ basedir ] + sys.path
+
+settings_module = args.settings or "ietf.settings"
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_module)
+
+import django.test
+
+# prevent memory from leaking when settings.DEBUG=True
+from django.db import connection
+class DontSaveQueries(object):
+    def append(self, x):
+        pass
+connection.queries = DontSaveQueries()
+
+MAX_URL_LENGTH = 500
+
 slow_threshold = args.slow_threshold
 
 initial_urls = []

ietf/settings_testcrawl.py

@@ -0,0 +1,22 @@
+# Standard settings except we enable caching like in the production
+# environment, this is useful for speeding up the test crawl, try for
+# instance:
+#
+#   bin/test-crawl --settings=ietf.settings_testcrawl
+#
+
+from settings import *                  # pyflakes:ignore
+
+TEMPLATE_LOADERS = (
+    ('django.template.loaders.cached.Loader', (
+        'django.template.loaders.filesystem.Loader',
+        'django.template.loaders.app_directories.Loader',
+    )),
+    'ietf.dbtemplate.template.Loader',
+)
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+    }
+}