aa5e61d958
in order to autogenerate dotted path url pattern names. Updated a number of url reverses to use dotted path, and removed explicit url pattern names as needed. Changed some imports to prevent import of ietf.urls before django initialization was complete. Changed 3 cases of form classes being curried to functions; django 1.10 didn't accept that. Started converting old-style middleware classes to new-style middleware functions (incomplete). Tweaked a nomcom decorator to preserve function names and attributes, like a good decorator should. Replaced the removed django templatetag 'removetags' with our own version which uses bleach, and does sanitizing in addition to removing explicitly mentionied html tags. Rewrote the filename argument handling in a management command which had broken with the upgrade. - Legacy-Id: 12818
567 lines
25 KiB
Python
567 lines
25 KiB
Python
# Copyright The IETF Trust 2007, All Rights Reserved
|
|
|
|
# Portion Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
|
|
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
#
|
|
# * Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
#
|
|
# * Redistributions in binary form must reproduce the above
|
|
# copyright notice, this list of conditions and the following
|
|
# disclaimer in the documentation and/or other materials provided
|
|
# with the distribution.
|
|
#
|
|
# * Neither the name of the Nokia Corporation and/or its
|
|
# subsidiary(-ies) nor the names of its contributors may be used
|
|
# to endorse or promote products derived from this software
|
|
# without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
|
import re
|
|
import os
|
|
import sys
|
|
import time
|
|
import json
|
|
import pytz
|
|
import importlib
|
|
import socket
|
|
import datetime
|
|
import codecs
|
|
import gzip
|
|
import unittest
|
|
from fnmatch import fnmatch
|
|
|
|
from coverage.report import Reporter
|
|
from coverage.results import Numbers
|
|
from coverage.misc import NotPython
|
|
|
|
from django.conf import settings
|
|
from django.template import TemplateDoesNotExist
|
|
from django.template.loaders.base import Loader as BaseLoader
|
|
from django.test.runner import DiscoverRunner
|
|
from django.core.management import call_command
|
|
from django.core.urlresolvers import RegexURLResolver
|
|
|
|
import debug # pyflakes:ignore
|
|
debug.debug = True
|
|
|
|
import ietf
|
|
import ietf.utils.mail
|
|
from ietf.utils.test_smtpserver import SMTPTestServerDriver
|
|
from ietf.utils.test_utils import TestCase
|
|
|
|
loaded_templates = set()
|
|
visited_urls = set()
|
|
test_database_name = None
|
|
old_destroy = None
|
|
old_create = None
|
|
|
|
template_coverage_collection = None
|
|
url_coverage_collection = None
|
|
|
|
|
|
def safe_create_test_db(self, verbosity, *args, **kwargs):
|
|
global test_database_name, old_create
|
|
keepdb = kwargs.get('keepdb', False)
|
|
if not keepdb:
|
|
print " Creating test database..."
|
|
if settings.DATABASES["default"]["ENGINE"] == 'django.db.backends.mysql':
|
|
settings.DATABASES["default"]["OPTIONS"] = settings.DATABASE_TEST_OPTIONS
|
|
print " Using OPTIONS: %s" % settings.DATABASES["default"]["OPTIONS"]
|
|
test_database_name = old_create(self, 0, *args, **kwargs)
|
|
|
|
if settings.GLOBAL_TEST_FIXTURES:
|
|
print " Loading global test fixtures: %s" % ", ".join(settings.GLOBAL_TEST_FIXTURES)
|
|
loadable = [f for f in settings.GLOBAL_TEST_FIXTURES if "." not in f]
|
|
call_command('loaddata', *loadable, verbosity=int(verbosity)-1, commit=False, database="default")
|
|
|
|
for f in settings.GLOBAL_TEST_FIXTURES:
|
|
if f not in loadable:
|
|
# try to execute the fixture
|
|
components = f.split(".")
|
|
module_name = ".".join(components[:-1])
|
|
module = importlib.import_module(module_name)
|
|
fn = getattr(module, components[-1])
|
|
fn()
|
|
|
|
return test_database_name
|
|
|
|
def safe_destroy_test_db(*args, **kwargs):
|
|
global test_database_name, old_destroy
|
|
keepdb = kwargs.get('keepdb', False)
|
|
if not keepdb:
|
|
print " Checking that it's safe to destroy test database..."
|
|
if settings.DATABASES["default"]["NAME"] != test_database_name:
|
|
print ' NOT SAFE; Changing settings.DATABASES["default"]["NAME"] from %s to %s' % (settings.DATABASES["default"]["NAME"], test_database_name)
|
|
settings.DATABASES["default"]["NAME"] = test_database_name
|
|
return old_destroy(*args, **kwargs)
|
|
|
|
class TemplateCoverageLoader(BaseLoader):
|
|
is_usable = True
|
|
|
|
def load_template_source(self, template_name, dirs):
|
|
global template_coverage_collection, loaded_templates
|
|
if template_coverage_collection == True:
|
|
loaded_templates.add(str(template_name))
|
|
raise TemplateDoesNotExist(template_name)
|
|
load_template_source.is_usable = True
|
|
|
|
class RecordUrlsMiddleware(object):
|
|
def process_request(self, request):
|
|
global url_coverage_collection, visited_urls
|
|
if url_coverage_collection == True:
|
|
visited_urls.add(request.path)
|
|
|
|
def get_url_patterns(module, apps=None):
|
|
def include(name):
|
|
if not apps:
|
|
return True
|
|
for app in apps:
|
|
if name.startswith(app+'.'):
|
|
return True
|
|
return False
|
|
def exclude(name):
|
|
for pat in settings.TEST_URL_COVERAGE_EXCLUDE:
|
|
if re.search(pat, name):
|
|
return True
|
|
return False
|
|
if not hasattr(module, 'urlpatterns'):
|
|
return []
|
|
res = []
|
|
for item in module.urlpatterns:
|
|
if isinstance(item, RegexURLResolver) and not type(item.urlconf_module) is list:
|
|
if include(item.urlconf_module.__name__) and not exclude(item.regex.pattern):
|
|
subpatterns = get_url_patterns(item.urlconf_module)
|
|
for sub, subitem in subpatterns:
|
|
if sub.startswith("^"):
|
|
res.append((item.regex.pattern + sub[1:], subitem))
|
|
else:
|
|
res.append((item.regex.pattern + ".*" + sub, subitem))
|
|
else:
|
|
res.append((item.regex.pattern, item))
|
|
return res
|
|
|
|
_all_templates = None
|
|
def get_template_paths(apps=None):
|
|
global _all_templates
|
|
if not _all_templates:
|
|
# TODO: Add app templates to the full list, if we are using
|
|
# django.template.loaders.app_directories.Loader
|
|
templates = set()
|
|
templatepaths = settings.TEMPLATES[0]['DIRS']
|
|
for templatepath in templatepaths:
|
|
for dirpath, dirs, files in os.walk(templatepath):
|
|
if ".svn" in dirs:
|
|
dirs.remove(".svn")
|
|
relative_path = dirpath[len(templatepath)+1:]
|
|
for file in files:
|
|
ignore = False
|
|
for pattern in settings.TEST_TEMPLATE_IGNORE:
|
|
if fnmatch(file, pattern):
|
|
ignore = True
|
|
break
|
|
if ignore:
|
|
continue
|
|
if relative_path != "":
|
|
file = os.path.join(relative_path, file)
|
|
templates.add(file)
|
|
if apps:
|
|
templates = [ t for t in templates if t.split(os.path.sep)[0] in apps ]
|
|
_all_templates = templates
|
|
return _all_templates
|
|
|
|
def save_test_results(failures, test_labels):
|
|
# Record the test result in a file, in order to be able to check the
|
|
# results and avoid re-running tests if we've alread run them with OK
|
|
# result after the latest code changes:
|
|
topdir = os.path.dirname(os.path.dirname(settings.BASE_DIR))
|
|
tfile = codecs.open(os.path.join(topdir,".testresult"), "a", encoding='utf-8')
|
|
timestr = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
if failures:
|
|
tfile.write("%s FAILED (failures=%s)\n" % (timestr, failures))
|
|
else:
|
|
if test_labels:
|
|
tfile.write("%s SUCCESS (tests=%s)\n" % (timestr, test_labels))
|
|
else:
|
|
tfile.write("%s OK\n" % (timestr, ))
|
|
tfile.close()
|
|
|
|
|
|
def set_coverage_checking(flag=True):
|
|
global template_coverage_collection
|
|
global url_coverage_collection
|
|
if flag:
|
|
settings.TEST_CODE_COVERAGE_CHECKER.collector.resume()
|
|
template_coverage_collection = True
|
|
url_coverage_collection = True
|
|
else:
|
|
settings.TEST_CODE_COVERAGE_CHECKER.collector.pause()
|
|
template_coverage_collection = False
|
|
url_coverage_collection = False
|
|
|
|
class CoverageReporter(Reporter):
|
|
def report(self):
|
|
self.find_file_reporters(None)
|
|
|
|
total = Numbers()
|
|
result = {"coverage": 0.0, "covered": {}, "format": 2, }
|
|
for fr in self.file_reporters:
|
|
try:
|
|
analysis = self.coverage._analyze(fr)
|
|
nums = analysis.numbers
|
|
result["covered"][fr.relative_filename()] = (nums.n_statements, nums.pc_covered/100.0)
|
|
total += nums
|
|
except KeyboardInterrupt: # pragma: not covered
|
|
raise
|
|
except Exception:
|
|
report_it = not self.config.ignore_errors
|
|
if report_it:
|
|
typ, msg = sys.exc_info()[:2]
|
|
if typ is NotPython and not fr.should_be_python():
|
|
report_it = False
|
|
if report_it:
|
|
raise
|
|
result["coverage"] = total.pc_covered/100.0
|
|
return result
|
|
|
|
|
|
class CoverageTest(TestCase):
|
|
|
|
def __init__(self, test_runner=None, **kwargs):
|
|
self.runner = test_runner
|
|
super(CoverageTest, self).__init__(**kwargs)
|
|
|
|
def report_test_result(self, test):
|
|
latest_coverage_version = self.runner.coverage_master["version"]
|
|
|
|
master_data = self.runner.coverage_master[latest_coverage_version][test]
|
|
master_missing = [ k for k,v in master_data["covered"].items() if not v ]
|
|
master_coverage = master_data["coverage"]
|
|
|
|
test_data = self.runner.coverage_data[test]
|
|
test_missing = [ k for k,v in test_data["covered"].items() if not v ]
|
|
test_coverage = test_data["coverage"]
|
|
|
|
# Assert coverage failure only if we're running the full test suite -- if we're
|
|
# only running some tests, then of course the coverage is going to be low.
|
|
if self.runner.run_full_test_suite:
|
|
# Permit 0.02% variation in results -- otherwise small code changes become a pain
|
|
fudge_factor = 0.0002
|
|
self.assertGreaterEqual(test_coverage, master_coverage - fudge_factor,
|
|
msg = "The %s coverage percentage is now lower (%.2f%%) than for version %s (%.2f%%)" %
|
|
( test, test_coverage*100, latest_coverage_version, master_coverage*100, ))
|
|
self.assertLessEqual(len(test_missing), len(master_missing),
|
|
msg = "New %s without test coverage since %s: %s" % (test, latest_coverage_version, list(set(test_missing) - set(master_missing))))
|
|
|
|
def template_coverage_test(self):
|
|
global loaded_templates
|
|
if self.runner.check_coverage:
|
|
apps = [ app.split('.')[-1] for app in self.runner.test_apps ]
|
|
all = get_template_paths(apps)
|
|
# The calculations here are slightly complicated by the situation
|
|
# that loaded_templates also contain nomcom page templates loaded
|
|
# from the database. However, those don't appear in all
|
|
covered = [ k for k in all if k in loaded_templates ]
|
|
self.runner.coverage_data["template"] = {
|
|
"coverage": (1.0*len(covered)/len(all)) if len(all)>0 else float('nan'),
|
|
"covered": dict( (k, k in covered) for k in all ),
|
|
}
|
|
self.report_test_result("template")
|
|
else:
|
|
self.skipTest("Coverage switched off with --skip-coverage")
|
|
|
|
def url_coverage_test(self):
|
|
if self.runner.check_coverage:
|
|
import ietf.urls
|
|
url_patterns = get_url_patterns(ietf.urls, self.runner.test_apps)
|
|
|
|
# skip some patterns that we don't bother with
|
|
def ignore_pattern(regex, pattern):
|
|
import django.views.static
|
|
return (regex in ("^_test500/$", "^accounts/testemail/$")
|
|
or regex.startswith("^admin/")
|
|
or getattr(pattern.callback, "__name__", "") == "RedirectView"
|
|
or getattr(pattern.callback, "__name__", "") == "TemplateView"
|
|
or pattern.callback == django.views.static.serve)
|
|
|
|
patterns = [(regex, re.compile(regex)) for regex, pattern in url_patterns
|
|
if not ignore_pattern(regex, pattern)]
|
|
all = [ regex for regex, compiled in patterns ]
|
|
|
|
covered = set()
|
|
for url in visited_urls:
|
|
for regex, compiled in patterns:
|
|
if regex not in covered and compiled.match(url[1:]): # strip leading /
|
|
covered.add(regex)
|
|
break
|
|
|
|
self.runner.coverage_data["url"] = {
|
|
"coverage": 1.0*len(covered)/len(all),
|
|
"covered": dict( (k, k in covered) for k in all ),
|
|
}
|
|
|
|
self.report_test_result("url")
|
|
else:
|
|
self.skipTest("Coverage switched off with --skip-coverage")
|
|
|
|
def code_coverage_test(self):
|
|
if self.runner.check_coverage:
|
|
include = [ os.path.join(path, '*') for path in self.runner.test_paths ]
|
|
checker = self.runner.code_coverage_checker
|
|
checker.stop()
|
|
# Save to the .coverage file
|
|
checker.save()
|
|
# Apply the configured and requested omit and include data
|
|
checker.config.from_args(ignore_errors=None, omit=settings.TEST_CODE_COVERAGE_EXCLUDE,
|
|
include=include, file=None)
|
|
# Maybe output a html report
|
|
if self.runner.run_full_test_suite:
|
|
checker.html_report(directory=settings.TEST_CODE_COVERAGE_REPORT_DIR)
|
|
# In any case, build a dictionary with per-file data for this run
|
|
reporter = CoverageReporter(checker, checker.config)
|
|
self.runner.coverage_data["code"] = reporter.report()
|
|
self.report_test_result("code")
|
|
else:
|
|
self.skipTest("Coverage switched off with --skip-coverage")
|
|
|
|
class IetfTestRunner(DiscoverRunner):
|
|
|
|
@classmethod
|
|
def add_arguments(cls, parser):
|
|
super(IetfTestRunner, cls).add_arguments(parser)
|
|
parser.add_argument('--skip-coverage',
|
|
action='store_true', dest='skip_coverage', default=False,
|
|
help='Skip test coverage measurements for code, templates, and URLs. ' )
|
|
parser.add_argument('--save-version-coverage',
|
|
action='store', dest='save_version_coverage', default=False,
|
|
help='Save test coverage data under the given version label')
|
|
parser.add_argument('--save-testresult',
|
|
action='store_true', dest='save_testresult', default=False,
|
|
help='Save short test result data in %s/.testresult' % os.path.dirname(os.path.dirname(settings.BASE_DIR))),
|
|
|
|
def __init__(self, skip_coverage=False, save_version_coverage=None, **kwargs):
|
|
#
|
|
self.check_coverage = not skip_coverage
|
|
self.save_version_coverage = save_version_coverage
|
|
#
|
|
self.root_dir = os.path.dirname(settings.BASE_DIR)
|
|
self.coverage_file = os.path.join(self.root_dir, settings.TEST_COVERAGE_MASTER_FILE)
|
|
super(IetfTestRunner, self).__init__(**kwargs)
|
|
|
|
def setup_test_environment(self, **kwargs):
|
|
global template_coverage_collection
|
|
global url_coverage_collection
|
|
|
|
ietf.utils.mail.test_mode = True
|
|
ietf.utils.mail.SMTP_ADDR['ip4'] = '127.0.0.1'
|
|
ietf.utils.mail.SMTP_ADDR['port'] = 2025
|
|
# switch to a much faster hasher
|
|
settings.PASSWORD_HASHERS = ( 'django.contrib.auth.hashers.MD5PasswordHasher', )
|
|
settings.SERVER_MODE = 'test'
|
|
#
|
|
if self.check_coverage:
|
|
if self.coverage_file.endswith('.gz'):
|
|
with gzip.open(self.coverage_file, "rb") as file:
|
|
self.coverage_master = json.load(file)
|
|
else:
|
|
with codecs.open(self.coverage_file, encoding='utf-8') as file:
|
|
self.coverage_master = json.load(file)
|
|
self.coverage_data = {
|
|
"time": datetime.datetime.now(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"template": {
|
|
"coverage": 0.0,
|
|
"covered": {},
|
|
"format": 1, # default format, coverage data in 'covered' are just fractions
|
|
},
|
|
"url": {
|
|
"coverage": 0.0,
|
|
"covered": {},
|
|
"format": 1,
|
|
},
|
|
"code": {
|
|
"coverage": 0.0,
|
|
"covered": {},
|
|
"format": 1,
|
|
},
|
|
}
|
|
|
|
settings.TEMPLATES[0]['OPTIONS']['loaders'] = ('ietf.utils.test_runner.TemplateCoverageLoader',) + settings.TEMPLATES[0]['OPTIONS']['loaders']
|
|
|
|
settings.MIDDLEWARE_CLASSES = ('ietf.utils.test_runner.RecordUrlsMiddleware',) + tuple(settings.MIDDLEWARE_CLASSES)
|
|
url_coverage_collection = True
|
|
|
|
self.code_coverage_checker = settings.TEST_CODE_COVERAGE_CHECKER
|
|
if not self.code_coverage_checker._started:
|
|
sys.stderr.write(" ** Warning: In %s: Expected the coverage checker to have\n"
|
|
" been started already, but it wasn't. Doing so now. Coverage numbers\n"
|
|
" will be off, though.\n" % __name__)
|
|
self.code_coverage_checker.start()
|
|
|
|
if settings.SITE_ID != 1:
|
|
print " Changing SITE_ID to '1' during testing."
|
|
settings.SITE_ID = 1
|
|
|
|
if settings.TEMPLATES[0]['OPTIONS']['string_if_invalid'] != '':
|
|
print(" Changing TEMPLATES[0]['OPTIONS']['string_if_invalid'] to '' during testing")
|
|
settings.TEMPLATES[0]['OPTIONS']['string_if_invalid'] = ''
|
|
|
|
assert not settings.IDTRACKER_BASE_URL.endswith('/')
|
|
|
|
# Try to set up an SMTP test server. In case other test runs are
|
|
# going on at the same time, try a range of ports.
|
|
base = ietf.utils.mail.SMTP_ADDR['port']
|
|
for offset in range(10):
|
|
try:
|
|
# remember the value so ietf.utils.mail.send_smtp() will use the same
|
|
ietf.utils.mail.SMTP_ADDR['port'] = base + offset
|
|
self.smtpd_driver = SMTPTestServerDriver((ietf.utils.mail.SMTP_ADDR['ip4'],ietf.utils.mail.SMTP_ADDR['port']),None)
|
|
self.smtpd_driver.start()
|
|
print(" Running an SMTP test server on %(ip4)s:%(port)s to catch outgoing email." % ietf.utils.mail.SMTP_ADDR)
|
|
break
|
|
except socket.error:
|
|
pass
|
|
|
|
|
|
super(IetfTestRunner, self).setup_test_environment(**kwargs)
|
|
|
|
def teardown_test_environment(self, **kwargs):
|
|
self.smtpd_driver.stop()
|
|
if self.check_coverage:
|
|
latest_coverage_file = os.path.join(self.root_dir, settings.TEST_COVERAGE_LATEST_FILE)
|
|
coverage_latest = {}
|
|
coverage_latest["version"] = "latest"
|
|
coverage_latest["latest"] = self.coverage_data
|
|
with codecs.open(latest_coverage_file, "w", encoding='utf-8') as file:
|
|
json.dump(coverage_latest, file, indent=2, sort_keys=True)
|
|
if self.save_version_coverage:
|
|
self.coverage_master["version"] = self.save_version_coverage
|
|
self.coverage_master[self.save_version_coverage] = self.coverage_data
|
|
if self.coverage_file.endswith('.gz'):
|
|
with gzip.open(self.coverage_file, "wb") as file:
|
|
json.dump(self.coverage_master, file, sort_keys=True)
|
|
else:
|
|
with codecs.open(self.coverage_file, "w", encoding="utf-8") as file:
|
|
json.dump(self.coverage_master, file, indent=2, sort_keys=True)
|
|
super(IetfTestRunner, self).teardown_test_environment(**kwargs)
|
|
|
|
def get_test_paths(self, test_labels):
|
|
"""Find the apps and paths matching the test labels, so we later can limit
|
|
the coverage data to those apps and paths.
|
|
"""
|
|
test_apps = []
|
|
app_roots = set( app.split('.')[0] for app in settings.INSTALLED_APPS )
|
|
for label in test_labels:
|
|
part_list = label.split('.')
|
|
if label in settings.INSTALLED_APPS:
|
|
# The label is simply an app in installed apps
|
|
test_apps.append(label)
|
|
elif not (part_list[0] in app_roots):
|
|
# try to add an app root to get a match with installed apps
|
|
for root in app_roots:
|
|
for j in range(len(part_list)):
|
|
maybe_app = ".".join([root] + part_list[:j+1])
|
|
if maybe_app in settings.INSTALLED_APPS:
|
|
test_apps.append(maybe_app)
|
|
break
|
|
else:
|
|
continue
|
|
break
|
|
else:
|
|
# the label is more detailed than a plain app, and the
|
|
# root is in app_roots.
|
|
for j in range(len(part_list)):
|
|
maybe_app = ".".join(part_list[:j+1])
|
|
if maybe_app in settings.INSTALLED_APPS:
|
|
test_apps.append(maybe_app)
|
|
break
|
|
test_paths = [ os.path.join(*app.split('.')) for app in test_apps ]
|
|
return test_apps, test_paths
|
|
|
|
def run_tests(self, test_labels, extra_tests=[], **kwargs):
|
|
global old_destroy, old_create, test_database_name, template_coverage_collection
|
|
from django.db import connection
|
|
|
|
# Tests that involve switching back and forth between the real
|
|
# database and the test database are way too dangerous to run
|
|
# against the production database
|
|
if socket.gethostname().split('.')[0] in ['core3', 'ietfa', 'ietfb', 'ietfc', ]:
|
|
raise EnvironmentError("Refusing to run tests on production server")
|
|
|
|
old_create = connection.creation.__class__.create_test_db
|
|
connection.creation.__class__.create_test_db = safe_create_test_db
|
|
old_destroy = connection.creation.__class__.destroy_test_db
|
|
connection.creation.__class__.destroy_test_db = safe_destroy_test_db
|
|
|
|
self.run_full_test_suite = not test_labels
|
|
|
|
if not test_labels: # we only want to run our own tests
|
|
test_labels = ["ietf"]
|
|
|
|
self.test_apps, self.test_paths = self.get_test_paths(test_labels)
|
|
|
|
if self.check_coverage:
|
|
template_coverage_collection = True
|
|
extra_tests += [
|
|
CoverageTest(test_runner=self, methodName='url_coverage_test'),
|
|
CoverageTest(test_runner=self, methodName='template_coverage_test'),
|
|
CoverageTest(test_runner=self, methodName='code_coverage_test'),
|
|
]
|
|
|
|
self.reorder_by += (unittest.TestCase, ) # see to it that the coverage tests come last
|
|
|
|
failures = super(IetfTestRunner, self).run_tests(test_labels, extra_tests=extra_tests, **kwargs)
|
|
|
|
if self.check_coverage:
|
|
print("")
|
|
if self.run_full_test_suite:
|
|
print("Test coverage data:")
|
|
else:
|
|
print("Test coverage for this test run across the related app%s (%s):" % (("s" if len(self.test_apps)>1 else ""), ", ".join(self.test_apps)))
|
|
for test in ["template", "url", "code"]:
|
|
latest_coverage_version = self.coverage_master["version"]
|
|
|
|
master_data = self.coverage_master[latest_coverage_version][test]
|
|
#master_all = master_data["covered"]
|
|
#master_missing = [ k for k,v in master_data["covered"].items() if not v ]
|
|
master_coverage = master_data["coverage"]
|
|
|
|
test_data = self.coverage_data[test]
|
|
#test_all = test_data["covered"]
|
|
#test_missing = [ k for k,v in test_data["covered"].items() if not v ]
|
|
test_coverage = test_data["coverage"]
|
|
|
|
if self.run_full_test_suite:
|
|
print(" %8s coverage: %6.2f%% (%s: %6.2f%%)" %
|
|
(test.capitalize(), test_coverage*100, latest_coverage_version, master_coverage*100, ))
|
|
else:
|
|
print(" %8s coverage: %6.2f%%" %
|
|
(test.capitalize(), test_coverage*100, ))
|
|
|
|
print("""
|
|
Per-file code and template coverage and per-url-pattern url coverage data
|
|
for the latest test run has been written to %s.
|
|
|
|
Per-statement code coverage data has been written to '.coverage', readable
|
|
by the 'coverage' program.
|
|
""".replace(" ","") % (settings.TEST_COVERAGE_LATEST_FILE))
|
|
|
|
save_test_results(failures, test_labels)
|
|
|
|
return failures
|