563 lines
25 KiB
Python
563 lines
25 KiB
Python
# Copyright The IETF Trust 2007, All Rights Reserved
|
|
|
|
# Portion Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
|
|
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
#
|
|
# * Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
#
|
|
# * Redistributions in binary form must reproduce the above
|
|
# copyright notice, this list of conditions and the following
|
|
# disclaimer in the documentation and/or other materials provided
|
|
# with the distribution.
|
|
#
|
|
# * Neither the name of the Nokia Corporation and/or its
|
|
# subsidiary(-ies) nor the names of its contributors may be used
|
|
# to endorse or promote products derived from this software
|
|
# without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
|
import re
|
|
import os
|
|
import sys
|
|
import time
|
|
import json
|
|
import pytz
|
|
import importlib
|
|
import socket
|
|
import warnings
|
|
import datetime
|
|
import codecs
|
|
import gzip
|
|
import unittest
|
|
from fnmatch import fnmatch
|
|
|
|
from coverage.report import Reporter
|
|
from coverage.results import Numbers
|
|
from coverage.misc import NotPython
|
|
|
|
from django.conf import settings
|
|
from django.template import TemplateDoesNotExist
|
|
from django.template.loaders.base import Loader as BaseLoader
|
|
from django.test.runner import DiscoverRunner
|
|
from django.core.management import call_command
|
|
from django.core.urlresolvers import RegexURLResolver
|
|
|
|
import debug # pyflakes:ignore
|
|
debug.debug = True
|
|
|
|
import ietf
|
|
import ietf.utils.mail
|
|
from ietf.utils.test_smtpserver import SMTPTestServerDriver
|
|
from ietf.utils.test_utils import TestCase
|
|
|
|
loaded_templates = set()
|
|
visited_urls = set()
|
|
test_database_name = None
|
|
old_destroy = None
|
|
old_create = None
|
|
|
|
template_coverage_collection = None
|
|
url_coverage_collection = None
|
|
|
|
|
|
def safe_create_1(self, verbosity, *args, **kwargs):
|
|
global test_database_name, old_create
|
|
print " Creating test database..."
|
|
if settings.DATABASES["default"]["ENGINE"] == 'django.db.backends.mysql':
|
|
settings.DATABASES["default"]["OPTIONS"] = settings.DATABASE_TEST_OPTIONS
|
|
print " Using OPTIONS: %s" % settings.DATABASES["default"]["OPTIONS"]
|
|
test_database_name = old_create(self, 0, *args, **kwargs)
|
|
if settings.GLOBAL_TEST_FIXTURES:
|
|
print " Loading global test fixtures: %s" % ", ".join(settings.GLOBAL_TEST_FIXTURES)
|
|
loadable = [f for f in settings.GLOBAL_TEST_FIXTURES if "." not in f]
|
|
call_command('loaddata', *loadable, verbosity=verbosity, commit=False, database="default")
|
|
|
|
for f in settings.GLOBAL_TEST_FIXTURES:
|
|
if f not in loadable:
|
|
# try to execute the fixture
|
|
components = f.split(".")
|
|
module = importlib.import_module(".".join(components[:-1]))
|
|
fn = getattr(module, components[-1])
|
|
fn()
|
|
if verbosity < 2:
|
|
warnings.simplefilter("ignore", DeprecationWarning)
|
|
|
|
return test_database_name
|
|
|
|
def safe_destroy_0_1(*args, **kwargs):
|
|
global test_database_name, old_destroy
|
|
print " Checking that it's safe to destroy test database..."
|
|
if settings.DATABASES["default"]["NAME"] != test_database_name:
|
|
print ' NOT SAFE; Changing settings.DATABASES["default"]["NAME"] from %s to %s' % (settings.DATABASES["default"]["NAME"], test_database_name)
|
|
settings.DATABASES["default"]["NAME"] = test_database_name
|
|
return old_destroy(*args, **kwargs)
|
|
|
|
class TemplateCoverageLoader(BaseLoader):
|
|
is_usable = True
|
|
|
|
def load_template_source(self, template_name, dirs):
|
|
global template_coverage_collection, loaded_templates
|
|
if template_coverage_collection == True:
|
|
loaded_templates.add(str(template_name))
|
|
raise TemplateDoesNotExist
|
|
load_template_source.is_usable = True
|
|
|
|
class RecordUrlsMiddleware(object):
|
|
def process_request(self, request):
|
|
global url_coverage_collection, visited_urls
|
|
if url_coverage_collection == True:
|
|
visited_urls.add(request.path)
|
|
|
|
def get_url_patterns(module, apps=None):
|
|
def include(name):
|
|
if not apps:
|
|
return True
|
|
for app in apps:
|
|
if name.startswith(app+'.'):
|
|
return True
|
|
return False
|
|
def exclude(name):
|
|
for pat in settings.TEST_URL_COVERAGE_EXCLUDE:
|
|
if re.search(pat, name):
|
|
return True
|
|
return False
|
|
if not hasattr(module, 'urlpatterns'):
|
|
return []
|
|
res = []
|
|
for item in module.urlpatterns:
|
|
if isinstance(item, RegexURLResolver) and not type(item.urlconf_module) is list:
|
|
if include(item.urlconf_module.__name__) and not exclude(item.regex.pattern):
|
|
subpatterns = get_url_patterns(item.urlconf_module)
|
|
for sub, subitem in subpatterns:
|
|
if sub.startswith("^"):
|
|
res.append((item.regex.pattern + sub[1:], subitem))
|
|
else:
|
|
res.append((item.regex.pattern + ".*" + sub, subitem))
|
|
else:
|
|
res.append((item.regex.pattern, item))
|
|
return res
|
|
|
|
_all_templates = None
|
|
def get_template_paths(apps=None):
|
|
global _all_templates
|
|
if not _all_templates:
|
|
# TODO: Add app templates to the full list, if we are using
|
|
# django.template.loaders.app_directories.Loader
|
|
templates = set()
|
|
templatepaths = settings.TEMPLATES[0]['DIRS']
|
|
for templatepath in templatepaths:
|
|
for dirpath, dirs, files in os.walk(templatepath):
|
|
if ".svn" in dirs:
|
|
dirs.remove(".svn")
|
|
relative_path = dirpath[len(templatepath)+1:]
|
|
for file in files:
|
|
ignore = False
|
|
for pattern in settings.TEST_TEMPLATE_IGNORE:
|
|
if fnmatch(file, pattern):
|
|
ignore = True
|
|
break
|
|
if ignore:
|
|
continue
|
|
if relative_path != "":
|
|
file = os.path.join(relative_path, file)
|
|
templates.add(file)
|
|
if apps:
|
|
templates = [ t for t in templates if t.split(os.path.sep)[0] in apps ]
|
|
_all_templates = templates
|
|
return _all_templates
|
|
|
|
def save_test_results(failures, test_labels):
|
|
# Record the test result in a file, in order to be able to check the
|
|
# results and avoid re-running tests if we've alread run them with OK
|
|
# result after the latest code changes:
|
|
topdir = os.path.dirname(os.path.dirname(settings.BASE_DIR))
|
|
tfile = codecs.open(os.path.join(topdir,".testresult"), "a", encoding='utf-8')
|
|
timestr = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
if failures:
|
|
tfile.write("%s FAILED (failures=%s)\n" % (timestr, failures))
|
|
else:
|
|
if test_labels:
|
|
tfile.write("%s SUCCESS (tests=%s)\n" % (timestr, test_labels))
|
|
else:
|
|
tfile.write("%s OK\n" % (timestr, ))
|
|
tfile.close()
|
|
|
|
|
|
def set_coverage_checking(flag=True):
|
|
global template_coverage_collection
|
|
global url_coverage_collection
|
|
if flag:
|
|
settings.TEST_CODE_COVERAGE_CHECKER.collector.resume()
|
|
template_coverage_collection = True
|
|
url_coverage_collection = True
|
|
else:
|
|
settings.TEST_CODE_COVERAGE_CHECKER.collector.pause()
|
|
template_coverage_collection = False
|
|
url_coverage_collection = False
|
|
|
|
class CoverageReporter(Reporter):
|
|
def report(self):
|
|
self.find_file_reporters(None)
|
|
|
|
total = Numbers()
|
|
result = {"coverage": 0.0, "covered": {}, "format": 2, }
|
|
for fr in self.file_reporters:
|
|
try:
|
|
analysis = self.coverage._analyze(fr)
|
|
nums = analysis.numbers
|
|
result["covered"][fr.relative_filename()] = (nums.n_statements, nums.pc_covered/100.0)
|
|
total += nums
|
|
except KeyboardInterrupt: # pragma: not covered
|
|
raise
|
|
except Exception:
|
|
report_it = not self.config.ignore_errors
|
|
if report_it:
|
|
typ, msg = sys.exc_info()[:2]
|
|
if typ is NotPython and not fr.should_be_python():
|
|
report_it = False
|
|
if report_it:
|
|
raise
|
|
result["coverage"] = total.pc_covered/100.0
|
|
return result
|
|
|
|
|
|
class CoverageTest(TestCase):
|
|
|
|
def __init__(self, test_runner=None, **kwargs):
|
|
self.runner = test_runner
|
|
super(CoverageTest, self).__init__(**kwargs)
|
|
|
|
def report_test_result(self, test):
|
|
latest_coverage_version = self.runner.coverage_master["version"]
|
|
|
|
master_data = self.runner.coverage_master[latest_coverage_version][test]
|
|
master_missing = [ k for k,v in master_data["covered"].items() if not v ]
|
|
master_coverage = master_data["coverage"]
|
|
|
|
test_data = self.runner.coverage_data[test]
|
|
test_missing = [ k for k,v in test_data["covered"].items() if not v ]
|
|
test_coverage = test_data["coverage"]
|
|
|
|
# Assert coverage failure only if we're running the full test suite -- if we're
|
|
# only running some tests, then of course the coverage is going to be low.
|
|
if self.runner.run_full_test_suite:
|
|
# Permit 0.02% variation in results -- otherwise small code changes become a pain
|
|
fudge_factor = 0.00005 # 0.005% -- a small change, less than the last digit we show
|
|
self.assertGreaterEqual(test_coverage, master_coverage - fudge_factor,
|
|
msg = "The %s coverage percentage is now lower (%.2f%%) than for version %s (%.2f%%)" %
|
|
( test, test_coverage*100, latest_coverage_version, master_coverage*100, ))
|
|
self.assertLessEqual(len(test_missing), len(master_missing),
|
|
msg = "New %s without test coverage since %s: %s" % (test, latest_coverage_version, list(set(test_missing) - set(master_missing))))
|
|
|
|
def template_coverage_test(self):
|
|
global loaded_templates
|
|
if self.runner.check_coverage:
|
|
apps = [ app.split('.')[-1] for app in self.runner.test_apps ]
|
|
all = get_template_paths(apps)
|
|
# The calculations here are slightly complicated by the situation
|
|
# that loaded_templates also contain nomcom page templates loaded
|
|
# from the database. However, those don't appear in all
|
|
covered = [ k for k in all if k in loaded_templates ]
|
|
self.runner.coverage_data["template"] = {
|
|
"coverage": (1.0*len(covered)/len(all)) if len(all)>0 else float('nan'),
|
|
"covered": dict( (k, k in covered) for k in all ),
|
|
}
|
|
self.report_test_result("template")
|
|
else:
|
|
self.skipTest("Coverage switched off with --skip-coverage")
|
|
|
|
def url_coverage_test(self):
|
|
if self.runner.check_coverage:
|
|
import ietf.urls
|
|
url_patterns = get_url_patterns(ietf.urls, self.runner.test_apps)
|
|
|
|
# skip some patterns that we don't bother with
|
|
def ignore_pattern(regex, pattern):
|
|
import django.views.static
|
|
return (regex in ("^_test500/$", "^accounts/testemail/$")
|
|
or regex.startswith("^admin/")
|
|
or getattr(pattern.callback, "__name__", "") == "RedirectView"
|
|
or getattr(pattern.callback, "__name__", "") == "TemplateView"
|
|
or pattern.callback == django.views.static.serve)
|
|
|
|
patterns = [(regex, re.compile(regex)) for regex, pattern in url_patterns
|
|
if not ignore_pattern(regex, pattern)]
|
|
all = [ regex for regex, compiled in patterns ]
|
|
|
|
covered = set()
|
|
for url in visited_urls:
|
|
for regex, compiled in patterns:
|
|
if regex not in covered and compiled.match(url[1:]): # strip leading /
|
|
covered.add(regex)
|
|
break
|
|
|
|
self.runner.coverage_data["url"] = {
|
|
"coverage": 1.0*len(covered)/len(all),
|
|
"covered": dict( (k, k in covered) for k in all ),
|
|
}
|
|
|
|
self.report_test_result("url")
|
|
else:
|
|
self.skipTest("Coverage switched off with --skip-coverage")
|
|
|
|
def code_coverage_test(self):
|
|
if self.runner.check_coverage:
|
|
include = [ os.path.join(path, '*') for path in self.runner.test_paths ]
|
|
checker = self.runner.code_coverage_checker
|
|
checker.stop()
|
|
# Save to the .coverage file
|
|
checker.save()
|
|
# Apply the configured and requested omit and include data
|
|
checker.config.from_args(ignore_errors=None, omit=settings.TEST_CODE_COVERAGE_EXCLUDE,
|
|
include=include, file=None)
|
|
# Maybe output a html report
|
|
if self.runner.run_full_test_suite:
|
|
checker.html_report(directory=settings.TEST_CODE_COVERAGE_REPORT_DIR)
|
|
# In any case, build a dictionary with per-file data for this run
|
|
reporter = CoverageReporter(checker, checker.config)
|
|
self.runner.coverage_data["code"] = reporter.report()
|
|
self.report_test_result("code")
|
|
else:
|
|
self.skipTest("Coverage switched off with --skip-coverage")
|
|
|
|
class IetfTestRunner(DiscoverRunner):
|
|
|
|
@classmethod
|
|
def add_arguments(cls, parser):
|
|
parser.add_argument('--skip-coverage',
|
|
action='store_true', dest='skip_coverage', default=False,
|
|
help='Skip test coverage measurements for code, templates, and URLs. ' )
|
|
parser.add_argument('--save-version-coverage',
|
|
action='store', dest='save_version_coverage', default=False,
|
|
help='Save test coverage data under the given version label')
|
|
parser.add_argument('--save-testresult',
|
|
action='store_true', dest='save_testresult', default=False,
|
|
help='Save short test result data in %s/.testresult' % os.path.dirname(os.path.dirname(settings.BASE_DIR))),
|
|
|
|
def __init__(self, skip_coverage=False, save_version_coverage=None, **kwargs):
|
|
#
|
|
self.check_coverage = not skip_coverage
|
|
self.save_version_coverage = save_version_coverage
|
|
#
|
|
self.root_dir = os.path.dirname(settings.BASE_DIR)
|
|
self.coverage_file = os.path.join(self.root_dir, settings.TEST_COVERAGE_MASTER_FILE)
|
|
super(IetfTestRunner, self).__init__(**kwargs)
|
|
|
|
def setup_test_environment(self, **kwargs):
|
|
global template_coverage_collection
|
|
global url_coverage_collection
|
|
|
|
ietf.utils.mail.test_mode = True
|
|
ietf.utils.mail.SMTP_ADDR['ip4'] = '127.0.0.1'
|
|
ietf.utils.mail.SMTP_ADDR['port'] = 2025
|
|
# switch to a much faster hasher
|
|
settings.PASSWORD_HASHERS = ( 'django.contrib.auth.hashers.MD5PasswordHasher', )
|
|
settings.SERVER_MODE = 'test'
|
|
#
|
|
if self.check_coverage:
|
|
if self.coverage_file.endswith('.gz'):
|
|
with gzip.open(self.coverage_file, "rb") as file:
|
|
self.coverage_master = json.load(file)
|
|
else:
|
|
with codecs.open(self.coverage_file, encoding='utf-8') as file:
|
|
self.coverage_master = json.load(file)
|
|
self.coverage_data = {
|
|
"time": datetime.datetime.now(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"template": {
|
|
"coverage": 0.0,
|
|
"covered": {},
|
|
"format": 1, # default format, coverage data in 'covered' are just fractions
|
|
},
|
|
"url": {
|
|
"coverage": 0.0,
|
|
"covered": {},
|
|
"format": 1,
|
|
},
|
|
"code": {
|
|
"coverage": 0.0,
|
|
"covered": {},
|
|
"format": 1,
|
|
},
|
|
}
|
|
|
|
settings.TEMPLATES[0]['OPTIONS']['loaders'] = ('ietf.utils.test_runner.TemplateCoverageLoader',) + settings.TEMPLATES[0]['OPTIONS']['loaders']
|
|
|
|
settings.MIDDLEWARE_CLASSES = ('ietf.utils.test_runner.RecordUrlsMiddleware',) + settings.MIDDLEWARE_CLASSES
|
|
url_coverage_collection = True
|
|
|
|
self.code_coverage_checker = settings.TEST_CODE_COVERAGE_CHECKER
|
|
if not self.code_coverage_checker._started:
|
|
sys.stderr.write(" ** Warning: In %s: Expected the coverage checker to have\n"
|
|
" been started already, but it wasn't. Doing so now. Coverage numbers\n"
|
|
" will be off, though.\n" % __name__)
|
|
self.code_coverage_checker.start()
|
|
|
|
if settings.SITE_ID != 1:
|
|
print " Changing SITE_ID to '1' during testing."
|
|
settings.SITE_ID = 1
|
|
|
|
if settings.TEMPLATES[0]['OPTIONS']['string_if_invalid'] != '':
|
|
print(" Changing TEMPLATES[0]['OPTIONS']['string_if_invalid'] to '' during testing")
|
|
settings.TEMPLATES[0]['OPTIONS']['string_if_invalid'] = ''
|
|
|
|
assert not settings.IDTRACKER_BASE_URL.endswith('/')
|
|
|
|
# Try to set up an SMTP test server. In case other test runs are
|
|
# going on at the same time, try a range of ports.
|
|
base = ietf.utils.mail.SMTP_ADDR['port']
|
|
for offset in range(10):
|
|
try:
|
|
# remember the value so ietf.utils.mail.send_smtp() will use the same
|
|
ietf.utils.mail.SMTP_ADDR['port'] = base + offset
|
|
self.smtpd_driver = SMTPTestServerDriver((ietf.utils.mail.SMTP_ADDR['ip4'],ietf.utils.mail.SMTP_ADDR['port']),None)
|
|
self.smtpd_driver.start()
|
|
print(" Running an SMTP test server on %(ip4)s:%(port)s to catch outgoing email." % ietf.utils.mail.SMTP_ADDR)
|
|
break
|
|
except socket.error:
|
|
pass
|
|
|
|
|
|
super(IetfTestRunner, self).setup_test_environment(**kwargs)
|
|
|
|
def teardown_test_environment(self, **kwargs):
|
|
self.smtpd_driver.stop()
|
|
if self.check_coverage:
|
|
latest_coverage_file = os.path.join(self.root_dir, settings.TEST_COVERAGE_LATEST_FILE)
|
|
coverage_latest = {}
|
|
coverage_latest["version"] = "latest"
|
|
coverage_latest["latest"] = self.coverage_data
|
|
with codecs.open(latest_coverage_file, "w", encoding='utf-8') as file:
|
|
json.dump(coverage_latest, file, indent=2, sort_keys=True)
|
|
if self.save_version_coverage:
|
|
self.coverage_master["version"] = self.save_version_coverage
|
|
self.coverage_master[self.save_version_coverage] = self.coverage_data
|
|
if self.coverage_file.endswith('.gz'):
|
|
with gzip.open(self.coverage_file, "wb") as file:
|
|
json.dump(self.coverage_master, file, sort_keys=True)
|
|
else:
|
|
with codecs.open(self.coverage_file, "w", encoding="utf-8") as file:
|
|
json.dump(self.coverage_master, file, indent=2, sort_keys=True)
|
|
super(IetfTestRunner, self).teardown_test_environment(**kwargs)
|
|
|
|
def get_test_paths(self, test_labels):
|
|
"""Find the apps and paths matching the test labels, so we later can limit
|
|
the coverage data to those apps and paths.
|
|
"""
|
|
test_apps = []
|
|
app_roots = set( app.split('.')[0] for app in settings.INSTALLED_APPS )
|
|
for label in test_labels:
|
|
part_list = label.split('.')
|
|
if label in settings.INSTALLED_APPS:
|
|
# The label is simply an app in installed apps
|
|
test_apps.append(label)
|
|
elif not (part_list[0] in app_roots):
|
|
# try to add an app root to get a match with installed apps
|
|
for root in app_roots:
|
|
for j in range(len(part_list)):
|
|
maybe_app = ".".join([root] + part_list[:j+1])
|
|
if maybe_app in settings.INSTALLED_APPS:
|
|
test_apps.append(maybe_app)
|
|
break
|
|
else:
|
|
continue
|
|
break
|
|
else:
|
|
# the label is more detailed than a plain app, and the
|
|
# root is in app_roots.
|
|
for j in range(len(part_list)):
|
|
maybe_app = ".".join(part_list[:j+1])
|
|
if maybe_app in settings.INSTALLED_APPS:
|
|
test_apps.append(maybe_app)
|
|
break
|
|
test_paths = [ os.path.join(*app.split('.')) for app in test_apps ]
|
|
return test_apps, test_paths
|
|
|
|
def run_tests(self, test_labels, extra_tests=[], **kwargs):
|
|
global old_destroy, old_create, test_database_name, template_coverage_collection
|
|
from django.db import connection
|
|
|
|
# Tests that involve switching back and forth between the real
|
|
# database and the test database are way too dangerous to run
|
|
# against the production database
|
|
if socket.gethostname().split('.')[0] in ['core3', 'ietfa', 'ietfb', 'ietfc', ]:
|
|
raise EnvironmentError("Refusing to run tests on production server")
|
|
|
|
old_create = connection.creation.__class__.create_test_db
|
|
connection.creation.__class__.create_test_db = safe_create_1
|
|
old_destroy = connection.creation.__class__.destroy_test_db
|
|
connection.creation.__class__.destroy_test_db = safe_destroy_0_1
|
|
|
|
self.run_full_test_suite = not test_labels
|
|
|
|
if not test_labels: # we only want to run our own tests
|
|
test_labels = ["ietf"]
|
|
|
|
self.test_apps, self.test_paths = self.get_test_paths(test_labels)
|
|
|
|
if self.check_coverage:
|
|
template_coverage_collection = True
|
|
extra_tests += [
|
|
CoverageTest(test_runner=self, methodName='url_coverage_test'),
|
|
CoverageTest(test_runner=self, methodName='template_coverage_test'),
|
|
CoverageTest(test_runner=self, methodName='code_coverage_test'),
|
|
]
|
|
|
|
self.reorder_by += (unittest.TestCase, ) # see to it that the coverage tests come last
|
|
|
|
failures = super(IetfTestRunner, self).run_tests(test_labels, extra_tests=extra_tests, **kwargs)
|
|
|
|
if self.check_coverage:
|
|
print("")
|
|
if self.run_full_test_suite:
|
|
print("Test coverage data:")
|
|
else:
|
|
print("Test coverage for this test run across the related app%s (%s):" % (("s" if len(self.test_apps)>1 else ""), ", ".join(self.test_apps)))
|
|
for test in ["template", "url", "code"]:
|
|
latest_coverage_version = self.coverage_master["version"]
|
|
|
|
master_data = self.coverage_master[latest_coverage_version][test]
|
|
#master_all = master_data["covered"]
|
|
#master_missing = [ k for k,v in master_data["covered"].items() if not v ]
|
|
master_coverage = master_data["coverage"]
|
|
|
|
test_data = self.coverage_data[test]
|
|
#test_all = test_data["covered"]
|
|
#test_missing = [ k for k,v in test_data["covered"].items() if not v ]
|
|
test_coverage = test_data["coverage"]
|
|
|
|
if self.run_full_test_suite:
|
|
print(" %8s coverage: %6.2f%% (%s: %6.2f%%)" %
|
|
(test.capitalize(), test_coverage*100, latest_coverage_version, master_coverage*100, ))
|
|
else:
|
|
print(" %8s coverage: %6.2f%%" %
|
|
(test.capitalize(), test_coverage*100, ))
|
|
|
|
print("""
|
|
Per-file code and template coverage and per-url-pattern url coverage data
|
|
for the latest test run has been written to %s.
|
|
|
|
Per-statement code coverage data has been written to '.coverage', readable
|
|
by the 'coverage' program.
|
|
""".replace(" ","") % (settings.TEST_COVERAGE_LATEST_FILE))
|
|
|
|
save_test_results(failures, test_labels)
|
|
|
|
return failures
|