altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

Added per-app coverage measurements, to make the numbers shown when not running the full tests suite a bit more meaningful.

Browse source 
- Legacy-Id: 9156
This commit is contained in:
Henrik Levkowetz 2015-03-06 20:42:20 +00:00
parent 37ed58055f
commit e94ac990fc
3 changed files with 272 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
ietf/settings.py
View file

@ -269,6 +269,12 @@ TEST_MATERIALS_DIR = "tmp-meeting-materials-dir"
TEST_BLUESHEET_DIR = "tmp-bluesheet-dir"
# These are regexes
TEST_URL_COVERAGE_EXCLUDE = [
"^\^admin/",
]
# Tese are filename globs
TEST_CODE_COVERAGE_EXCLUDE = [
"*/tests*",
"*/admin.py",
@ -276,8 +282,9 @@ TEST_CODE_COVERAGE_EXCLUDE = [
"ietf/settings*",
"ietf/utils/test_runner.py",
]
TEST_CODE_COVERAGE_MASTER_FILE = "coverage-master.json"
TEST_CODE_COVERAGE_LATEST_FILE = "coverage-latest.json"
TEST_COVERAGE_MASTER_FILE = "release-coverage.json"
TEST_COVERAGE_LATEST_FILE = "latest-coverage.json"
# WG Chair configuration
MAX_WG_DELEGATES = 3

145
ietf/utils/test_runner.py
View file

@ -54,6 +54,7 @@ from django.template import TemplateDoesNotExist
from django.test import TestCase
from django.test.runner import DiscoverRunner
from django.core.management import call_command
from django.core.urlresolvers import RegexURLResolver
import debug # pyflakes:ignore
@ -108,43 +109,51 @@ class RecordUrlsMiddleware(object):
def process_request(self, request):
visited_urls.add(request.path)
def get_url_patterns(module):
def get_url_patterns(module, apps=None):
def include(name):
if not apps:
return True
for app in apps:
if name.startswith(app+'.'):
return True
return False
def exclude(name):
for pat in settings.TEST_URL_COVERAGE_EXCLUDE:
if re.search(pat, name):
return True
return False
if not hasattr(module, 'urlpatterns'):
return []
res = []
try:
patterns = module.urlpatterns
except AttributeError:
patterns = []
for item in patterns:
try:
subpatterns = get_url_patterns(item.urlconf_module)
except:
subpatterns = [("", None)]
for sub, subitem in subpatterns:
if not sub:
res.append((item.regex.pattern, item))
elif sub.startswith("^"):
res.append((item.regex.pattern + sub[1:], subitem))
else:
res.append((item.regex.pattern + ".*" + sub, subitem))
for item in module.urlpatterns:
if isinstance(item, RegexURLResolver) and not type(item.urlconf_module) is list:
if include(item.urlconf_module.__name__) and not exclude(item.regex.pattern):
subpatterns = get_url_patterns(item.urlconf_module)
for sub, subitem in subpatterns:
if sub.startswith("^"):
res.append((item.regex.pattern + sub[1:], subitem))
else:
res.append((item.regex.pattern + ".*" + sub, subitem))
else:
res.append((item.regex.pattern, item))
return res
def get_templates():
def get_templates(apps=None):
templates = set()
# Should we teach this to use TEMPLATE_DIRS?
templatepath = os.path.join(settings.BASE_DIR, "templates")
for root, dirs, files in os.walk(templatepath):
if ".svn" in dirs:
dirs.remove(".svn")
relative_path = root[len(templatepath)+1:]
for file in files:
if file.endswith("~") or file.startswith("#"):
continue
if relative_path == "":
templatepaths = settings.TEMPLATE_DIRS
for templatepath in templatepaths:
for dirpath, dirs, files in os.walk(templatepath):
if ".svn" in dirs:
dirs.remove(".svn")
relative_path = dirpath[len(templatepath)+1:]
for file in files:
if file.endswith("~") or file.startswith("#"):
continue
if relative_path != "":
file = os.path.join(relative_path, file)
templates.add(file)
else:
templates.add(os.path.join(relative_path, file))
if apps:
templates = [ t for t in templates if t.split(os.path.sep)[0] in apps ]
return templates
def save_test_results(failures, test_labels):
@ -219,13 +228,14 @@ class CoverageTest(TestCase):
def template_coverage_test(self):
global loaded_templates
if self.runner.check_coverage:
all = get_templates()
apps = [ app.split('.')[-1] for app in self.runner.test_apps ]
all = get_templates(apps)
# The calculations here are slightly complicated by the situation
# that loaded_templates also contain nomcom page templates loaded
# from the database. However, those don't appear in all
covered = [ k for k in all if k in loaded_templates ]
self.runner.coverage_data["template"] = {
"coverage": 1.0*len(covered)/len(all),
"coverage": 1.0*len(covered)/len(all if len(all)>0 else float('nan')),
"covered": dict( (k, k in covered) for k in all ),
}
self.report_test_result("template")
@ -235,7 +245,7 @@ class CoverageTest(TestCase):
def url_coverage_test(self):
if self.runner.check_coverage:
import ietf.urls
url_patterns = get_url_patterns(ietf.urls)
url_patterns = get_url_patterns(ietf.urls, self.runner.test_apps)
# skip some patterns that we don't bother with
def ignore_pattern(regex, pattern):
@ -268,12 +278,13 @@ class CoverageTest(TestCase):
def code_coverage_test(self):
if self.runner.check_coverage:
include = [ os.path.join(path, '*') for path in self.runner.test_paths ]
checker = self.runner.code_coverage_checker
checker.stop()
checker.save()
checker._harvest_data()
checker.config.from_args(ignore_errors=None, omit=settings.TEST_CODE_COVERAGE_EXCLUDE,
include=None, file=None)
include=include, file=None)
reporter = CoverageReporter(checker, checker.config)
self.runner.coverage_data["code"] = reporter.report()
self.report_test_result("code")
@ -297,7 +308,7 @@ class IetfTestRunner(DiscoverRunner):
self.save_version_coverage = save_version_coverage
#
self.root_dir = os.path.dirname(settings.BASE_DIR)
self.coverage_file = os.path.join(self.root_dir, settings.TEST_CODE_COVERAGE_MASTER_FILE)
self.coverage_file = os.path.join(self.root_dir, settings.TEST_COVERAGE_MASTER_FILE)
super(IetfTestRunner, self).__init__(**kwargs)
def setup_test_environment(self, **kwargs):
@ -348,7 +359,7 @@ class IetfTestRunner(DiscoverRunner):
def teardown_test_environment(self, **kwargs):
self.smtpd_driver.stop()
if self.check_coverage:
latest_coverage_file = os.path.join(self.root_dir, settings.TEST_CODE_COVERAGE_LATEST_FILE)
latest_coverage_file = os.path.join(self.root_dir, settings.TEST_COVERAGE_LATEST_FILE)
with open(latest_coverage_file, "w") as file:
json.dump(self.coverage_data, file, indent=2, sort_keys=True)
if self.save_version_coverage:
@ -358,6 +369,39 @@ class IetfTestRunner(DiscoverRunner):
json.dump(self.coverage_master, file, indent=2, sort_keys=True)
super(IetfTestRunner, self).teardown_test_environment(**kwargs)
def get_test_paths(self, test_labels):
"""Find the apps and paths matching the test labels, so we later can limit
the coverage data to those apps and paths.
"""
test_apps = []
app_roots = set( app.split('.')[0] for app in settings.INSTALLED_APPS )
for label in test_labels:
part_list = label.split('.')
if label in settings.INSTALLED_APPS:
# The label is simply an app in installed apps
test_apps.append(label)
elif not (part_list[0] in app_roots):
# try to add an app root to get a match with installed apps
for root in app_roots:
for j in range(len(part_list)):
maybe_app = ".".join([root] + part_list[:j+1])
if maybe_app in settings.INSTALLED_APPS:
test_apps.append(maybe_app)
break
else:
continue
break
else:
# the label is more detailed than a plain app, and the
# root is in app_roots.
for j in range(len(part_list)):
maybe_app = ".".join(part_list[:j+1])
if maybe_app in settings.INSTALLED_APPS:
test_apps.append(maybe_app)
break
test_paths = [ os.path.join(*app.split('.')) for app in test_apps ]
return test_apps, test_paths
def run_tests(self, test_labels, extra_tests=None, **kwargs):
# Tests that involve switching back and forth between the real
# database and the test database are way too dangerous to run
@ -375,7 +419,9 @@ class IetfTestRunner(DiscoverRunner):
self.run_full_test_suite = not test_labels
if not test_labels: # we only want to run our own tests
test_labels = [app for app in settings.INSTALLED_APPS if app.startswith("ietf")]
test_labels = ["ietf"]
self.test_apps, self.test_paths = self.get_test_paths(test_labels)
extra_tests = [
CoverageTest(test_runner=self, methodName='url_coverage_test'),
@ -389,6 +435,10 @@ class IetfTestRunner(DiscoverRunner):
if self.check_coverage:
print("")
if self.run_full_test_suite:
print("Test coverage data:")
else:
print("Test coverage data for %s:" % ", ".join(test_labels))
for test in ["template", "url", "code"]:
latest_coverage_version = self.coverage_master["version"]
@ -402,13 +452,20 @@ class IetfTestRunner(DiscoverRunner):
#test_missing = [ k for k,v in test_data["covered"].items() if not v ]
test_coverage = test_data["coverage"]
print( "%-8s coverage: %.1f%% (%s: %.1f%%)" %
(test.capitalize(), test_coverage*100, latest_coverage_version, master_coverage*100, ))
if self.run_full_test_suite:
print(" %8s coverage: %6.2f%% (%s: %6.2f%%)" %
(test.capitalize(), test_coverage*100, latest_coverage_version, master_coverage*100, ))
else:
print(" %8s coverage: %6.2f%%" %
(test.capitalize(), test_coverage*100, ))
print("""
Code coverage data has been written to '.coverage', readable by
%s/bin/coverage.
""".replace(" ","") % settings.BASE_DIR)
Per-file code and template coverage and per-url-pattern url coverage data
for the latest test run has been written to %s.
Per-statement code coverage data has been written to '.coverage', readable
by the 'coverage' program.
""".replace(" ","") % (settings.TEST_COVERAGE_LATEST_FILE))
save_test_results(failures, test_labels)

169
coverage-master.json → release-coverage.json
View file

@ -2433,7 +2433,7 @@
},
"5.12.1": {
"code": {
"coverage": 0.5889938180301134,
"coverage": 0.5894313569287963,
"covered": {
"ietf/__init__": 0.0,
"ietf/api/__init__": 0.24675324675324675,
@ -2556,7 +2556,7 @@
"ietf/ipr/templatetags/ipr_filters": 0.6,
"ietf/ipr/urls": 1.0,
"ietf/ipr/utils": 0.75,
"ietf/ipr/views": 0.6697892271662763,
"ietf/ipr/views": 0.6932084309133489,
"ietf/legacy_router": 0.0,
"ietf/liaisons/__init__": 0.0,
"ietf/liaisons/accounts": 0.6483516483516483,
@ -2609,7 +2609,7 @@
"ietf/nomcom/__init__": 1.0,
"ietf/nomcom/decorators": 0.5,
"ietf/nomcom/fields": 0.48,
"ietf/nomcom/forms": 0.7517605633802817,
"ietf/nomcom/forms": 0.7491103202846975,
"ietf/nomcom/management/__init__": 1.0,
"ietf/nomcom/management/commands/__init__": 1.0,
"ietf/nomcom/management/commands/feedback_email": 0.0,
@ -2712,7 +2712,7 @@
"ietf/secr/utils/group": 0.5882352941176471,
"ietf/secr/utils/mail": 0.8125,
"ietf/secr/utils/meeting": 0.5510204081632653,
"ietf/secr/utils/test": 0.0,
"ietf/secr/utils/test": 0.3076923076923077,
"ietf/submit/__init__": 1.0,
"ietf/submit/forms": 0.8203883495145631,
"ietf/submit/mail": 0.9223300970873787,
@ -2767,7 +2767,7 @@
}
},
"template": {
"coverage": 0.7604166666666666,
"coverage": 0.6505576208178439,
"covered": {
"401.html": false,
"404.html": true,
@ -2775,10 +2775,19 @@
"admin/group/group/change_form.html": false,
"admin/group/group/change_list.html": false,
"admin/group/group/send_sdo_reminder.html": false,
"announcement/confirm.html": true,
"announcement/main.html": true,
"areas/add.html": false,
"areas/edit.html": false,
"areas/list.html": true,
"areas/people.html": false,
"areas/view.html": true,
"base.html": true,
"base/left_menu.html": true,
"base/streams_menu.html": true,
"base/wg_menu.html": true,
"base_secr.html": true,
"base_site.html": true,
"community/customize_display.html": false,
"community/display_field.html": false,
"community/manage_clist.html": false,
@ -2793,6 +2802,7 @@
"community/public/view_list.html": false,
"community/raw_view.html": false,
"community/view_list.html": false,
"console/main.html": false,
"cookies/settings.html": false,
"debug.html": true,
"doc/add_comment.html": true,
@ -2921,6 +2931,32 @@
"doc/status_change/submit.html": true,
"doc/submit_to_iesg.html": false,
"doc/submit_to_iesg_email.txt": false,
"drafts/abstract.html": false,
"drafts/add.html": false,
"drafts/approvals.html": false,
"drafts/authors.html": false,
"drafts/confirm.html": false,
"drafts/dates.html": false,
"drafts/edit.html": false,
"drafts/email.html": false,
"drafts/error.html": false,
"drafts/extend.html": false,
"drafts/makerfc.html": false,
"drafts/message_extend.txt": false,
"drafts/message_new.txt": false,
"drafts/message_replace.txt": false,
"drafts/message_resurrect.txt": false,
"drafts/message_revision.txt": false,
"drafts/message_update.txt": false,
"drafts/message_withdraw.txt": false,
"drafts/replace.html": false,
"drafts/report_id_activity.txt": false,
"drafts/report_nudge.html": false,
"drafts/report_progress_report.txt": false,
"drafts/revision.html": false,
"drafts/search.html": true,
"drafts/view.html": true,
"drafts/withdraw.html": false,
"email_failed.html": false,
"googlea30ad1dacffb5e5b.html": true,
"group/1wg-charters-by-acronym.txt": true,
@ -2956,6 +2992,15 @@
"group/reset_charter_milestones.html": true,
"group/stream_documents.html": true,
"group/stream_edit.html": true,
"groups/add.html": true,
"groups/blue_dot_report.txt": false,
"groups/charter.html": false,
"groups/edit.html": false,
"groups/edit_gm.html": false,
"groups/people.html": true,
"groups/search.html": false,
"groups/view.html": true,
"groups/view_gm.html": false,
"help/help_html.html": false,
"help/help_text.html": false,
"help/index.html": false,
@ -2987,6 +3032,37 @@
"iesg/scribe_doc_ballot.html": true,
"iesg/scribe_template.html": true,
"ietfauth/testemail.html": false,
"includes/activities.html": true,
"includes/awp_add_form.html": true,
"includes/awp_edit_form.html": false,
"includes/awp_view.html": true,
"includes/buttons_back.html": true,
"includes/buttons_next_cancel.html": false,
"includes/buttons_proceed.html": false,
"includes/buttons_save.html": true,
"includes/buttons_save_cancel.html": true,
"includes/buttons_search.html": true,
"includes/buttons_submit.html": false,
"includes/buttons_submit_back.html": true,
"includes/buttons_submit_cancel.html": false,
"includes/docevents.html": true,
"includes/draft_search_results.html": true,
"includes/draft_upload_form.html": false,
"includes/group_search_results.html": false,
"includes/meetings_footer.html": true,
"includes/proceeding_area.html": false,
"includes/proceeding_footer.html": true,
"includes/proceeding_header.html": true,
"includes/proceeding_title.html": false,
"includes/proceedings_functions.html": false,
"includes/search_results_table.html": true,
"includes/session_info.txt": true,
"includes/sessions_footer.html": true,
"includes/sessions_request_form.1_2": false,
"includes/sessions_request_form.html": true,
"includes/sessions_request_view.html": true,
"includes/slides.html": true,
"includes/upload_footer.html": true,
"ipr/add_comment.html": true,
"ipr/add_email.html": true,
"ipr/admin_base.html": true,
@ -3043,6 +3119,7 @@
"liaisons/sdo_reminder.txt": true,
"m_base.html": true,
"mailinglists/group_archives.html": true,
"main.html": true,
"meeting/agenda-utc.html": false,
"meeting/agenda.csv": true,
"meeting/agenda.html": true,
@ -3067,6 +3144,22 @@
"meeting/session_list.html": false,
"meeting/timeslot_edit.html": true,
"meeting/week-view.html": true,
"meetings/add.html": false,
"meetings/base_rooms_times.html": true,
"meetings/blue_sheet.html": true,
"meetings/edit_meeting.html": false,
"meetings/main.html": true,
"meetings/non_session.html": true,
"meetings/non_session_edit.html": false,
"meetings/notifications.html": true,
"meetings/rooms.html": true,
"meetings/schedule.html": false,
"meetings/select.html": true,
"meetings/select_group.html": true,
"meetings/session_schedule_notification.txt": true,
"meetings/times.html": true,
"meetings/times_edit.html": false,
"meetings/view.html": true,
"message/message.html": true,
"nomcom/announcements.html": true,
"nomcom/delete_nomcom.html": false,
@ -3109,6 +3202,34 @@
"notify_expirations/body.txt": false,
"notify_expirations/subject.txt": false,
"person/mail/possible_duplicates.txt": true,
"proceedings/acknowledgement.html": false,
"proceedings/agenda.html": false,
"proceedings/area.html": false,
"proceedings/attendee.html": false,
"proceedings/convert.html": false,
"proceedings/edit_slide.html": false,
"proceedings/index.html": false,
"proceedings/interim_directory.html": true,
"proceedings/interim_meeting.html": false,
"proceedings/interim_select.html": false,
"proceedings/irtf.html": false,
"proceedings/main.html": true,
"proceedings/overview.html": false,
"proceedings/plenary.html": false,
"proceedings/proceedings.html": true,
"proceedings/proceedings_template.html": false,
"proceedings/progress.html": false,
"proceedings/recording.html": true,
"proceedings/recording_edit.html": false,
"proceedings/replace_slide.html": false,
"proceedings/rg_irtf.html": false,
"proceedings/select.html": false,
"proceedings/status.html": false,
"proceedings/training.html": false,
"proceedings/upload_presentation.html": false,
"proceedings/upload_unified.html": true,
"proceedings/view.html": false,
"proceedings/wait.html": false,
"registration/add_email_email.txt": false,
"registration/base.html": true,
"registration/change_password.html": false,
@ -3125,6 +3246,29 @@
"registration/password_reset.html": false,
"registration/password_reset_email.txt": false,
"release/release.html": false,
"roles/base_roles.html": false,
"roles/chairs.html": false,
"roles/liaisons.html": false,
"roles/main.html": true,
"roles/roles.html": false,
"rolodex/add.html": false,
"rolodex/add_proceed.html": false,
"rolodex/delete.html": false,
"rolodex/edit.html": false,
"rolodex/search.html": true,
"rolodex/view.html": true,
"sreq/confirm.html": false,
"sreq/edit.html": true,
"sreq/locked.html": false,
"sreq/main.1_2": false,
"sreq/main.html": true,
"sreq/new.html": true,
"sreq/not_meeting_notification.txt": false,
"sreq/session_approval_notification.txt": false,
"sreq/session_cancel_notification.txt": false,
"sreq/session_request_notification.txt": false,
"sreq/tool_status.html": false,
"sreq/view.html": true,
"submit/add_preapproval.html": true,
"submit/announce_new_version.txt": true,
"submit/announce_to_authors.txt": true,
@ -3150,14 +3294,24 @@
"sync/discrepancies_report.txt": false,
"sync/notify.html": true,
"sync/rfceditor_undo.html": true,
"telechat/base_telechat.html": true,
"telechat/bash.html": false,
"telechat/doc.html": true,
"telechat/doc_template.html": false,
"telechat/group.html": false,
"telechat/main.html": true,
"telechat/management.html": false,
"telechat/minutes.html": false,
"telechat/roll_call.html": false,
"test/mail_body.txt": false,
"test/mail_subject.txt": false,
"unauthorized.html": false,
"utils/header_change_content.txt": false
}
},
"time": "2015-03-04T18:24:44Z",
"time": "2015-03-06T20:07:18Z",
"url": {
"coverage": 0.5374732334047109,
"coverage": 0.5163934426229508,
"covered": {
"^$": true,
"^(?P<group_type>(wg|rg))/$": true,
@ -3202,6 +3356,7 @@
"^accounts/settings/full_draft/(?P<enabled>.*)$": false,
"^accounts/settings/new_enough/": false,
"^accounts/settings/new_enough/(?P<days>.*)$": false,
"^api/v1/": true,
"^api/v1/?$": true,
"^community/(?P<list_id>[\\d]+)/remove_document/(?P<document_name>[^/]+)/$": false,
"^community/(?P<list_id>[\\d]+)/remove_rule/(?P<rule_id>[^/]+)/$": false,