diff --git a/ietf/settings.py b/ietf/settings.py index 256b4752e..9deccdd35 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -269,6 +269,12 @@ TEST_MATERIALS_DIR = "tmp-meeting-materials-dir" TEST_BLUESHEET_DIR = "tmp-bluesheet-dir" +# These are regexes +TEST_URL_COVERAGE_EXCLUDE = [ + "^\^admin/", +] + +# Tese are filename globs TEST_CODE_COVERAGE_EXCLUDE = [ "*/tests*", "*/admin.py", @@ -276,8 +282,9 @@ TEST_CODE_COVERAGE_EXCLUDE = [ "ietf/settings*", "ietf/utils/test_runner.py", ] -TEST_CODE_COVERAGE_MASTER_FILE = "coverage-master.json" -TEST_CODE_COVERAGE_LATEST_FILE = "coverage-latest.json" + +TEST_COVERAGE_MASTER_FILE = "release-coverage.json" +TEST_COVERAGE_LATEST_FILE = "latest-coverage.json" # WG Chair configuration MAX_WG_DELEGATES = 3 diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 350a020af..4f762d21f 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -54,6 +54,7 @@ from django.template import TemplateDoesNotExist from django.test import TestCase from django.test.runner import DiscoverRunner from django.core.management import call_command +from django.core.urlresolvers import RegexURLResolver import debug # pyflakes:ignore @@ -108,43 +109,51 @@ class RecordUrlsMiddleware(object): def process_request(self, request): visited_urls.add(request.path) -def get_url_patterns(module): +def get_url_patterns(module, apps=None): + def include(name): + if not apps: + return True + for app in apps: + if name.startswith(app+'.'): + return True + return False + def exclude(name): + for pat in settings.TEST_URL_COVERAGE_EXCLUDE: + if re.search(pat, name): + return True + return False + if not hasattr(module, 'urlpatterns'): + return [] res = [] - try: - patterns = module.urlpatterns - except AttributeError: - patterns = [] - for item in patterns: - try: - subpatterns = get_url_patterns(item.urlconf_module) - except: - subpatterns = [("", None)] - for sub, subitem in subpatterns: - if not sub: - res.append((item.regex.pattern, item)) - elif sub.startswith("^"): - res.append((item.regex.pattern + sub[1:], subitem)) - else: - res.append((item.regex.pattern + ".*" + sub, subitem)) + for item in module.urlpatterns: + if isinstance(item, RegexURLResolver) and not type(item.urlconf_module) is list: + if include(item.urlconf_module.__name__) and not exclude(item.regex.pattern): + subpatterns = get_url_patterns(item.urlconf_module) + for sub, subitem in subpatterns: + if sub.startswith("^"): + res.append((item.regex.pattern + sub[1:], subitem)) + else: + res.append((item.regex.pattern + ".*" + sub, subitem)) + else: + res.append((item.regex.pattern, item)) return res - -def get_templates(): +def get_templates(apps=None): templates = set() - # Should we teach this to use TEMPLATE_DIRS? - templatepath = os.path.join(settings.BASE_DIR, "templates") - for root, dirs, files in os.walk(templatepath): - if ".svn" in dirs: - dirs.remove(".svn") - relative_path = root[len(templatepath)+1:] - for file in files: - if file.endswith("~") or file.startswith("#"): - continue - if relative_path == "": + templatepaths = settings.TEMPLATE_DIRS + for templatepath in templatepaths: + for dirpath, dirs, files in os.walk(templatepath): + if ".svn" in dirs: + dirs.remove(".svn") + relative_path = dirpath[len(templatepath)+1:] + for file in files: + if file.endswith("~") or file.startswith("#"): + continue + if relative_path != "": + file = os.path.join(relative_path, file) templates.add(file) - else: - templates.add(os.path.join(relative_path, file)) - + if apps: + templates = [ t for t in templates if t.split(os.path.sep)[0] in apps ] return templates def save_test_results(failures, test_labels): @@ -219,13 +228,14 @@ class CoverageTest(TestCase): def template_coverage_test(self): global loaded_templates if self.runner.check_coverage: - all = get_templates() + apps = [ app.split('.')[-1] for app in self.runner.test_apps ] + all = get_templates(apps) # The calculations here are slightly complicated by the situation # that loaded_templates also contain nomcom page templates loaded # from the database. However, those don't appear in all covered = [ k for k in all if k in loaded_templates ] self.runner.coverage_data["template"] = { - "coverage": 1.0*len(covered)/len(all), + "coverage": 1.0*len(covered)/len(all if len(all)>0 else float('nan')), "covered": dict( (k, k in covered) for k in all ), } self.report_test_result("template") @@ -235,7 +245,7 @@ class CoverageTest(TestCase): def url_coverage_test(self): if self.runner.check_coverage: import ietf.urls - url_patterns = get_url_patterns(ietf.urls) + url_patterns = get_url_patterns(ietf.urls, self.runner.test_apps) # skip some patterns that we don't bother with def ignore_pattern(regex, pattern): @@ -268,12 +278,13 @@ class CoverageTest(TestCase): def code_coverage_test(self): if self.runner.check_coverage: + include = [ os.path.join(path, '*') for path in self.runner.test_paths ] checker = self.runner.code_coverage_checker checker.stop() checker.save() checker._harvest_data() checker.config.from_args(ignore_errors=None, omit=settings.TEST_CODE_COVERAGE_EXCLUDE, - include=None, file=None) + include=include, file=None) reporter = CoverageReporter(checker, checker.config) self.runner.coverage_data["code"] = reporter.report() self.report_test_result("code") @@ -297,7 +308,7 @@ class IetfTestRunner(DiscoverRunner): self.save_version_coverage = save_version_coverage # self.root_dir = os.path.dirname(settings.BASE_DIR) - self.coverage_file = os.path.join(self.root_dir, settings.TEST_CODE_COVERAGE_MASTER_FILE) + self.coverage_file = os.path.join(self.root_dir, settings.TEST_COVERAGE_MASTER_FILE) super(IetfTestRunner, self).__init__(**kwargs) def setup_test_environment(self, **kwargs): @@ -348,7 +359,7 @@ class IetfTestRunner(DiscoverRunner): def teardown_test_environment(self, **kwargs): self.smtpd_driver.stop() if self.check_coverage: - latest_coverage_file = os.path.join(self.root_dir, settings.TEST_CODE_COVERAGE_LATEST_FILE) + latest_coverage_file = os.path.join(self.root_dir, settings.TEST_COVERAGE_LATEST_FILE) with open(latest_coverage_file, "w") as file: json.dump(self.coverage_data, file, indent=2, sort_keys=True) if self.save_version_coverage: @@ -358,6 +369,39 @@ class IetfTestRunner(DiscoverRunner): json.dump(self.coverage_master, file, indent=2, sort_keys=True) super(IetfTestRunner, self).teardown_test_environment(**kwargs) + def get_test_paths(self, test_labels): + """Find the apps and paths matching the test labels, so we later can limit + the coverage data to those apps and paths. + """ + test_apps = [] + app_roots = set( app.split('.')[0] for app in settings.INSTALLED_APPS ) + for label in test_labels: + part_list = label.split('.') + if label in settings.INSTALLED_APPS: + # The label is simply an app in installed apps + test_apps.append(label) + elif not (part_list[0] in app_roots): + # try to add an app root to get a match with installed apps + for root in app_roots: + for j in range(len(part_list)): + maybe_app = ".".join([root] + part_list[:j+1]) + if maybe_app in settings.INSTALLED_APPS: + test_apps.append(maybe_app) + break + else: + continue + break + else: + # the label is more detailed than a plain app, and the + # root is in app_roots. + for j in range(len(part_list)): + maybe_app = ".".join(part_list[:j+1]) + if maybe_app in settings.INSTALLED_APPS: + test_apps.append(maybe_app) + break + test_paths = [ os.path.join(*app.split('.')) for app in test_apps ] + return test_apps, test_paths + def run_tests(self, test_labels, extra_tests=None, **kwargs): # Tests that involve switching back and forth between the real # database and the test database are way too dangerous to run @@ -375,7 +419,9 @@ class IetfTestRunner(DiscoverRunner): self.run_full_test_suite = not test_labels if not test_labels: # we only want to run our own tests - test_labels = [app for app in settings.INSTALLED_APPS if app.startswith("ietf")] + test_labels = ["ietf"] + + self.test_apps, self.test_paths = self.get_test_paths(test_labels) extra_tests = [ CoverageTest(test_runner=self, methodName='url_coverage_test'), @@ -389,6 +435,10 @@ class IetfTestRunner(DiscoverRunner): if self.check_coverage: print("") + if self.run_full_test_suite: + print("Test coverage data:") + else: + print("Test coverage data for %s:" % ", ".join(test_labels)) for test in ["template", "url", "code"]: latest_coverage_version = self.coverage_master["version"] @@ -402,13 +452,20 @@ class IetfTestRunner(DiscoverRunner): #test_missing = [ k for k,v in test_data["covered"].items() if not v ] test_coverage = test_data["coverage"] - print( "%-8s coverage: %.1f%% (%s: %.1f%%)" % - (test.capitalize(), test_coverage*100, latest_coverage_version, master_coverage*100, )) + if self.run_full_test_suite: + print(" %8s coverage: %6.2f%% (%s: %6.2f%%)" % + (test.capitalize(), test_coverage*100, latest_coverage_version, master_coverage*100, )) + else: + print(" %8s coverage: %6.2f%%" % + (test.capitalize(), test_coverage*100, )) print(""" - Code coverage data has been written to '.coverage', readable by - %s/bin/coverage. - """.replace(" ","") % settings.BASE_DIR) + Per-file code and template coverage and per-url-pattern url coverage data + for the latest test run has been written to %s. + + Per-statement code coverage data has been written to '.coverage', readable + by the 'coverage' program. + """.replace(" ","") % (settings.TEST_COVERAGE_LATEST_FILE)) save_test_results(failures, test_labels) diff --git a/coverage-master.json b/release-coverage.json similarity index 96% rename from coverage-master.json rename to release-coverage.json index 6351936ce..f02276c52 100644 --- a/coverage-master.json +++ b/release-coverage.json @@ -2433,7 +2433,7 @@ }, "5.12.1": { "code": { - "coverage": 0.5889938180301134, + "coverage": 0.5894313569287963, "covered": { "ietf/__init__": 0.0, "ietf/api/__init__": 0.24675324675324675, @@ -2556,7 +2556,7 @@ "ietf/ipr/templatetags/ipr_filters": 0.6, "ietf/ipr/urls": 1.0, "ietf/ipr/utils": 0.75, - "ietf/ipr/views": 0.6697892271662763, + "ietf/ipr/views": 0.6932084309133489, "ietf/legacy_router": 0.0, "ietf/liaisons/__init__": 0.0, "ietf/liaisons/accounts": 0.6483516483516483, @@ -2609,7 +2609,7 @@ "ietf/nomcom/__init__": 1.0, "ietf/nomcom/decorators": 0.5, "ietf/nomcom/fields": 0.48, - "ietf/nomcom/forms": 0.7517605633802817, + "ietf/nomcom/forms": 0.7491103202846975, "ietf/nomcom/management/__init__": 1.0, "ietf/nomcom/management/commands/__init__": 1.0, "ietf/nomcom/management/commands/feedback_email": 0.0, @@ -2712,7 +2712,7 @@ "ietf/secr/utils/group": 0.5882352941176471, "ietf/secr/utils/mail": 0.8125, "ietf/secr/utils/meeting": 0.5510204081632653, - "ietf/secr/utils/test": 0.0, + "ietf/secr/utils/test": 0.3076923076923077, "ietf/submit/__init__": 1.0, "ietf/submit/forms": 0.8203883495145631, "ietf/submit/mail": 0.9223300970873787, @@ -2767,7 +2767,7 @@ } }, "template": { - "coverage": 0.7604166666666666, + "coverage": 0.6505576208178439, "covered": { "401.html": false, "404.html": true, @@ -2775,10 +2775,19 @@ "admin/group/group/change_form.html": false, "admin/group/group/change_list.html": false, "admin/group/group/send_sdo_reminder.html": false, + "announcement/confirm.html": true, + "announcement/main.html": true, + "areas/add.html": false, + "areas/edit.html": false, + "areas/list.html": true, + "areas/people.html": false, + "areas/view.html": true, "base.html": true, "base/left_menu.html": true, "base/streams_menu.html": true, "base/wg_menu.html": true, + "base_secr.html": true, + "base_site.html": true, "community/customize_display.html": false, "community/display_field.html": false, "community/manage_clist.html": false, @@ -2793,6 +2802,7 @@ "community/public/view_list.html": false, "community/raw_view.html": false, "community/view_list.html": false, + "console/main.html": false, "cookies/settings.html": false, "debug.html": true, "doc/add_comment.html": true, @@ -2921,6 +2931,32 @@ "doc/status_change/submit.html": true, "doc/submit_to_iesg.html": false, "doc/submit_to_iesg_email.txt": false, + "drafts/abstract.html": false, + "drafts/add.html": false, + "drafts/approvals.html": false, + "drafts/authors.html": false, + "drafts/confirm.html": false, + "drafts/dates.html": false, + "drafts/edit.html": false, + "drafts/email.html": false, + "drafts/error.html": false, + "drafts/extend.html": false, + "drafts/makerfc.html": false, + "drafts/message_extend.txt": false, + "drafts/message_new.txt": false, + "drafts/message_replace.txt": false, + "drafts/message_resurrect.txt": false, + "drafts/message_revision.txt": false, + "drafts/message_update.txt": false, + "drafts/message_withdraw.txt": false, + "drafts/replace.html": false, + "drafts/report_id_activity.txt": false, + "drafts/report_nudge.html": false, + "drafts/report_progress_report.txt": false, + "drafts/revision.html": false, + "drafts/search.html": true, + "drafts/view.html": true, + "drafts/withdraw.html": false, "email_failed.html": false, "googlea30ad1dacffb5e5b.html": true, "group/1wg-charters-by-acronym.txt": true, @@ -2956,6 +2992,15 @@ "group/reset_charter_milestones.html": true, "group/stream_documents.html": true, "group/stream_edit.html": true, + "groups/add.html": true, + "groups/blue_dot_report.txt": false, + "groups/charter.html": false, + "groups/edit.html": false, + "groups/edit_gm.html": false, + "groups/people.html": true, + "groups/search.html": false, + "groups/view.html": true, + "groups/view_gm.html": false, "help/help_html.html": false, "help/help_text.html": false, "help/index.html": false, @@ -2987,6 +3032,37 @@ "iesg/scribe_doc_ballot.html": true, "iesg/scribe_template.html": true, "ietfauth/testemail.html": false, + "includes/activities.html": true, + "includes/awp_add_form.html": true, + "includes/awp_edit_form.html": false, + "includes/awp_view.html": true, + "includes/buttons_back.html": true, + "includes/buttons_next_cancel.html": false, + "includes/buttons_proceed.html": false, + "includes/buttons_save.html": true, + "includes/buttons_save_cancel.html": true, + "includes/buttons_search.html": true, + "includes/buttons_submit.html": false, + "includes/buttons_submit_back.html": true, + "includes/buttons_submit_cancel.html": false, + "includes/docevents.html": true, + "includes/draft_search_results.html": true, + "includes/draft_upload_form.html": false, + "includes/group_search_results.html": false, + "includes/meetings_footer.html": true, + "includes/proceeding_area.html": false, + "includes/proceeding_footer.html": true, + "includes/proceeding_header.html": true, + "includes/proceeding_title.html": false, + "includes/proceedings_functions.html": false, + "includes/search_results_table.html": true, + "includes/session_info.txt": true, + "includes/sessions_footer.html": true, + "includes/sessions_request_form.1_2": false, + "includes/sessions_request_form.html": true, + "includes/sessions_request_view.html": true, + "includes/slides.html": true, + "includes/upload_footer.html": true, "ipr/add_comment.html": true, "ipr/add_email.html": true, "ipr/admin_base.html": true, @@ -3043,6 +3119,7 @@ "liaisons/sdo_reminder.txt": true, "m_base.html": true, "mailinglists/group_archives.html": true, + "main.html": true, "meeting/agenda-utc.html": false, "meeting/agenda.csv": true, "meeting/agenda.html": true, @@ -3067,6 +3144,22 @@ "meeting/session_list.html": false, "meeting/timeslot_edit.html": true, "meeting/week-view.html": true, + "meetings/add.html": false, + "meetings/base_rooms_times.html": true, + "meetings/blue_sheet.html": true, + "meetings/edit_meeting.html": false, + "meetings/main.html": true, + "meetings/non_session.html": true, + "meetings/non_session_edit.html": false, + "meetings/notifications.html": true, + "meetings/rooms.html": true, + "meetings/schedule.html": false, + "meetings/select.html": true, + "meetings/select_group.html": true, + "meetings/session_schedule_notification.txt": true, + "meetings/times.html": true, + "meetings/times_edit.html": false, + "meetings/view.html": true, "message/message.html": true, "nomcom/announcements.html": true, "nomcom/delete_nomcom.html": false, @@ -3109,6 +3202,34 @@ "notify_expirations/body.txt": false, "notify_expirations/subject.txt": false, "person/mail/possible_duplicates.txt": true, + "proceedings/acknowledgement.html": false, + "proceedings/agenda.html": false, + "proceedings/area.html": false, + "proceedings/attendee.html": false, + "proceedings/convert.html": false, + "proceedings/edit_slide.html": false, + "proceedings/index.html": false, + "proceedings/interim_directory.html": true, + "proceedings/interim_meeting.html": false, + "proceedings/interim_select.html": false, + "proceedings/irtf.html": false, + "proceedings/main.html": true, + "proceedings/overview.html": false, + "proceedings/plenary.html": false, + "proceedings/proceedings.html": true, + "proceedings/proceedings_template.html": false, + "proceedings/progress.html": false, + "proceedings/recording.html": true, + "proceedings/recording_edit.html": false, + "proceedings/replace_slide.html": false, + "proceedings/rg_irtf.html": false, + "proceedings/select.html": false, + "proceedings/status.html": false, + "proceedings/training.html": false, + "proceedings/upload_presentation.html": false, + "proceedings/upload_unified.html": true, + "proceedings/view.html": false, + "proceedings/wait.html": false, "registration/add_email_email.txt": false, "registration/base.html": true, "registration/change_password.html": false, @@ -3125,6 +3246,29 @@ "registration/password_reset.html": false, "registration/password_reset_email.txt": false, "release/release.html": false, + "roles/base_roles.html": false, + "roles/chairs.html": false, + "roles/liaisons.html": false, + "roles/main.html": true, + "roles/roles.html": false, + "rolodex/add.html": false, + "rolodex/add_proceed.html": false, + "rolodex/delete.html": false, + "rolodex/edit.html": false, + "rolodex/search.html": true, + "rolodex/view.html": true, + "sreq/confirm.html": false, + "sreq/edit.html": true, + "sreq/locked.html": false, + "sreq/main.1_2": false, + "sreq/main.html": true, + "sreq/new.html": true, + "sreq/not_meeting_notification.txt": false, + "sreq/session_approval_notification.txt": false, + "sreq/session_cancel_notification.txt": false, + "sreq/session_request_notification.txt": false, + "sreq/tool_status.html": false, + "sreq/view.html": true, "submit/add_preapproval.html": true, "submit/announce_new_version.txt": true, "submit/announce_to_authors.txt": true, @@ -3150,14 +3294,24 @@ "sync/discrepancies_report.txt": false, "sync/notify.html": true, "sync/rfceditor_undo.html": true, + "telechat/base_telechat.html": true, + "telechat/bash.html": false, + "telechat/doc.html": true, + "telechat/doc_template.html": false, + "telechat/group.html": false, + "telechat/main.html": true, + "telechat/management.html": false, + "telechat/minutes.html": false, + "telechat/roll_call.html": false, "test/mail_body.txt": false, "test/mail_subject.txt": false, + "unauthorized.html": false, "utils/header_change_content.txt": false } }, - "time": "2015-03-04T18:24:44Z", + "time": "2015-03-06T20:07:18Z", "url": { - "coverage": 0.5374732334047109, + "coverage": 0.5163934426229508, "covered": { "^$": true, "^(?P(wg|rg))/$": true, @@ -3202,6 +3356,7 @@ "^accounts/settings/full_draft/(?P.*)$": false, "^accounts/settings/new_enough/": false, "^accounts/settings/new_enough/(?P.*)$": false, + "^api/v1/": true, "^api/v1/?$": true, "^community/(?P[\\d]+)/remove_document/(?P[^/]+)/$": false, "^community/(?P[\\d]+)/remove_rule/(?P[^/]+)/$": false,