From aecbeb6abfe1defef80fea63b51052c3a01c146a Mon Sep 17 00:00:00 2001 From: buildbot Date: Mon, 12 Sep 2016 11:16:43 +0000 Subject: [PATCH] Made the error buildbot counting for the test-crawler more sophisticated. - Legacy-Id: 11983 --- buildbot/masters/datatracker/master.cfg | 113 ++++++++++++++++++++++-- 1 file changed, 107 insertions(+), 6 deletions(-) diff --git a/buildbot/masters/datatracker/master.cfg b/buildbot/masters/datatracker/master.cfg index e739df9da..5cc206414 100644 --- a/buildbot/masters/datatracker/master.cfg +++ b/buildbot/masters/datatracker/master.cfg @@ -82,7 +82,7 @@ c['schedulers'] = [ # Periodic Schedulers Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],), Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],), - NightlyTriggerable(name="crawler", hour=[5,7,9,11,13,15,17], minute=00, builderNames=["Test-Crawler"],), + NightlyTriggerable(name="crawler", hour=[5,7,9,11,13,15,17], minute=00, builderNames=["Test-Crawler"],), # Force schedulers ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]), @@ -124,7 +124,102 @@ class TestCrawlerShellCommand(WarningCountingShellCommand): flunkOnFailure = 1 descriptionDone = ["test crawler"] command=["bin/test-crawl"] - warningPattern = '.* FAIL ' + + warningPatterns = { + "exceptions": "^Traceback", + "failed": " FAIL", + "warnings": " WARN", + "slow": " SLOW", + "invalid_html": " invalid html:", + } + + logline = "^ *(?P\d+:\d+:\d+) +(?P\d+) +(?P\d+) +(?P\d+) +(?P\d+.\d+)s +(?P.+)" + + def setTestResults(self, **kwargs): + """ + Called by subclasses to set the relevant statistics; this actually + adds to any statistics already present + """ + for kw in kwargs: + value = kwargs[kw] + if value.isdigit(): + # Counter + value = int(value) + value += self.step_status.getStatistic(kw, 0) + elif re.search("^[0-9]+\.[0-9]+$", value): + # Runtime + value = float(value) + value += self.step_status.getStatistic(kw, 0) + else: + # This is a percentage, and we can't add them + pass + self.step_status.setStatistic(kw, value) + + def createSummary(self, log): + """ + Match log lines against warningPattern. + + Warnings are collected into another log for this step, and the + build-wide 'warnings-count' is updated.""" + + warnings = {} + wregex = {} + + regex_class = re.compile("").__class__ + + if not isinstance(self.logline, regex_class): + self.logline = re.compile(self.logline) + + for key in self.warningPatterns: + warnings[key] = [] + pattern = self.warningPatterns[key] + if not isinstance(pattern, regex_class): + wregex[key] = re.compile(pattern) + else: + wregex[key] = pattern + + # Count matches to the various warning patterns + for line in log.getText().split("\n"): + for key in wregex: + match = re.search(wregex[key], line) + if match: + warnings[key].append(line) + if re.search(self.logline, line): + last_line = line + + # If there were any warnings, make the log if lines with warnings + # available + log = syslog.syslog + for key in warnings: + if len(warnings[key]) > 0: + self.addCompleteLog("%s (%d)" % (key, len(warnings[key])), + "\n".join(warnings[key]) + "\n") + self.step_status.setStatistic(key, len(warnings[key])) + self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand") + + match = re.search(self.logline, last_line) + for key in ['elapsed', 'pages']: + info = match.group(key) + self.step_status.setStatistic(key, info) + self.setProperty(key, info, "TestCrawlerShellCommand") + + def describe(self, done=False): + description = WarningCountingShellCommand.describe(self, done) + if done: + description = description[:] # make a private copy + for name in ["time", "elapsed", "pages", "failed", "warnings", "slow", "invalid_html", ]: + if name in self.step_status.statistics: + value = self.step_status.getStatistic(name) + displayName = name.replace('_', ' ') + # special case. Mph. + if type(value) is float: # this is run-time + description.append('%s: %.2fs' % (displayName, value)) + elif type(value) is int: + description.append('%s: %d' % (displayName, value)) + else: + description.append('%s: %s' % (displayName, value)) + return description + class UnitTest(WarningCountingShellCommand): @@ -156,7 +251,7 @@ class UnitTest(WarningCountingShellCommand): # Counter value = int(value) value += self.step_status.getStatistic(kw, 0) - elif re.match("[0-9]+\.[0-9]+$", value): + elif re.search("^[0-9]+\.[0-9]+$", value): # Runtime value = float(value) value += self.step_status.getStatistic(kw, 0) @@ -170,7 +265,7 @@ class UnitTest(WarningCountingShellCommand): for line in log.getText().split("\n"): for key in self.regexPatterns: regex = self.regexPatterns[key] - match = re.match(regex, line) + match = re.search(regex, line) if match: info[key] = match.group(1) self.setTestResults(**info) @@ -198,7 +293,7 @@ class UnitTest(WarningCountingShellCommand): else: description.append('%s: %s' % (displayName, value)) return description - + ## Set up builders @@ -259,6 +354,12 @@ factory.addStep(SVN( descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) +factory.addStep(ShellCommand( + descriptionDone="remove tmp-* dirs", + workdir=Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["rm", "-rf", "tmp-*/"], + )) factory.addStep(ShellCommand( descriptionDone="install requirements", workdir=Interpolate('build/%(src::branch)s'), @@ -300,7 +401,7 @@ c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory # to the steps above factory.addStep(Trigger(schedulerNames=['crawler'], waitForFinish=False, - updateSourceStamp=False, + alwaysUseLatest=True, set_properties={}, ))