diff --git a/buildbot/masters/datatracker/master.cfg b/buildbot/masters/datatracker/master.cfg
index e739df9da..5cc206414 100644
--- a/buildbot/masters/datatracker/master.cfg
+++ b/buildbot/masters/datatracker/master.cfg
@@ -82,7 +82,7 @@ c['schedulers'] = [
# Periodic Schedulers
Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],),
Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],),
- NightlyTriggerable(name="crawler", hour=[5,7,9,11,13,15,17], minute=00, builderNames=["Test-Crawler"],),
+ NightlyTriggerable(name="crawler", hour=[5,7,9,11,13,15,17], minute=00, builderNames=["Test-Crawler"],),
# Force schedulers
ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]),
@@ -124,7 +124,102 @@ class TestCrawlerShellCommand(WarningCountingShellCommand):
flunkOnFailure = 1
descriptionDone = ["test crawler"]
command=["bin/test-crawl"]
- warningPattern = '.* FAIL '
+
+ warningPatterns = {
+ "exceptions": "^Traceback",
+ "failed": " FAIL",
+ "warnings": " WARN",
+ "slow": " SLOW",
+ "invalid_html": " invalid html:",
+ }
+
+ logline = "^ *(?P<elapsed>\d+:\d+:\d+) +(?P<pages>\d+) +(?P<queue>\d+) +(?P<result>\d+) +(?P<runtime>\d+.\d+)s +(?P<message>.+)"
+
+ def setTestResults(self, **kwargs):
+ """
+ Called by subclasses to set the relevant statistics; this actually
+ adds to any statistics already present
+ """
+ for kw in kwargs:
+ value = kwargs[kw]
+ if value.isdigit():
+ # Counter
+ value = int(value)
+ value += self.step_status.getStatistic(kw, 0)
+ elif re.search("^[0-9]+\.[0-9]+$", value):
+ # Runtime
+ value = float(value)
+ value += self.step_status.getStatistic(kw, 0)
+ else:
+ # This is a percentage, and we can't add them
+ pass
+ self.step_status.setStatistic(kw, value)
+
+ def createSummary(self, log):
+ """
+ Match log lines against warningPattern.
+
+ Warnings are collected into another log for this step, and the
+ build-wide 'warnings-count' is updated."""
+
+ warnings = {}
+ wregex = {}
+
+ regex_class = re.compile("").__class__
+
+ if not isinstance(self.logline, regex_class):
+ self.logline = re.compile(self.logline)
+
+ for key in self.warningPatterns:
+ warnings[key] = []
+ pattern = self.warningPatterns[key]
+ if not isinstance(pattern, regex_class):
+ wregex[key] = re.compile(pattern)
+ else:
+ wregex[key] = pattern
+
+ # Count matches to the various warning patterns
+ for line in log.getText().split("\n"):
+ for key in wregex:
+ match = re.search(wregex[key], line)
+ if match:
+ warnings[key].append(line)
+ if re.search(self.logline, line):
+ last_line = line
+
+ # If there were any warnings, make the log if lines with warnings
+ # available
+ log = syslog.syslog
+ for key in warnings:
+ if len(warnings[key]) > 0:
+ self.addCompleteLog("%s (%d)" % (key, len(warnings[key])),
+ "\n".join(warnings[key]) + "\n")
+ self.step_status.setStatistic(key, len(warnings[key]))
+ self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand")
+
+ match = re.search(self.logline, last_line)
+ for key in ['elapsed', 'pages']:
+ info = match.group(key)
+ self.step_status.setStatistic(key, info)
+ self.setProperty(key, info, "TestCrawlerShellCommand")
+
+ def describe(self, done=False):
+ description = WarningCountingShellCommand.describe(self, done)
+ if done:
+ description = description[:] # make a private copy
+ for name in ["time", "elapsed", "pages", "failed", "warnings", "slow", "invalid_html", ]:
+ if name in self.step_status.statistics:
+ value = self.step_status.getStatistic(name)
+ displayName = name.replace('_', ' ')
+ # special case. Mph.
+ if type(value) is float: # this is run-time
+ description.append('%s: %.2fs' % (displayName, value))
+ elif type(value) is int:
+ description.append('%s: %d' % (displayName, value))
+ else:
+ description.append('%s: %s' % (displayName, value))
+ return description
+
class UnitTest(WarningCountingShellCommand):
@@ -156,7 +251,7 @@ class UnitTest(WarningCountingShellCommand):
# Counter
value = int(value)
value += self.step_status.getStatistic(kw, 0)
- elif re.match("[0-9]+\.[0-9]+$", value):
+ elif re.search("^[0-9]+\.[0-9]+$", value):
# Runtime
value = float(value)
value += self.step_status.getStatistic(kw, 0)
@@ -170,7 +265,7 @@ class UnitTest(WarningCountingShellCommand):
for line in log.getText().split("\n"):
for key in self.regexPatterns:
regex = self.regexPatterns[key]
- match = re.match(regex, line)
+ match = re.search(regex, line)
if match:
info[key] = match.group(1)
self.setTestResults(**info)
@@ -198,7 +293,7 @@ class UnitTest(WarningCountingShellCommand):
else:
description.append('%s: %s' % (displayName, value))
return description
-
+
## Set up builders
@@ -259,6 +354,12 @@ factory.addStep(SVN(
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
+factory.addStep(ShellCommand(
+ descriptionDone="remove tmp-* dirs",
+ workdir=Interpolate('build/%(src::branch)s'),
+ haltOnFailure=True,
+ command=["rm", "-rf", "tmp-*/"],
+ ))
factory.addStep(ShellCommand(
descriptionDone="install requirements",
workdir=Interpolate('build/%(src::branch)s'),
@@ -300,7 +401,7 @@ c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory
# to the steps above
factory.addStep(Trigger(schedulerNames=['crawler'],
waitForFinish=False,
- updateSourceStamp=False,
+ alwaysUseLatest=True,
set_properties={},
))