From be03c9e527383379120f446fad966c35914470c5 Mon Sep 17 00:00:00 2001 From: buildbot Date: Mon, 20 Apr 2020 12:00:07 +0000 Subject: [PATCH] Updated buildbot master config file. - Legacy-Id: 17659 --- buildbot/masters/datatracker/master.cfg | 691 ++++++++++++++++-------- 1 file changed, 480 insertions(+), 211 deletions(-) diff --git a/buildbot/masters/datatracker/master.cfg b/buildbot/masters/datatracker/master.cfg index 46e40aca1..dace7868d 100644 --- a/buildbot/masters/datatracker/master.cfg +++ b/buildbot/masters/datatracker/master.cfg @@ -1,258 +1,476 @@ -# -*- python -*- -# ex: set filetype=python: +# Copyright The IETF Trust 2015-2020, All Rights Reserved +# -*- python; coding: utf-8 -*- -from os.path import expanduser as expandtilde -from buildbot.plugins import worker, changes, schedulers, util, steps -import buildbot_passwords -import custom_steps +# This is a buildbot config file for buildbot 0.8.14.p1 (patched to work with +# workers of version 2.7 in addition to 0.8 workers). + + +import re +from buildbot_passwords import * -# This is a sample buildmaster config file. It must be installed as -# 'master.cfg' in your buildmaster's base directory. # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. c = BuildmasterConfig = {} -####### SETTINGS +# -*- section BuildSlaves -*- -# For miscellaneous settings, see MISC. SETTINGS at the bottom of the file - -####### WORKERS - -# The 'workers' list defines the set of recognized workers. Each element is -# a Worker object, specifying a unique worker name and password. The same -# worker name and password must be configured on the worker. -c['workers'] = [ - worker.Worker("datatracker_lin_py36_1", buildbot_passwords.datatracker_lin_py36_1_pw), - worker.Worker("datatracker_lin_py36_2", buildbot_passwords.datatracker_lin_py36_2_pw), - worker.Worker("datatracker_lin_py36_3", buildbot_passwords.datatracker_lin_py36_3_pw), - worker.Worker("datatracker_lin_py36_4", buildbot_passwords.datatracker_lin_py36_4_pw), - worker.Worker("datatracker_lin_py36_5", buildbot_passwords.datatracker_lin_py36_5_pw), - worker.Worker("datatracker_lin_py36_6", buildbot_passwords.datatracker_lin_py36_6_pw), +# The 'slaves' list defines the set of recognized buildslaves. Each element is +# a BuildSlave object, specifying a unique slave name and password. The same +# slave name and password must be configured on the slave. +from buildbot.buildslave import BuildSlave +c['slaves'] = [ + BuildSlave("datatracker_lin_py27_1", datatracker_lin_py27_1_pw), + BuildSlave("datatracker_lin_py27_2", datatracker_lin_py27_2_pw), + BuildSlave("datatracker_lin_py27_3", datatracker_lin_py27_3_pw), + BuildSlave("datatracker_osx_py27_4", datatracker_osx_py27_4_pw), + BuildSlave("datatracker_lin_py27_5", datatracker_lin_py27_5_pw), + BuildSlave("datatracker_lin_py27_6", datatracker_lin_py27_6_pw), +# + BuildSlave("datatracker_lin_py36_1", datatracker_lin_py36_1_pw), + BuildSlave("datatracker_lin_py36_2", datatracker_lin_py36_2_pw), + BuildSlave("datatracker_lin_py36_3", datatracker_lin_py36_3_pw), + BuildSlave("datatracker_lin_py36_4", datatracker_lin_py36_4_pw), + BuildSlave("datatracker_lin_py36_5", datatracker_lin_py36_5_pw), + BuildSlave("datatracker_lin_py36_6", datatracker_lin_py36_6_pw), ] # 'protocols' contains information about protocols which master will use for -# communicating with workers. You must define at least 'port' option that workers -# could connect to your master with this protocol. -# 'port' must match the value configured into the workers (with their +# communicating with slaves. +# You must define at least 'port' option that slaves could connect to your master +# with this protocol. +# 'port' must match the value configured into the buildslaves (with their # --master option) -c['protocols'] = {'pb': {'port': 9989}} +c['protocols'] = {'pb': {'host':'zinfandel.tools.ietf.org', 'port': 9989}} ####### CHANGESOURCES +# -*- section ChangeSources -*- # the 'change_source' setting tells the buildmaster how it should find out -# about source code changes. Here we point to the buildbot version of a python hello-world project. +# about source code changes. +from buildbot.changes.pb import PBChangeSource +# c['change_source'] = [] +# with open("users") as file: +# userinfo = json.read(file) +# for user in userinfo: +# prefix = userinfo[user]["prefix"] +# c.['change_source'].append(PBChangeSource(user=user, passwd="BRiR6XcT7x3$", prefix=prefix)) c['change_source'] = [ - changes.PBChangeSource(user='ietfdb', passwd=buildbot_passwords.ietfdb_svn_hook_pw), + PBChangeSource(user="ietfdb", passwd=ietfdb_svn_hook_pw), ] ####### SCHEDULERS +# -*- section Schedulers -*- # Configure the Schedulers, which decide how to react to incoming changes. In this # case, just kick off a 'runtests' build +from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler +from buildbot.schedulers.forcesched import ForceScheduler +from buildbot.schedulers.timed import Nightly +from buildbot.changes import filter c['schedulers'] = [ # Branch schedulers - schedulers.SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"], - change_filter=util.ChangeFilter(branch='trunk')), - schedulers.SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"], - change_filter=util.ChangeFilter(branch='trunk')), -# schedulers.SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"], -# change_filter=util.ChangeFilter(branch='trunk')), + SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"], + change_filter=filter.ChangeFilter(branch='trunk')), + SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"], + change_filter=filter.ChangeFilter(branch='trunk')), +# SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"], +# change_filter=filter.ChangeFilter(branch='trunk')), # - schedulers.AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"], - change_filter=util.ChangeFilter(branch_re='branch/.*')), -# schedulers.AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"], -# change_filter=util.ChangeFilter(branch_re='branch/.*')), -# schedulers.AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"], -# change_filter=util.ChangeFilter(branch_re='branch/.*')), + AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"], + change_filter=filter.ChangeFilter(branch_re='branch/.*')), +# AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"], +# change_filter=filter.ChangeFilter(branch_re='branch/.*')), +# AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"], +# change_filter=filter.ChangeFilter(branch_re='branch/.*')), # - schedulers.AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"], - change_filter=util.ChangeFilter(branch_re='personal/.*')), - schedulers.AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"], - change_filter=util.ChangeFilter(branch_re='personal/.*')), + AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"], + change_filter=filter.ChangeFilter(branch_re='personal/.*')), + AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"], + change_filter=filter.ChangeFilter(branch_re='personal/.*')), # Periodic Schedulers - schedulers.Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],), - schedulers.Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],), - schedulers.Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],), + Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],), + Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],), + Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],), - # schedulers.Force schedulers - schedulers.ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]), - schedulers.ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]), -# schedulers.ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]), - schedulers.ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]), + # Force schedulers + ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]), + ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]), +# ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]), + ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]), # - schedulers.ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]), - schedulers.ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]), -# schedulers.ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]), + ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]), + ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]), +# ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]), # - schedulers.ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]), - schedulers.ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]), - + ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]), + ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]), ] ####### BUILDERS +# -*- section Builders -*- # The 'builders' list defines the Builders, which tell Buildbot how to perform a build: -# what steps, and which workers can execute them. Note that any particular build will -# only take place on one worker. +# what steps, and which slaves can execute them. Note that any particular build will +# only take place on one slave. +from buildbot.process.factory import BuildFactory +from buildbot.steps.source.svn import SVN +from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand +from buildbot.steps.python import PyFlakes +from buildbot.steps.python_twisted import RemovePYCs +from buildbot.steps.slave import SetPropertiesFromEnv +# +from buildbot.process.properties import Property, Interpolate +from buildbot.config import BuilderConfig #### Custom subclassed builder -factory = util.BuildFactory() -# check out the source -factory.addStep(steps.Git(repourl='git://github.com/buildbot/hello-world.git', mode='incremental')) -# run the tests (note that this will require that 'trial' is installed) -factory.addStep(steps.ShellCommand(command=["trial", "hello"], - env={"PYTHONPATH": "."})) +class TestCrawlerShellCommand(WarningCountingShellCommand): + name = "testcrawl" + haltOnFailure = 1 + flunkOnFailure = 1 + descriptionDone = ["test crawler"] + command=["bin/test-crawl"] + + warningPatterns = { + "exceptions": "^(Traceback| File| |.*Error|.*Exception)", + "failed": " FAIL ", + "warnings": " WARN", + "slow": " SLOW", + "invalid_html": " invalid html:", + } + + logline = "^ *(?P\d+:\d+:\d+) +(?P\d+) +(?P\d+) +(?P\d+) +(?P\d+.\d+)s +(?P.+)" + + def setTestResults(self, **kwargs): + """ + Called by subclasses to set the relevant statistics; this actually + adds to any statistics already present + """ + for kw in kwargs: + value = kwargs[kw] + if value.isdigit(): + # Counter + value = int(value) + value += self.step_status.getStatistic(kw, 0) + elif re.search("^[0-9]+\.[0-9]+$", value): + # Runtime + value = float(value) + value += self.step_status.getStatistic(kw, 0) + else: + # This is a percentage, and we can't add them + pass + self.step_status.setStatistic(kw, value) + + def createSummary(self, log): + """ + Match log lines against warningPattern. + + Warnings are collected into another log for this step, and the + build-wide 'warnings-count' is updated.""" + + warnings = {} + wregex = {} + + regex_class = re.compile("").__class__ + + if not isinstance(self.logline, regex_class): + self.logline = re.compile(self.logline) + + for key in self.warningPatterns: + warnings[key] = [] + pattern = self.warningPatterns[key] + if not isinstance(pattern, regex_class): + wregex[key] = re.compile(pattern) + else: + wregex[key] = pattern + + # Count matches to the various warning patterns + for line in log.getText().split("\n"): + for key in wregex: + match = re.search(wregex[key], line) + if match: + warnings[key].append(line) + if re.search(self.logline, line): + last_line = line + + # If there were any warnings, make the log if lines with warnings + # available + for key in warnings: + if len(warnings[key]) > 0: + self.addCompleteLog("%s (%d)" % (key, len(warnings[key])), + "\n".join(warnings[key]) + "\n") + self.step_status.setStatistic(key, len(warnings[key])) + self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand") + + match = re.search(self.logline, last_line) + for key in ['elapsed', 'pages']: + info = match.group(key) + self.step_status.setStatistic(key, info) + self.setProperty(key, info, "TestCrawlerShellCommand") + + def describe(self, done=False): + description = WarningCountingShellCommand.describe(self, done) + if done: + description = description[:] # make a private copy + for name in ["time", "elapsed", "pages", "failed", "warnings", "slow", "invalid_html", ]: + if name in self.step_status.statistics: + value = self.step_status.getStatistic(name) + displayName = name.replace('_', ' ') + # special case. Mph. + if type(value) is float: # this is run-time + description.append('%s: %.2fs' % (displayName, value)) + elif type(value) is int: + description.append('%s: %d' % (displayName, value)) + else: + description.append('%s: %s' % (displayName, value)) + return description + + +class UnitTest(WarningCountingShellCommand): + + name = "test" + warnOnFailure = 1 + description = ["testing"] + descriptionDone = ["test"] + command = ["python", "-m", "unittest", "discover"] + + regexPatterns = { + "tests": "Ran (\d+) tests in [0-9.]+s", + "time": "Ran \d+ tests in ([0-9.]+)s", + "skipped": "(?:OK|FAILED).*skipped=(\d+)", + "failed": "FAILED.*failures=(\d+)", + "errors": "FAILED.*errors=(\d+)", + "template_coverage":" +Template coverage: +([0-9.]+%)", + "url_coverage": " +Url coverage: +([0-9.]+%)", + "code_coverage": " +Code coverage: +([0-9.]+%)", + } + + def setTestResults(self, **kwargs): + """ + Called by subclasses to set the relevant statistics; this actually + adds to any statistics already present + """ + for kw in kwargs: + value = kwargs[kw] + if value.isdigit(): + # Counter + value = int(value) + value += self.step_status.getStatistic(kw, 0) + elif re.search("^[0-9]+\.[0-9]+$", value): + # Runtime + value = float(value) + value += self.step_status.getStatistic(kw, 0) + else: + # This is a percentage, and we can't add them + pass + self.step_status.setStatistic(kw, value) + + def createSummary(self, log): + info = {} + for line in log.getText().split("\n"): + for key in self.regexPatterns: + regex = self.regexPatterns[key] + match = re.search(regex, line) + if match: + info[key] = match.group(1) + self.setTestResults(**info) + + def describe(self, done=False): + description = WarningCountingShellCommand.describe(self, done) + if done: + description = description[:] # make a private copy + self.step_status.statistics["passed"] = ( + self.step_status.getStatistic("tests",0) - + self.step_status.getStatistic("skipped",0) - + self.step_status.getStatistic("failed",0) - + self.step_status.getStatistic("errors",0)) + for name in ["time", "tests", "passed", "skipped", "failed", "errors", "template_coverage", "url_coverage", "code_coverage", ]: + if name in self.step_status.statistics: + value = self.step_status.getStatistic(name) + displayName = name.replace('_', ' ') + # special case. Mph. + if displayName == 'template coverage': + displayName = 'templ. coverage' + if type(value) is float: # this is run-time + description.append('%s: %.2fs' % (displayName, value)) + elif type(value) is int: + description.append('%s: %d' % (displayName, value)) + else: + description.append('%s: %s' % (displayName, value)) + return description + + +## Set up builders c['builders'] = [] # -*- section Builder_Run_pyflakes -*- -factory = util.BuildFactory() -factory.addStep(steps.SVN( +factory = BuildFactory() +factory.addStep(SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, - repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], + usePTY=False, + repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(steps.ShellCommand( +factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) +factory.addStep(ShellCommand( descriptionDone="seting up settings_local.py", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, - command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], + usePTY=False, + command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"], )) -factory.addStep(steps.PyFlakes( - workdir=util.Interpolate('build/%(src::branch)s'), +factory.addStep(PyFlakes( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["ietf/manage.py", "pyflakes", "--verbosity=0"], )) # This should be the last action -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="mark as passed", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), + usePTY=False, command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", - "propset", "--revprop", "-r", util.Property('got_revision'), "test:pyflakes", "passed" ], + "propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ], )) -c['builders'].append(util.BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk", - workernames=["datatracker_lin_py36_1", ])) -c['builders'].append(util.BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch", - workernames=["datatracker_lin_py36_2", ])) -c['builders'].append(util.BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal", - workernames=["datatracker_lin_py36_3", ])) +c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk", + slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ])) +c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch", + slavenames=["datatracker_lin_py36_2", ])) +c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal", + slavenames=["datatracker_lin_py36_3", ])) # -*- section Builder_TestSuite -*- -factory = util.BuildFactory() -factory.addStep(steps.SVN( +factory = BuildFactory() +factory.addStep(SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, - repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], + usePTY=False, + repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) -factory.addStep(steps.ShellCommand( +factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) +factory.addStep(ShellCommand( descriptionDone="remove tmp-* dirs", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["rm", "-rf", "tmp-*/"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="install requirements", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["pip", "install", "-r", "requirements.txt"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) +factory.addStep(ShellCommand( descriptionDone="copy settings_local.py", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, - command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], + usePTY=False, + command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="collect static files", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=False, flunkOnFailure=False, + usePTY=False, command=["ietf/manage.py", "collectstatic", "--noinput", ], )) -factory.addStep(custom_steps.DjangoTest( - workdir=util.Interpolate('build/%(src::branch)s'), +factory.addStep(UnitTest( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], )) # This should come after tests -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="mark as passed", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), + usePTY=False, command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", - "propset", "--revprop", "-r", util.Property('got_revision'), "test:unittest", "passed" ], + "propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ], )) -c['builders'].append(util.BuilderConfig(name="Test Suite", factory=factory, category="1. trunk", - workernames=["datatracker_lin_py36_1", ])) -c['builders'].append(util.BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch", - workernames=["datatracker_lin_py36_2", ])) -c['builders'].append(util.BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal", - workernames=["datatracker_lin_py36_3", ])) +c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk", + slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ])) +c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch", + slavenames=["datatracker_lin_py36_2", ])) +c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal", + slavenames=["datatracker_lin_py36_3", ])) # -*- section Builder_TestCrawler -*- -factory = util.BuildFactory() -factory.addStep(steps.SVN( +factory = BuildFactory() +factory.addStep(SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, - repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], + usePTY=False, + repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) -factory.addStep(steps.ShellCommand( +factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) +factory.addStep(ShellCommand( descriptionDone="install requirements", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["pip", "install", "-r", "requirements.txt"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) +factory.addStep(ShellCommand( descriptionDone="copy settings_local.py", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, - command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], + usePTY=False, + command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="run migrations", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["ietf/manage.py", "migrate"], )) -factory.addStep(custom_steps.TestCrawlerShellCommand( - workdir=util.Interpolate('build/%(src::branch)s'), +factory.addStep(TestCrawlerShellCommand( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"], )) # This should be the last action -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="mark as passed", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), + usePTY=False, command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", - "propset", "--revprop", "-r", util.Property('got_revision'), "test:crawler", "passed" ], + "propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ], )) -c['builders'].append(util.BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk", - workernames=["datatracker_lin_py36_6", ])) +c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk", + slavenames=["datatracker_lin_py36_6", ])) -# -*- section Builder_Verify_Minimum_Libs -*- +# -*- section Builder_Verify_Old_Libs -*- # This build runs pip install --upgrade, to make sure that we install the earliest version of # all dependencies, in order to get an indication if/when an incompatibility turns up with a new @@ -260,65 +478,74 @@ c['builders'].append(util.BuilderConfig(name="Test-Crawler", factory=factory, ca # change the external test conditions and produce spurious errors because of version changes in # dependencies. -factory = util.BuildFactory() -factory.addStep(steps.ShellCommand( +factory = BuildFactory() +factory.addStep(ShellCommand( descriptionDone="remove tweaked requirements", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=False, flunkOnFailure=False, + usePTY=False, command=["rm", "requirements.txt"], )) -factory.addStep(steps.SVN( +factory.addStep(SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), alwaysUseLatest=True, haltOnFailure=True, - repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], + usePTY=False, + repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) -factory.addStep(steps.ShellCommand( +factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) +factory.addStep(ShellCommand( descriptionDone="edit requirements", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="install/upgrade requirements", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["pip", "install", "--upgrade", "-r", "requirements.txt"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) +factory.addStep(ShellCommand( descriptionDone="seting up settings_local.py", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, - command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], + usePTY=False, + command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="list installed pyton modules", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["pip", "freeze"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="collect static files", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=False, flunkOnFailure=False, + usePTY=False, command=["ietf/manage.py", "collectstatic", "--noinput", ], )) -factory.addStep(custom_steps.DjangoTest( - workdir=util.Interpolate('build/%(src::branch)s'), +factory.addStep(UnitTest( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], )) -c['builders'].append(util.BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk", - workernames=["datatracker_lin_py36_5", ])) +c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk", + slavenames=["datatracker_lin_py36_5", ])) -# -*- section Builder_Veryfy_Latest_Libs -*- +# -*- section Builder_Dependencies -*- # This build runs pip install --upgrade, to make sure that we install the latest version of all # dependencies, in order to get an indication if/when an incompatibility turns up with a new @@ -326,99 +553,141 @@ c['builders'].append(util.BuilderConfig(name="Verify Minimum Libs", factory=fact # change the external test conditions and produce spurious errors because of version changes in # dependencies. -factory = util.BuildFactory() -factory.addStep(steps.SVN( +factory = BuildFactory() +factory.addStep(SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), alwaysUseLatest=True, haltOnFailure=True, - repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], + usePTY=False, + repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) -factory.addStep(steps.ShellCommand( +factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) +factory.addStep(ShellCommand( descriptionDone="install/upgrade requirements", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["pip", "install", "--upgrade", "-r", "requirements.txt"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) +factory.addStep(ShellCommand( descriptionDone="seting up settings_local.py", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, - command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], + usePTY=False, + command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="list installed pyton modules", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["pip", "freeze"], )) -factory.addStep(steps.ShellCommand( +factory.addStep(ShellCommand( descriptionDone="collect static files", - workdir=util.Interpolate('build/%(src::branch)s'), + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=False, flunkOnFailure=False, + usePTY=False, command=["ietf/manage.py", "collectstatic", "--noinput", ], )) -factory.addStep(custom_steps.DjangoTest( - workdir=util.Interpolate('build/%(src::branch)s'), +factory.addStep(UnitTest( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + usePTY=False, command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], )) -c['builders'].append(util.BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk", - workernames=["datatracker_lin_py36_5", ])) +c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk", + slavenames=["datatracker_lin_py36_5", ])) -####### BUILDBOT SERVICES -# 'services' is a list of BuildbotService items like reporter targets. The -# status of each build will be pushed to these targets. buildbot/reporters/*.py -# has a variety to choose from, like IRC bots. +####### STATUS TARGETS +# -*- section StatusTargets -*- -c['services'] = [] +# 'status' is a list of Status Targets. The results of each build will be +# pushed to these targets. buildbot/status/*.py has a variety to choose from, +# including web pages, email senders, and IRC bots. + +c['status'] = [] + +from buildbot.status import html, mail +from buildbot.status.web import authz, auth + +authz_cfg=authz.Authz( + # change any of these to True to enable; see the manual for more + # options + auth=auth.BasicAuth([("ietfdb","ietfdb")]), + gracefulShutdown = False, + forceBuild = 'auth', # use this to test your slave once it is set up + forceAllBuilds = False, + pingBuilder = False, + stopBuild = 'auth', + stopAllBuilds = False, + cancelPendingBuild = 'auth', +) +c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) + +# A second web status with slightly different rendering +from twisted.python import log +def changelinkfilter(html, project): + log.msg(" * changelinkfilter(html='%s', project='%s')" % (html, project)) + return html + +import jinja2, os +trac_template_loaders = [jinja2.FileSystemLoader(os.path.join(os.getcwd(), 'trac_view'))] +c['status'].append(html.WebStatus(http_port=8011, jinja_loaders=trac_template_loaders, + authz=authz_cfg)) + +# Email notifications +from zope.interface import implements +from buildbot import interfaces + +class UsernameIsEmailAddress(): + "This IEmailLookup provider assumes that the svn username is a valid email address." + implements(interfaces.IEmailLookup) + def getAddress(self, name): + return name + +c['status'].append(mail.MailNotifier( + fromaddr='buildbot@tools.ietf.org', + sendToInterestedUsers=False, + extraRecipients=['henrik@levkowetz.com',], + mode="problem", + )) + +c['status'].append(mail.MailNotifier( + fromaddr='buildbot@tools.ietf.org', + lookup=UsernameIsEmailAddress(), + mode="problem", + )) ####### PROJECT IDENTITY +# -*- section Project -*- -# the 'title' string will appear at the top of this buildbot installation's -# home pages (linked to the 'titleURL'). +# the 'title' string will appear at the top of this buildbot +# installation's html.WebStatus home page (linked to the +# 'titleURL') and is embedded in the title of the waterfall HTML page. -c['title'] = "Buildbot: IETF Datatracker" +c['title'] = "IETF Datatracker" c['titleURL'] = "https://datatracker.ietf.org/" # the 'buildbotURL' string should point to the location where the buildbot's -# internal web server is visible. This typically uses the port number set in -# the 'www' entry below, but with an externally-visible host name which the -# buildbot cannot figure out without some help. - -c['buildbotURL'] = "http://dunkelfelder.tools.ietf.org:8010/" - -# minimalistic config to activate new web UI -c['www'] = { - 'port': 8010, - 'plugins': { - 'waterfall_view': True, - 'console_view': True, - 'grid_view': True, - }, - 'default_page': 'waterfall_view', - 'debug': True, - 'auth': util.UserPasswordAuth({"ietfdb": "ietfdb"}), -} +# internal web server (usually the html.WebStatus page) is visible. This +# typically uses the port number set in the Waterfall 'status' entry, but +# with an externally-visible host name which the buildbot cannot figure out +# without some help. +c['buildbotURL'] = "http://zinfandel.tools.ietf.org:8010/" ####### DB URL c['db'] = { - # This specifies what database buildbot uses to store its state. - # It's easy to start with sqlite, but it's recommended to switch to a dedicated - # database, such as PostgreSQL or MySQL, for use in production environments. - # http://docs.buildbot.net/current/manual/configuration/global.html#database-specification + # This specifies what database buildbot uses to store its state. You can leave + # this at its default for all but the largest installations. 'db_url' : "sqlite:///state.sqlite", } - -####### MISC. SETTINGS - -c['buildbotNetUsageData'] = 'full' -