altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Updated buildbot master config file.

Browse source 
- Legacy-Id: 17659
This commit is contained in:
buildbot 2020-04-20 12:00:07 +00:00
parent 4f9a112645
commit be03c9e527
1 changed files with 480 additions and 211 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

691
buildbot/masters/datatracker/master.cfg
View file

@ -1,258 +1,476 @@
# -*- python -*- # Copyright The IETF Trust 2015-2020, All Rights Reserved
# ex: set filetype=python: # -*- python; coding: utf-8 -*-
from os.path import expanduser as expandtilde # This is a buildbot config file for buildbot 0.8.14.p1 (patched to work with
from buildbot.plugins import worker, changes, schedulers, util, steps # workers of version 2.7 in addition to 0.8 workers).
import buildbot_passwords
import custom_steps
import re
from buildbot_passwords import *
# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.
# This is the dictionary that the buildmaster pays attention to. We also use # This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing. # a shorter alias to save typing.
c = BuildmasterConfig = {} c = BuildmasterConfig = {}
####### SETTINGS # -*- section BuildSlaves -*-
# For miscellaneous settings, see MISC. SETTINGS at the bottom of the file # The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password. The same
####### WORKERS # slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave
# The 'workers' list defines the set of recognized workers. Each element is c['slaves'] = [
# a Worker object, specifying a unique worker name and password. The same BuildSlave("datatracker_lin_py27_1", datatracker_lin_py27_1_pw),
# worker name and password must be configured on the worker. BuildSlave("datatracker_lin_py27_2", datatracker_lin_py27_2_pw),
c['workers'] = [ BuildSlave("datatracker_lin_py27_3", datatracker_lin_py27_3_pw),
worker.Worker("datatracker_lin_py36_1", buildbot_passwords.datatracker_lin_py36_1_pw), BuildSlave("datatracker_osx_py27_4", datatracker_osx_py27_4_pw),
worker.Worker("datatracker_lin_py36_2", buildbot_passwords.datatracker_lin_py36_2_pw), BuildSlave("datatracker_lin_py27_5", datatracker_lin_py27_5_pw),
worker.Worker("datatracker_lin_py36_3", buildbot_passwords.datatracker_lin_py36_3_pw), BuildSlave("datatracker_lin_py27_6", datatracker_lin_py27_6_pw),
worker.Worker("datatracker_lin_py36_4", buildbot_passwords.datatracker_lin_py36_4_pw), #
worker.Worker("datatracker_lin_py36_5", buildbot_passwords.datatracker_lin_py36_5_pw), BuildSlave("datatracker_lin_py36_1", datatracker_lin_py36_1_pw),
worker.Worker("datatracker_lin_py36_6", buildbot_passwords.datatracker_lin_py36_6_pw), BuildSlave("datatracker_lin_py36_2", datatracker_lin_py36_2_pw),
BuildSlave("datatracker_lin_py36_3", datatracker_lin_py36_3_pw),
BuildSlave("datatracker_lin_py36_4", datatracker_lin_py36_4_pw),
BuildSlave("datatracker_lin_py36_5", datatracker_lin_py36_5_pw),
BuildSlave("datatracker_lin_py36_6", datatracker_lin_py36_6_pw),
] ]
# 'protocols' contains information about protocols which master will use for # 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers # communicating with slaves.
# could connect to your master with this protocol. # You must define at least 'port' option that slaves could connect to your master
# 'port' must match the value configured into the workers (with their # with this protocol.
# 'port' must match the value configured into the buildslaves (with their
# --master option) # --master option)
c['protocols'] = {'pb': {'port': 9989}} c['protocols'] = {'pb': {'host':'zinfandel.tools.ietf.org', 'port': 9989}}
####### CHANGESOURCES ####### CHANGESOURCES
# -*- section ChangeSources -*-
# the 'change_source' setting tells the buildmaster how it should find out # the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot version of a python hello-world project. # about source code changes.
from buildbot.changes.pb import PBChangeSource
# c['change_source'] = []
# with open("users") as file:
# userinfo = json.read(file)
# for user in userinfo:
# prefix = userinfo[user]["prefix"]
# c.['change_source'].append(PBChangeSource(user=user, passwd="BRiR6XcT7x3$", prefix=prefix))
c['change_source'] = [ c['change_source'] = [
changes.PBChangeSource(user='ietfdb', passwd=buildbot_passwords.ietfdb_svn_hook_pw), PBChangeSource(user="ietfdb", passwd=ietfdb_svn_hook_pw),
] ]
####### SCHEDULERS ####### SCHEDULERS
# -*- section Schedulers -*-
# Configure the Schedulers, which decide how to react to incoming changes. In this # Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'runtests' build # case, just kick off a 'runtests' build
from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.timed import Nightly
from buildbot.changes import filter
c['schedulers'] = [ c['schedulers'] = [
# Branch schedulers # Branch schedulers
schedulers.SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"], SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"],
change_filter=util.ChangeFilter(branch='trunk')), change_filter=filter.ChangeFilter(branch='trunk')),
schedulers.SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"], SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"],
change_filter=util.ChangeFilter(branch='trunk')), change_filter=filter.ChangeFilter(branch='trunk')),
# schedulers.SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"], # SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"],
# change_filter=util.ChangeFilter(branch='trunk')), # change_filter=filter.ChangeFilter(branch='trunk')),
# #
schedulers.AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"], AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"],
change_filter=util.ChangeFilter(branch_re='branch/.*')), change_filter=filter.ChangeFilter(branch_re='branch/.*')),
# schedulers.AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"], # AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"],
# change_filter=util.ChangeFilter(branch_re='branch/.*')), # change_filter=filter.ChangeFilter(branch_re='branch/.*')),
# schedulers.AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"], # AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"],
# change_filter=util.ChangeFilter(branch_re='branch/.*')), # change_filter=filter.ChangeFilter(branch_re='branch/.*')),
# #
schedulers.AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"], AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"],
change_filter=util.ChangeFilter(branch_re='personal/.*')), change_filter=filter.ChangeFilter(branch_re='personal/.*')),
schedulers.AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"], AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"],
change_filter=util.ChangeFilter(branch_re='personal/.*')), change_filter=filter.ChangeFilter(branch_re='personal/.*')),
# Periodic Schedulers # Periodic Schedulers
schedulers.Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],), Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],),
schedulers.Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],), Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],),
schedulers.Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],), Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
# schedulers.Force schedulers # Force schedulers
schedulers.ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]), ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]),
schedulers.ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]), ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]),
# schedulers.ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]), # ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]),
schedulers.ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]), ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]),
# #
schedulers.ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]), ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]),
schedulers.ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]), ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]),
# schedulers.ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]), # ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]),
# #
schedulers.ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]), ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]),
schedulers.ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]), ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]),
] ]
####### BUILDERS ####### BUILDERS
# -*- section Builders -*-
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them. Note that any particular build will # what steps, and which slaves can execute them. Note that any particular build will
# only take place on one worker. # only take place on one slave.
from buildbot.process.factory import BuildFactory
from buildbot.steps.source.svn import SVN
from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand
from buildbot.steps.python import PyFlakes
from buildbot.steps.python_twisted import RemovePYCs
from buildbot.steps.slave import SetPropertiesFromEnv
#
from buildbot.process.properties import Property, Interpolate
from buildbot.config import BuilderConfig
#### Custom subclassed builder #### Custom subclassed builder
factory = util.BuildFactory() class TestCrawlerShellCommand(WarningCountingShellCommand):
# check out the source name = "testcrawl"
factory.addStep(steps.Git(repourl='git://github.com/buildbot/hello-world.git', mode='incremental')) haltOnFailure = 1
# run the tests (note that this will require that 'trial' is installed) flunkOnFailure = 1
factory.addStep(steps.ShellCommand(command=["trial", "hello"], descriptionDone = ["test crawler"]
env={"PYTHONPATH": "."})) command=["bin/test-crawl"]
warningPatterns = {
"exceptions": "^(Traceback| File| |.*Error|.*Exception)",
"failed": " FAIL ",
"warnings": " WARN",
"slow": " SLOW",
"invalid_html": " invalid html:",
}
logline = "^ *(?P<elapsed>\d+:\d+:\d+) +(?P<pages>\d+) +(?P<queue>\d+) +(?P<result>\d+) +(?P<runtime>\d+.\d+)s +(?P<message>.+)"
def setTestResults(self, **kwargs):
"""
Called by subclasses to set the relevant statistics; this actually
adds to any statistics already present
"""
for kw in kwargs:
value = kwargs[kw]
if value.isdigit():
# Counter
value = int(value)
value += self.step_status.getStatistic(kw, 0)
elif re.search("^[0-9]+\.[0-9]+$", value):
# Runtime
value = float(value)
value += self.step_status.getStatistic(kw, 0)
else:
# This is a percentage, and we can't add them
pass
self.step_status.setStatistic(kw, value)
def createSummary(self, log):
"""
Match log lines against warningPattern.
Warnings are collected into another log for this step, and the
build-wide 'warnings-count' is updated."""
warnings = {}
wregex = {}
regex_class = re.compile("").__class__
if not isinstance(self.logline, regex_class):
self.logline = re.compile(self.logline)
for key in self.warningPatterns:
warnings[key] = []
pattern = self.warningPatterns[key]
if not isinstance(pattern, regex_class):
wregex[key] = re.compile(pattern)
else:
wregex[key] = pattern
# Count matches to the various warning patterns
for line in log.getText().split("\n"):
for key in wregex:
match = re.search(wregex[key], line)
if match:
warnings[key].append(line)
if re.search(self.logline, line):
last_line = line
# If there were any warnings, make the log if lines with warnings
# available
for key in warnings:
if len(warnings[key]) > 0:
self.addCompleteLog("%s (%d)" % (key, len(warnings[key])),
"\n".join(warnings[key]) + "\n")
self.step_status.setStatistic(key, len(warnings[key]))
self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand")
match = re.search(self.logline, last_line)
for key in ['elapsed', 'pages']:
info = match.group(key)
self.step_status.setStatistic(key, info)
self.setProperty(key, info, "TestCrawlerShellCommand")
def describe(self, done=False):
description = WarningCountingShellCommand.describe(self, done)
if done:
description = description[:] # make a private copy
for name in ["time", "elapsed", "pages", "failed", "warnings", "slow", "invalid_html", ]:
if name in self.step_status.statistics:
value = self.step_status.getStatistic(name)
displayName = name.replace('_', ' ')
# special case. Mph.
if type(value) is float: # this is run-time
description.append('%s: %.2fs' % (displayName, value))
elif type(value) is int:
description.append('%s: %d' % (displayName, value))
else:
description.append('%s: %s' % (displayName, value))
return description
class UnitTest(WarningCountingShellCommand):
name = "test"
warnOnFailure = 1
description = ["testing"]
descriptionDone = ["test"]
command = ["python", "-m", "unittest", "discover"]
regexPatterns = {
"tests": "Ran (\d+) tests in [0-9.]+s",
"time": "Ran \d+ tests in ([0-9.]+)s",
"skipped": "(?:OK|FAILED).*skipped=(\d+)",
"failed": "FAILED.*failures=(\d+)",
"errors": "FAILED.*errors=(\d+)",
"template_coverage":" +Template coverage: +([0-9.]+%)",
"url_coverage": " +Url coverage: +([0-9.]+%)",
"code_coverage": " +Code coverage: +([0-9.]+%)",
}
def setTestResults(self, **kwargs):
"""
Called by subclasses to set the relevant statistics; this actually
adds to any statistics already present
"""
for kw in kwargs:
value = kwargs[kw]
if value.isdigit():
# Counter
value = int(value)
value += self.step_status.getStatistic(kw, 0)
elif re.search("^[0-9]+\.[0-9]+$", value):
# Runtime
value = float(value)
value += self.step_status.getStatistic(kw, 0)
else:
# This is a percentage, and we can't add them
pass
self.step_status.setStatistic(kw, value)
def createSummary(self, log):
info = {}
for line in log.getText().split("\n"):
for key in self.regexPatterns:
regex = self.regexPatterns[key]
match = re.search(regex, line)
if match:
info[key] = match.group(1)
self.setTestResults(**info)
def describe(self, done=False):
description = WarningCountingShellCommand.describe(self, done)
if done:
description = description[:] # make a private copy
self.step_status.statistics["passed"] = (
self.step_status.getStatistic("tests",0) -
self.step_status.getStatistic("skipped",0) -
self.step_status.getStatistic("failed",0) -
self.step_status.getStatistic("errors",0))
for name in ["time", "tests", "passed", "skipped", "failed", "errors", "template_coverage", "url_coverage", "code_coverage", ]:
if name in self.step_status.statistics:
value = self.step_status.getStatistic(name)
displayName = name.replace('_', ' ')
# special case. Mph.
if displayName == 'template coverage':
displayName = 'templ. coverage'
if type(value) is float: # this is run-time
description.append('%s: %.2fs' % (displayName, value))
elif type(value) is int:
description.append('%s: %d' % (displayName, value))
else:
description.append('%s: %s' % (displayName, value))
return description
## Set up builders
c['builders'] = [] c['builders'] = []
# -*- section Builder_Run_pyflakes -*- # -*- section Builder_Run_pyflakes -*-
factory = util.BuildFactory() factory = BuildFactory()
factory.addStep(steps.SVN( factory.addStep(SVN(
username='buildbot@tools.ietf.org', username='buildbot@tools.ietf.org',
descriptionDone="svn update", descriptionDone="svn update",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), usePTY=False,
descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="seting up settings_local.py", descriptionDone="seting up settings_local.py",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], usePTY=False,
command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
)) ))
factory.addStep(steps.PyFlakes( factory.addStep(PyFlakes(
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["ietf/manage.py", "pyflakes", "--verbosity=0"], command=["ietf/manage.py", "pyflakes", "--verbosity=0"],
)) ))
# This should be the last action # This should be the last action
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="mark as passed", descriptionDone="mark as passed",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
usePTY=False,
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", util.Property('got_revision'), "test:pyflakes", "passed" ], "propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ],
)) ))
c['builders'].append(util.BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk", c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk",
workernames=["datatracker_lin_py36_1", ])) slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ]))
c['builders'].append(util.BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch", c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch",
workernames=["datatracker_lin_py36_2", ])) slavenames=["datatracker_lin_py36_2", ]))
c['builders'].append(util.BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal", c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal",
workernames=["datatracker_lin_py36_3", ])) slavenames=["datatracker_lin_py36_3", ]))
# -*- section Builder_TestSuite -*- # -*- section Builder_TestSuite -*-
factory = util.BuildFactory() factory = BuildFactory()
factory.addStep(steps.SVN( factory.addStep(SVN(
username='buildbot@tools.ietf.org', username='buildbot@tools.ietf.org',
descriptionDone="svn update", descriptionDone="svn update",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), usePTY=False,
descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
)) ))
factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="remove tmp-* dirs", descriptionDone="remove tmp-* dirs",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["rm", "-rf", "tmp-*/"], command=["rm", "-rf", "tmp-*/"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="install requirements", descriptionDone="install requirements",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["pip", "install", "-r", "requirements.txt"], command=["pip", "install", "-r", "requirements.txt"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="copy settings_local.py", descriptionDone="copy settings_local.py",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], usePTY=False,
command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="collect static files", descriptionDone="collect static files",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=False, haltOnFailure=False,
flunkOnFailure=False, flunkOnFailure=False,
usePTY=False,
command=["ietf/manage.py", "collectstatic", "--noinput", ], command=["ietf/manage.py", "collectstatic", "--noinput", ],
)) ))
factory.addStep(custom_steps.DjangoTest( factory.addStep(UnitTest(
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
)) ))
# This should come after tests # This should come after tests
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="mark as passed", descriptionDone="mark as passed",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
usePTY=False,
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", util.Property('got_revision'), "test:unittest", "passed" ], "propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ],
)) ))
c['builders'].append(util.BuilderConfig(name="Test Suite", factory=factory, category="1. trunk", c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
workernames=["datatracker_lin_py36_1", ])) slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ]))
c['builders'].append(util.BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch", c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
workernames=["datatracker_lin_py36_2", ])) slavenames=["datatracker_lin_py36_2", ]))
c['builders'].append(util.BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal", c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
workernames=["datatracker_lin_py36_3", ])) slavenames=["datatracker_lin_py36_3", ]))
# -*- section Builder_TestCrawler -*- # -*- section Builder_TestCrawler -*-
factory = util.BuildFactory() factory = BuildFactory()
factory.addStep(steps.SVN( factory.addStep(SVN(
username='buildbot@tools.ietf.org', username='buildbot@tools.ietf.org',
descriptionDone="svn update", descriptionDone="svn update",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), usePTY=False,
descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
)) ))
factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="install requirements", descriptionDone="install requirements",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["pip", "install", "-r", "requirements.txt"], command=["pip", "install", "-r", "requirements.txt"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="copy settings_local.py", descriptionDone="copy settings_local.py",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], usePTY=False,
command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="run migrations", descriptionDone="run migrations",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["ietf/manage.py", "migrate"], command=["ietf/manage.py", "migrate"],
)) ))
factory.addStep(custom_steps.TestCrawlerShellCommand( factory.addStep(TestCrawlerShellCommand(
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"], command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"],
)) ))
# This should be the last action # This should be the last action
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="mark as passed", descriptionDone="mark as passed",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
usePTY=False,
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", util.Property('got_revision'), "test:crawler", "passed" ], "propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ],
)) ))
c['builders'].append(util.BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk", c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk",
workernames=["datatracker_lin_py36_6", ])) slavenames=["datatracker_lin_py36_6", ]))
# -*- section Builder_Verify_Minimum_Libs -*- # -*- section Builder_Verify_Old_Libs -*-
# This build runs pip install --upgrade, to make sure that we install the earliest version of # This build runs pip install --upgrade, to make sure that we install the earliest version of
# all dependencies, in order to get an indication if/when an incompatibility turns up with a new # all dependencies, in order to get an indication if/when an incompatibility turns up with a new
@ -260,65 +478,74 @@ c['builders'].append(util.BuilderConfig(name="Test-Crawler", factory=factory, ca
# change the external test conditions and produce spurious errors because of version changes in # change the external test conditions and produce spurious errors because of version changes in
# dependencies. # dependencies.
factory = util.BuildFactory() factory = BuildFactory()
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="remove tweaked requirements", descriptionDone="remove tweaked requirements",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=False, haltOnFailure=False,
flunkOnFailure=False, flunkOnFailure=False,
usePTY=False,
command=["rm", "requirements.txt"], command=["rm", "requirements.txt"],
)) ))
factory.addStep(steps.SVN( factory.addStep(SVN(
username='buildbot@tools.ietf.org', username='buildbot@tools.ietf.org',
descriptionDone="svn update", descriptionDone="svn update",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
alwaysUseLatest=True, alwaysUseLatest=True,
haltOnFailure=True, haltOnFailure=True,
repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), usePTY=False,
descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
)) ))
factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="edit requirements", descriptionDone="edit requirements",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"], command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="install/upgrade requirements", descriptionDone="install/upgrade requirements",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["pip", "install", "--upgrade", "-r", "requirements.txt"], command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="seting up settings_local.py", descriptionDone="seting up settings_local.py",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], usePTY=False,
command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="list installed pyton modules", descriptionDone="list installed pyton modules",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["pip", "freeze"], command=["pip", "freeze"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="collect static files", descriptionDone="collect static files",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=False, haltOnFailure=False,
flunkOnFailure=False, flunkOnFailure=False,
usePTY=False,
command=["ietf/manage.py", "collectstatic", "--noinput", ], command=["ietf/manage.py", "collectstatic", "--noinput", ],
)) ))
factory.addStep(custom_steps.DjangoTest( factory.addStep(UnitTest(
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
)) ))
c['builders'].append(util.BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk", c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk",
workernames=["datatracker_lin_py36_5", ])) slavenames=["datatracker_lin_py36_5", ]))
# -*- section Builder_Veryfy_Latest_Libs -*- # -*- section Builder_Dependencies -*-
# This build runs pip install --upgrade, to make sure that we install the latest version of all # This build runs pip install --upgrade, to make sure that we install the latest version of all
# dependencies, in order to get an indication if/when an incompatibility turns up with a new # dependencies, in order to get an indication if/when an incompatibility turns up with a new
@ -326,99 +553,141 @@ c['builders'].append(util.BuilderConfig(name="Verify Minimum Libs", factory=fact
# change the external test conditions and produce spurious errors because of version changes in # change the external test conditions and produce spurious errors because of version changes in
# dependencies. # dependencies.
factory = util.BuildFactory() factory = BuildFactory()
factory.addStep(steps.SVN( factory.addStep(SVN(
username='buildbot@tools.ietf.org', username='buildbot@tools.ietf.org',
descriptionDone="svn update", descriptionDone="svn update",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
alwaysUseLatest=True, alwaysUseLatest=True,
haltOnFailure=True, haltOnFailure=True,
repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), usePTY=False,
descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
)) ))
factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="install/upgrade requirements", descriptionDone="install/upgrade requirements",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["pip", "install", "--upgrade", "-r", "requirements.txt"], command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="seting up settings_local.py", descriptionDone="seting up settings_local.py",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], usePTY=False,
command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="list installed pyton modules", descriptionDone="list installed pyton modules",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["pip", "freeze"], command=["pip", "freeze"],
)) ))
factory.addStep(steps.ShellCommand( factory.addStep(ShellCommand(
descriptionDone="collect static files", descriptionDone="collect static files",
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=False, haltOnFailure=False,
flunkOnFailure=False, flunkOnFailure=False,
usePTY=False,
command=["ietf/manage.py", "collectstatic", "--noinput", ], command=["ietf/manage.py", "collectstatic", "--noinput", ],
)) ))
factory.addStep(custom_steps.DjangoTest( factory.addStep(UnitTest(
workdir=util.Interpolate('build/%(src::branch)s'), workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
usePTY=False,
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
)) ))
c['builders'].append(util.BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk", c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk",
workernames=["datatracker_lin_py36_5", ])) slavenames=["datatracker_lin_py36_5", ]))
####### BUILDBOT SERVICES
# 'services' is a list of BuildbotService items like reporter targets. The ####### STATUS TARGETS
# status of each build will be pushed to these targets. buildbot/reporters/*.py # -*- section StatusTargets -*-
# has a variety to choose from, like IRC bots.
c['services'] = [] # 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.
c['status'] = []
from buildbot.status import html, mail
from buildbot.status.web import authz, auth
authz_cfg=authz.Authz(
# change any of these to True to enable; see the manual for more
# options
auth=auth.BasicAuth([("ietfdb","ietfdb")]),
gracefulShutdown = False,
forceBuild = 'auth', # use this to test your slave once it is set up
forceAllBuilds = False,
pingBuilder = False,
stopBuild = 'auth',
stopAllBuilds = False,
cancelPendingBuild = 'auth',
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
# A second web status with slightly different rendering
from twisted.python import log
def changelinkfilter(html, project):
log.msg(" * changelinkfilter(html='%s', project='%s')" % (html, project))
return html
import jinja2, os
trac_template_loaders = [jinja2.FileSystemLoader(os.path.join(os.getcwd(), 'trac_view'))]
c['status'].append(html.WebStatus(http_port=8011, jinja_loaders=trac_template_loaders,
authz=authz_cfg))
# Email notifications
from zope.interface import implements
from buildbot import interfaces
class UsernameIsEmailAddress():
"This IEmailLookup provider assumes that the svn username is a valid email address."
implements(interfaces.IEmailLookup)
def getAddress(self, name):
return name
c['status'].append(mail.MailNotifier(
fromaddr='buildbot@tools.ietf.org',
sendToInterestedUsers=False,
extraRecipients=['henrik@levkowetz.com',],
mode="problem",
))
c['status'].append(mail.MailNotifier(
fromaddr='buildbot@tools.ietf.org',
lookup=UsernameIsEmailAddress(),
mode="problem",
))
####### PROJECT IDENTITY ####### PROJECT IDENTITY
# -*- section Project -*-
# the 'title' string will appear at the top of this buildbot installation's # the 'title' string will appear at the top of this buildbot
# home pages (linked to the 'titleURL'). # installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.
c['title'] = "Buildbot: IETF Datatracker" c['title'] = "IETF Datatracker"
c['titleURL'] = "https://datatracker.ietf.org/" c['titleURL'] = "https://datatracker.ietf.org/"
# the 'buildbotURL' string should point to the location where the buildbot's # the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in # internal web server (usually the html.WebStatus page) is visible. This
# the 'www' entry below, but with an externally-visible host name which the # typically uses the port number set in the Waterfall 'status' entry, but
# buildbot cannot figure out without some help. # with an externally-visible host name which the buildbot cannot figure out
# without some help.
c['buildbotURL'] = "http://dunkelfelder.tools.ietf.org:8010/"
# minimalistic config to activate new web UI
c['www'] = {
'port': 8010,
'plugins': {
'waterfall_view': True,
'console_view': True,
'grid_view': True,
},
'default_page': 'waterfall_view',
'debug': True,
'auth': util.UserPasswordAuth({"ietfdb": "ietfdb"}),
}
c['buildbotURL'] = "http://zinfandel.tools.ietf.org:8010/"
####### DB URL ####### DB URL
c['db'] = { c['db'] = {
# This specifies what database buildbot uses to store its state. # This specifies what database buildbot uses to store its state. You can leave
# It's easy to start with sqlite, but it's recommended to switch to a dedicated # this at its default for all but the largest installations.
# database, such as PostgreSQL or MySQL, for use in production environments.
# http://docs.buildbot.net/current/manual/configuration/global.html#database-specification
'db_url' : "sqlite:///state.sqlite", 'db_url' : "sqlite:///state.sqlite",
} }
####### MISC. SETTINGS
c['buildbotNetUsageData'] = 'full'