717 lines
30 KiB
INI
717 lines
30 KiB
INI
# Copyright The IETF Trust 2015-2020, All Rights Reserved
|
|
# -*- python; coding: utf-8 -*-
|
|
|
|
# This is a buildbot config file for buildbot 0.8.14.p1 (patched to work with
|
|
# workers of version 2.7 in addition to 0.8 workers).
|
|
|
|
|
|
import re
|
|
from buildbot_passwords import *
|
|
|
|
|
|
# This is the dictionary that the buildmaster pays attention to. We also use
|
|
# a shorter alias to save typing.
|
|
c = BuildmasterConfig = {}
|
|
|
|
# -*- section BuildSlaves -*-
|
|
|
|
# The 'slaves' list defines the set of recognized buildslaves. Each element is
|
|
# a BuildSlave object, specifying a unique slave name and password. The same
|
|
# slave name and password must be configured on the slave.
|
|
from buildbot.buildslave import BuildSlave
|
|
c['slaves'] = [
|
|
#
|
|
BuildSlave("dunkelfelder_lin_py36_1", dunkelfelder_lin_py36_1_pw),
|
|
BuildSlave("dunkelfelder_lin_py36_2", dunkelfelder_lin_py36_2_pw),
|
|
BuildSlave("dunkelfelder_lin_py36_3", dunkelfelder_lin_py36_3_pw),
|
|
BuildSlave("dunkelfelder_lin_py36_4", dunkelfelder_lin_py36_4_pw),
|
|
|
|
BuildSlave("dornfelder_lin_py36_1", dornfelder_lin_py36_1_pw),
|
|
BuildSlave("dornfelder_lin_py36_2", dornfelder_lin_py36_2_pw),
|
|
BuildSlave("dornfelder_lin_py36_3", dornfelder_lin_py36_3_pw),
|
|
BuildSlave("dornfelder_lin_py36_4", dornfelder_lin_py36_4_pw),
|
|
]
|
|
|
|
# 'protocols' contains information about protocols which master will use for
|
|
# communicating with slaves.
|
|
# You must define at least 'port' option that slaves could connect to your master
|
|
# with this protocol.
|
|
# 'port' must match the value configured into the buildslaves (with their
|
|
# --master option)
|
|
c['protocols'] = {'pb': {'host':'zinfandel.tools.ietf.org', 'port': 9989}}
|
|
|
|
####### CHANGESOURCES
|
|
# -*- section ChangeSources -*-
|
|
|
|
# the 'change_source' setting tells the buildmaster how it should find out
|
|
# about source code changes.
|
|
|
|
from buildbot.changes.pb import PBChangeSource
|
|
# c['change_source'] = []
|
|
# with open("users") as file:
|
|
# userinfo = json.read(file)
|
|
# for user in userinfo:
|
|
# prefix = userinfo[user]["prefix"]
|
|
# c.['change_source'].append(PBChangeSource(user=user, passwd="BRiR6XcT7x3$", prefix=prefix))
|
|
c['change_source'] = [
|
|
PBChangeSource(user="ietfdb", passwd=ietfdb_svn_hook_pw),
|
|
]
|
|
|
|
####### SCHEDULERS
|
|
# -*- section Schedulers -*-
|
|
|
|
# Configure the Schedulers, which decide how to react to incoming changes. In this
|
|
# case, just kick off a 'runtests' build
|
|
|
|
from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler
|
|
from buildbot.schedulers.forcesched import ForceScheduler
|
|
from buildbot.schedulers.timed import Nightly
|
|
from buildbot.changes import filter
|
|
c['schedulers'] = [
|
|
# Branch schedulers
|
|
SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"],
|
|
change_filter=filter.ChangeFilter(branch='trunk')),
|
|
SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"],
|
|
change_filter=filter.ChangeFilter(branch='trunk')),
|
|
# SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"],
|
|
# change_filter=filter.ChangeFilter(branch='trunk')),
|
|
#
|
|
AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"],
|
|
change_filter=filter.ChangeFilter(branch_re='branch/.*')),
|
|
# AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"],
|
|
# change_filter=filter.ChangeFilter(branch_re='branch/.*')),
|
|
# AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"],
|
|
# change_filter=filter.ChangeFilter(branch_re='branch/.*')),
|
|
#
|
|
AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"],
|
|
change_filter=filter.ChangeFilter(branch_re='personal/.*')),
|
|
AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"],
|
|
change_filter=filter.ChangeFilter(branch_re='personal/.*')),
|
|
# Periodic Schedulers
|
|
Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],),
|
|
Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],),
|
|
Nightly(name="crawler", hour=23, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
|
|
|
|
# Force schedulers
|
|
ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]),
|
|
ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]),
|
|
# ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]),
|
|
ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]),
|
|
#
|
|
ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]),
|
|
ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]),
|
|
# ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]),
|
|
#
|
|
ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]),
|
|
ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]),
|
|
]
|
|
|
|
####### BUILDERS
|
|
# -*- section Builders -*-
|
|
|
|
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
|
|
# what steps, and which slaves can execute them. Note that any particular build will
|
|
# only take place on one slave.
|
|
|
|
from buildbot.process.factory import BuildFactory
|
|
from buildbot.steps.source.svn import SVN
|
|
from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand
|
|
from buildbot.steps.python import PyFlakes
|
|
from buildbot.steps.python_twisted import RemovePYCs
|
|
from buildbot.steps.slave import SetPropertiesFromEnv
|
|
#
|
|
from buildbot.process.properties import Property, Interpolate
|
|
from buildbot.config import BuilderConfig
|
|
|
|
#### Custom subclassed builder
|
|
|
|
class TestCrawlerShellCommand(WarningCountingShellCommand):
|
|
name = "testcrawl"
|
|
haltOnFailure = 1
|
|
flunkOnFailure = 1
|
|
descriptionDone = ["test crawler"]
|
|
command=["bin/test-crawl"]
|
|
|
|
warningPatterns = {
|
|
"exceptions": "^(Traceback| File| |.*Error|.*Exception)",
|
|
"failed": " FAIL ",
|
|
"warnings": " WARN",
|
|
"slow": " SLOW",
|
|
"invalid_html": " invalid html:",
|
|
}
|
|
|
|
logline = "^ *(?P<elapsed>\d+:\d+:\d+) +(?P<pages>\d+) +(?P<queue>\d+) +(?P<result>\d+) +(?P<runtime>\d+.\d+)s +(?P<message>.+)"
|
|
|
|
def setTestResults(self, **kwargs):
|
|
"""
|
|
Called by subclasses to set the relevant statistics; this actually
|
|
adds to any statistics already present
|
|
"""
|
|
for kw in kwargs:
|
|
value = kwargs[kw]
|
|
if value.isdigit():
|
|
# Counter
|
|
value = int(value)
|
|
value += self.step_status.getStatistic(kw, 0)
|
|
elif re.search("^[0-9]+\.[0-9]+$", value):
|
|
# Runtime
|
|
value = float(value)
|
|
value += self.step_status.getStatistic(kw, 0)
|
|
else:
|
|
# This is a percentage, and we can't add them
|
|
pass
|
|
self.step_status.setStatistic(kw, value)
|
|
|
|
def createSummary(self, log):
|
|
"""
|
|
Match log lines against warningPattern.
|
|
|
|
Warnings are collected into another log for this step, and the
|
|
build-wide 'warnings-count' is updated."""
|
|
|
|
warnings = {}
|
|
wregex = {}
|
|
|
|
regex_class = re.compile("").__class__
|
|
|
|
if not isinstance(self.logline, regex_class):
|
|
self.logline = re.compile(self.logline)
|
|
|
|
for key in self.warningPatterns:
|
|
warnings[key] = []
|
|
pattern = self.warningPatterns[key]
|
|
if not isinstance(pattern, regex_class):
|
|
wregex[key] = re.compile(pattern)
|
|
else:
|
|
wregex[key] = pattern
|
|
|
|
# Count matches to the various warning patterns
|
|
for line in log.getText().split("\n"):
|
|
for key in wregex:
|
|
match = re.search(wregex[key], line)
|
|
if match:
|
|
warnings[key].append(line)
|
|
if re.search(self.logline, line):
|
|
last_line = line
|
|
|
|
# If there were any warnings, make the log if lines with warnings
|
|
# available
|
|
for key in warnings:
|
|
if len(warnings[key]) > 0:
|
|
self.addCompleteLog("%s (%d)" % (key, len(warnings[key])),
|
|
"\n".join(warnings[key]) + "\n")
|
|
self.step_status.setStatistic(key, len(warnings[key]))
|
|
self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand")
|
|
|
|
match = re.search(self.logline, last_line)
|
|
for key in ['elapsed', 'pages']:
|
|
info = match.group(key)
|
|
self.step_status.setStatistic(key, info)
|
|
self.setProperty(key, info, "TestCrawlerShellCommand")
|
|
|
|
def describe(self, done=False):
|
|
description = WarningCountingShellCommand.describe(self, done)
|
|
if done:
|
|
description = description[:] # make a private copy
|
|
for name in ["time", "elapsed", "pages", "failed", "warnings", "slow", "invalid_html", ]:
|
|
if name in self.step_status.statistics:
|
|
value = self.step_status.getStatistic(name)
|
|
displayName = name.replace('_', ' ')
|
|
# special case. Mph.
|
|
if type(value) is float: # this is run-time
|
|
description.append('%s: %.2fs' % (displayName, value))
|
|
elif type(value) is int:
|
|
description.append('%s: %d' % (displayName, value))
|
|
else:
|
|
description.append('%s: %s' % (displayName, value))
|
|
return description
|
|
|
|
|
|
class UnitTest(WarningCountingShellCommand):
|
|
|
|
name = "test"
|
|
warnOnFailure = 1
|
|
description = ["testing"]
|
|
descriptionDone = ["test"]
|
|
command = ["python", "-m", "unittest", "discover"]
|
|
|
|
regexPatterns = {
|
|
"tests": "Ran (\d+) tests in [0-9.]+s",
|
|
"time": "Ran \d+ tests in ([0-9.]+)s",
|
|
"skipped": "(?:OK|FAILED).*skipped=(\d+)",
|
|
"failed": "FAILED.*failures=(\d+)",
|
|
"errors": "FAILED.*errors=(\d+)",
|
|
"template_coverage":" +Template coverage: +([0-9.]+%)",
|
|
"url_coverage": " +Url coverage: +([0-9.]+%)",
|
|
"code_coverage": " +Code coverage: +([0-9.]+%)",
|
|
}
|
|
|
|
def setTestResults(self, **kwargs):
|
|
"""
|
|
Called by subclasses to set the relevant statistics; this actually
|
|
adds to any statistics already present
|
|
"""
|
|
for kw in kwargs:
|
|
value = kwargs[kw]
|
|
if value.isdigit():
|
|
# Counter
|
|
value = int(value)
|
|
value += self.step_status.getStatistic(kw, 0)
|
|
elif re.search("^[0-9]+\.[0-9]+$", value):
|
|
# Runtime
|
|
value = float(value)
|
|
value += self.step_status.getStatistic(kw, 0)
|
|
else:
|
|
# This is a percentage, and we can't add them
|
|
pass
|
|
self.step_status.setStatistic(kw, value)
|
|
|
|
def createSummary(self, log):
|
|
info = {}
|
|
for line in log.getText().split("\n"):
|
|
for key in self.regexPatterns:
|
|
regex = self.regexPatterns[key]
|
|
match = re.search(regex, line)
|
|
if match:
|
|
info[key] = match.group(1)
|
|
self.setTestResults(**info)
|
|
|
|
def describe(self, done=False):
|
|
description = WarningCountingShellCommand.describe(self, done)
|
|
if done:
|
|
description = description[:] # make a private copy
|
|
self.step_status.statistics["passed"] = (
|
|
self.step_status.getStatistic("tests",0) -
|
|
self.step_status.getStatistic("skipped",0) -
|
|
self.step_status.getStatistic("failed",0) -
|
|
self.step_status.getStatistic("errors",0))
|
|
for name in ["time", "tests", "passed", "skipped", "failed", "errors", "template_coverage", "url_coverage", "code_coverage", ]:
|
|
if name in self.step_status.statistics:
|
|
value = self.step_status.getStatistic(name)
|
|
displayName = name.replace('_', ' ')
|
|
# special case. Mph.
|
|
if displayName == 'template coverage':
|
|
displayName = 'templ. coverage'
|
|
if type(value) is float: # this is run-time
|
|
description.append('%s: %.2fs' % (displayName, value))
|
|
elif type(value) is int:
|
|
description.append('%s: %d' % (displayName, value))
|
|
else:
|
|
description.append('%s: %s' % (displayName, value))
|
|
return description
|
|
|
|
|
|
## Set up builders
|
|
|
|
c['builders'] = []
|
|
|
|
# -*- section Builder_Run_pyflakes -*-
|
|
|
|
factory = BuildFactory()
|
|
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
|
|
factory.addStep(SVN(
|
|
username='buildbot@tools.ietf.org',
|
|
descriptionDone="svn update",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
|
|
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="install requirements",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["pip", "install", "-r", "requirements.txt"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="seting up settings_local.py",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
|
|
))
|
|
factory.addStep(PyFlakes(
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["ietf/manage.py", "pyflakes", "--verbosity=0"],
|
|
))
|
|
# This should be the last action
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="mark as passed",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
flunkOnFailure=False,
|
|
usePTY=False,
|
|
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
|
|
"propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ],
|
|
))
|
|
|
|
c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk",
|
|
slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
|
|
c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch",
|
|
slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
|
|
c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal",
|
|
slavenames=["dunkelfelder_lin_py36_2",]))
|
|
|
|
# -*- section Builder_TestSuite -*-
|
|
|
|
factory = BuildFactory()
|
|
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
|
|
factory.addStep(SVN(
|
|
username='buildbot@tools.ietf.org',
|
|
descriptionDone="svn update",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
|
|
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
|
|
))
|
|
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="remove tmp-* dirs",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["rm", "-rf", "tmp-*/"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="install requirements",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["pip", "install", "-r", "requirements.txt"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="copy settings_local.py",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="collect static files",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=False,
|
|
flunkOnFailure=False,
|
|
usePTY=False,
|
|
command=["ietf/manage.py", "collectstatic", "--noinput", ],
|
|
))
|
|
factory.addStep(UnitTest(
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
|
|
))
|
|
# This should come after tests
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="mark as passed",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
flunkOnFailure=False,
|
|
usePTY=False,
|
|
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
|
|
"propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ],
|
|
))
|
|
|
|
c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
|
|
slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
|
|
c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
|
|
slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
|
|
c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
|
|
slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
|
|
|
|
|
|
# -*- section Builder_TestCrawler -*-
|
|
|
|
factory = BuildFactory()
|
|
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="update database",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
timeout=3600, # 1 hour
|
|
command=["docker/updatedb", "-q"],
|
|
))
|
|
factory.addStep(SVN(
|
|
username='buildbot@tools.ietf.org',
|
|
descriptionDone="svn update",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
|
|
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
|
|
))
|
|
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="install requirements",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["pip", "install", "-r", "requirements.txt"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="copy settings_local.py",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="run migrations",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["ietf/manage.py", "migrate"],
|
|
))
|
|
# This will not only do a prelimnary sanity check, but also patch libs as needed:
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="run django checks",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["ietf/manage.py", "check"],
|
|
))
|
|
factory.addStep(TestCrawlerShellCommand(
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"],
|
|
))
|
|
# This should be the last action
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="mark as passed",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
flunkOnFailure=False,
|
|
usePTY=False,
|
|
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
|
|
"propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ],
|
|
))
|
|
|
|
c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk",
|
|
slavenames=["dunkelfelder_lin_py36_4", ]))
|
|
|
|
|
|
# -*- section Builder_Verify_Old_Libs -*-
|
|
|
|
# This build runs pip install --upgrade, to make sure that we install the earliest version of
|
|
# all dependencies, in order to get an indication if/when an incompatibility turns up with a new
|
|
# version of a dependency. The other test suite builders *don't* use --upgrade, in order to not
|
|
# change the external test conditions and produce spurious errors because of version changes in
|
|
# dependencies.
|
|
|
|
factory = BuildFactory()
|
|
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="remove tweaked requirements",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=False,
|
|
flunkOnFailure=False,
|
|
usePTY=False,
|
|
command=["rm", "requirements.txt"],
|
|
))
|
|
factory.addStep(SVN(
|
|
username='buildbot@tools.ietf.org',
|
|
descriptionDone="svn update",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
alwaysUseLatest=True,
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
|
|
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
|
|
))
|
|
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="edit requirements",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="install/upgrade requirements",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="seting up settings_local.py",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="list installed pyton modules",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["pip", "freeze"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="collect static files",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=False,
|
|
flunkOnFailure=False,
|
|
usePTY=False,
|
|
command=["ietf/manage.py", "collectstatic", "--noinput", ],
|
|
))
|
|
factory.addStep(UnitTest(
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
|
|
))
|
|
c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk",
|
|
slavenames=["dornfelder_lin_py36_3", ]))
|
|
|
|
|
|
# -*- section Verify_Latest_Libs -*-
|
|
|
|
# This build runs pip install --upgrade, to make sure that we install the latest version of all
|
|
# dependencies, in order to get an indication if/when an incompatibility turns up with a new
|
|
# version of a dependency. The other test suite builders *don't* use --upgrade, in order to not
|
|
# change the external test conditions and produce spurious errors because of version changes in
|
|
# dependencies.
|
|
|
|
factory = BuildFactory()
|
|
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
|
|
factory.addStep(SVN(
|
|
username='buildbot@tools.ietf.org',
|
|
descriptionDone="svn update",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
alwaysUseLatest=True,
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
|
|
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
|
|
))
|
|
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="install/upgrade requirements",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="seting up settings_local.py",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["cp", Interpolate("%(prop:HOME)s/settings_local.py"), "./ietf/"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="list installed pyton modules",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["pip", "freeze"],
|
|
))
|
|
factory.addStep(ShellCommand(
|
|
descriptionDone="collect static files",
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=False,
|
|
flunkOnFailure=False,
|
|
usePTY=False,
|
|
command=["ietf/manage.py", "collectstatic", "--noinput", ],
|
|
))
|
|
factory.addStep(UnitTest(
|
|
workdir=Interpolate('build/%(src::branch)s'),
|
|
haltOnFailure=True,
|
|
usePTY=False,
|
|
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
|
|
))
|
|
|
|
c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk",
|
|
slavenames=["dornfelder_lin_py36_3", ]))
|
|
|
|
|
|
####### STATUS TARGETS
|
|
# -*- section StatusTargets -*-
|
|
|
|
# 'status' is a list of Status Targets. The results of each build will be
|
|
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
|
|
# including web pages, email senders, and IRC bots.
|
|
|
|
c['status'] = []
|
|
|
|
from buildbot.status import html, mail
|
|
from buildbot.status.web import authz, auth
|
|
|
|
authz_cfg=authz.Authz(
|
|
# change any of these to True to enable; see the manual for more
|
|
# options
|
|
auth=auth.BasicAuth([("ietfdb","ietfdb")]),
|
|
gracefulShutdown = False,
|
|
forceBuild = 'auth', # use this to test your slave once it is set up
|
|
forceAllBuilds = False,
|
|
pingBuilder = False,
|
|
stopBuild = 'auth',
|
|
stopAllBuilds = False,
|
|
cancelPendingBuild = 'auth',
|
|
)
|
|
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
|
|
|
|
# A second web status with slightly different rendering
|
|
from twisted.python import log
|
|
def changelinkfilter(html, project):
|
|
log.msg(" * changelinkfilter(html='%s', project='%s')" % (html, project))
|
|
return html
|
|
|
|
import jinja2, os
|
|
trac_template_loaders = [jinja2.FileSystemLoader(os.path.join(os.getcwd(), 'trac_view'))]
|
|
c['status'].append(html.WebStatus(http_port=8011, jinja_loaders=trac_template_loaders,
|
|
authz=authz_cfg))
|
|
|
|
# Email notifications
|
|
from zope.interface import implements
|
|
from buildbot import interfaces
|
|
|
|
class UsernameIsEmailAddress():
|
|
"This IEmailLookup provider assumes that the svn username is a valid email address."
|
|
implements(interfaces.IEmailLookup)
|
|
def getAddress(self, name):
|
|
return name
|
|
|
|
c['status'].append(mail.MailNotifier(
|
|
fromaddr='buildbot@tools.ietf.org',
|
|
sendToInterestedUsers=False,
|
|
extraRecipients=['rjsparks@nostrum.com',],
|
|
mode="problem",
|
|
))
|
|
|
|
c['status'].append(mail.MailNotifier(
|
|
fromaddr='buildbot@tools.ietf.org',
|
|
lookup=UsernameIsEmailAddress(),
|
|
mode="problem",
|
|
))
|
|
|
|
####### PROJECT IDENTITY
|
|
# -*- section Project -*-
|
|
|
|
# the 'title' string will appear at the top of this buildbot
|
|
# installation's html.WebStatus home page (linked to the
|
|
# 'titleURL') and is embedded in the title of the waterfall HTML page.
|
|
|
|
c['title'] = "Buildbot: IETF Datatracker"
|
|
c['titleURL'] = "https://datatracker.ietf.org/"
|
|
|
|
# the 'buildbotURL' string should point to the location where the buildbot's
|
|
# internal web server (usually the html.WebStatus page) is visible. This
|
|
# typically uses the port number set in the Waterfall 'status' entry, but
|
|
# with an externally-visible host name which the buildbot cannot figure out
|
|
# without some help.
|
|
|
|
c['buildbotURL'] = "http://zinfandel.tools.ietf.org:8010/"
|
|
|
|
####### DB URL
|
|
|
|
c['db'] = {
|
|
# This specifies what database buildbot uses to store its state. You can leave
|
|
# this at its default for all but the largest installations.
|
|
'db_url' : "sqlite:///state.sqlite",
|
|
}
|