altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

Updated buildbot from 0.8 to 2.7, transitioned to Python 3, and added some convenience scripts.

Browse source 
- Legacy-Id: 17657
This commit is contained in:
buildbot 2020-04-19 13:21:49 +00:00
parent 19a26dc39a
commit cdaa2bc437
17 changed files with 590 additions and 480 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

181
buildbot/masters/datatracker/custom_steps.py Normal file
View file

@ -0,0 +1,181 @@
# Copyright The IETF Trust 2015-2020, All Rights Reserved
# -*- coding: utf-8 -*-
import re
from buildbot.plugins import steps
class TestCrawlerShellCommand(steps.WarningCountingShellCommand):
name = "testcrawl"
haltOnFailure = 1
flunkOnFailure = 1
descriptionDone = ["test crawler"]
command=["bin/test-crawl"]
warningPatterns = {
"exceptions": "^(Traceback| File| |.*Error|.*Exception)",
"failed": " FAIL ",
"warnings": " WARN",
"slow": " SLOW",
"invalid_html": " invalid html:",
}
logline = "^ *(?P<elapsed>\d+:\d+:\d+) +(?P<pages>\d+) +(?P<queue>\d+) +(?P<result>\d+) +(?P<runtime>\d+.\d+)s +(?P<message>.+)"
def setTestResults(self, **kwargs):
"""
Called by subclasses to set the relevant statistics; this actually
adds to any statistics already present
"""
for kw in kwargs:
value = kwargs[kw]
if value.isdigit():
# Counter
value = int(value)
value += self.step_status.getStatistic(kw, 0)
elif re.search("^[0-9]+\.[0-9]+$", value):
# Runtime
value = float(value)
value += self.step_status.getStatistic(kw, 0)
else:
# This is a percentage, and we can't add them
pass
self.step_status.setStatistic(kw, value)
def createSummary(self, log):
"""
Match log lines against warningPattern.
Warnings are collected into another log for this step, and the
build-wide 'warnings-count' is updated."""
warnings = {}
wregex = {}
regex_class = re.compile("").__class__
if not isinstance(self.logline, regex_class):
self.logline = re.compile(self.logline)
for key in self.warningPatterns:
warnings[key] = []
pattern = self.warningPatterns[key]
if not isinstance(pattern, regex_class):
wregex[key] = re.compile(pattern)
else:
wregex[key] = pattern
# Count matches to the various warning patterns
for line in log.getText().split("\n"):
for key in wregex:
match = re.search(wregex[key], line)
if match:
warnings[key].append(line)
if re.search(self.logline, line):
last_line = line
# If there were any warnings, make the log if lines with warnings
# available
for key in warnings:
if len(warnings[key]) > 0:
self.addCompleteLog("%s (%d)" % (key, len(warnings[key])),
"\n".join(warnings[key]) + "\n")
self.step_status.setStatistic(key, len(warnings[key]))
self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand")
match = re.search(self.logline, last_line)
for key in ['elapsed', 'pages']:
info = match.group(key)
self.step_status.setStatistic(key, info)
self.setProperty(key, info, "TestCrawlerShellCommand")
def describe(self, done=False):
description = steps.WarningCountingShellCommand.describe(self, done)
if done:
description = description[:] # make a private copy
for name in ["time", "elapsed", "pages", "failed", "warnings", "slow", "invalid_html", ]:
if name in self.step_status.statistics:
value = self.step_status.getStatistic(name)
displayName = name.replace('_', ' ')
# special case. Mph.
if type(value) is float: # this is run-time
description.append('%s: %.2fs' % (displayName, value))
elif type(value) is int:
description.append('%s: %d' % (displayName, value))
else:
description.append('%s: %s' % (displayName, value))
return description
class DjangoTest(steps.WarningCountingShellCommand):
name = "test"
warnOnFailure = 1
description = ["testing"]
descriptionDone = ["test"]
command = ["python", "-m", "unittest", "discover"]
regexPatterns = {
"tests": "Ran (\d+) tests in [0-9.]+s",
"time": "Ran \d+ tests in ([0-9.]+)s",
"skipped": "(?:OK|FAILED).*skipped=(\d+)",
"failed": "FAILED.*failures=(\d+)",
"errors": "FAILED.*errors=(\d+)",
"template_coverage":" +Template coverage: +([0-9.]+%)",
"url_coverage": " +Url coverage: +([0-9.]+%)",
"code_coverage": " +Code coverage: +([0-9.]+%)",
}
def setTestResults(self, **kwargs):
"""
Called by subclasses to set the relevant statistics; this actually
adds to any statistics already present
"""
for kw in kwargs:
value = kwargs[kw]
if value.isdigit():
# Counter
value = int(value)
value += self.step_status.getStatistic(kw, 0)
elif re.search("^[0-9]+\.[0-9]+$", value):
# Runtime
value = float(value)
value += self.step_status.getStatistic(kw, 0)
else:
# This is a percentage, and we can't add them
pass
self.step_status.setStatistic(kw, value)
def createSummary(self, log):
info = {}
for line in log.getText().split("\n"):
for key in self.regexPatterns:
regex = self.regexPatterns[key]
match = re.search(regex, line)
if match:
info[key] = match.group(1)
self.setTestResults(**info)
def describe(self, done=False):
description = steps.WarningCountingShellCommand.describe(self, done)
if done:
description = description[:] # make a private copy
self.step_status.statistics["passed"] = (
self.step_status.getStatistic("tests",0) -
self.step_status.getStatistic("skipped",0) -
self.step_status.getStatistic("failed",0) -
self.step_status.getStatistic("errors",0))
for name in ["time", "tests", "passed", "skipped", "failed", "errors", "template_coverage", "url_coverage", "code_coverage", ]:
if name in self.step_status.statistics:
value = self.step_status.getStatistic(name)
displayName = name.replace('_', ' ')
# special case. Mph.
if displayName == 'template coverage':
displayName = 'templ. coverage'
if type(value) is float: # this is run-time
description.append('%s: %.2fs' % (displayName, value))
elif type(value) is int:
description.append('%s: %d' % (displayName, value))
else:
description.append('%s: %s' % (displayName, value))
return description

691
buildbot/masters/datatracker/master.cfg
View file

@ -1,492 +1,258 @@
# -*- python -*- # -*- python -*-
# ex: set syntax=python: # ex: set filetype=python:
import re
from buildbot_passwords import *
from os.path import expanduser as expandtilde
from buildbot.plugins import worker, changes, schedulers, util, steps
import buildbot_passwords
import custom_steps
# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.
# This is the dictionary that the buildmaster pays attention to. We also use # This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing. # a shorter alias to save typing.
c = BuildmasterConfig = {} c = BuildmasterConfig = {}
# -*- section BuildSlaves -*- ####### SETTINGS
# The 'slaves' list defines the set of recognized buildslaves. Each element is # For miscellaneous settings, see MISC. SETTINGS at the bottom of the file
# a BuildSlave object, specifying a unique slave name and password. The same
# slave name and password must be configured on the slave. ####### WORKERS
from buildbot.buildslave import BuildSlave
c['slaves'] = [ # The 'workers' list defines the set of recognized workers. Each element is
BuildSlave("datatracker_lin_py27_1", datatracker_lin_py27_1_pw), # a Worker object, specifying a unique worker name and password. The same
BuildSlave("datatracker_lin_py27_2", datatracker_lin_py27_2_pw), # worker name and password must be configured on the worker.
BuildSlave("datatracker_lin_py27_3", datatracker_lin_py27_3_pw), c['workers'] = [
BuildSlave("datatracker_osx_py27_4", datatracker_osx_py27_4_pw), worker.Worker("datatracker_lin_py36_1", buildbot_passwords.datatracker_lin_py36_1_pw),
BuildSlave("datatracker_lin_py27_5", datatracker_lin_py27_5_pw), worker.Worker("datatracker_lin_py36_2", buildbot_passwords.datatracker_lin_py36_2_pw),
BuildSlave("datatracker_lin_py27_6", datatracker_lin_py27_6_pw), worker.Worker("datatracker_lin_py36_3", buildbot_passwords.datatracker_lin_py36_3_pw),
worker.Worker("datatracker_lin_py36_4", buildbot_passwords.datatracker_lin_py36_4_pw),
worker.Worker("datatracker_lin_py36_5", buildbot_passwords.datatracker_lin_py36_5_pw),
worker.Worker("datatracker_lin_py36_6", buildbot_passwords.datatracker_lin_py36_6_pw),
] ]
# 'protocols' contains information about protocols which master will use for # 'protocols' contains information about protocols which master will use for
# communicating with slaves. # communicating with workers. You must define at least 'port' option that workers
# You must define at least 'port' option that slaves could connect to your master # could connect to your master with this protocol.
# with this protocol. # 'port' must match the value configured into the workers (with their
# 'port' must match the value configured into the buildslaves (with their
# --master option) # --master option)
c['protocols'] = {'pb': {'host':'zinfandel.tools.ietf.org', 'port': 9989}} c['protocols'] = {'pb': {'port': 9989}}
####### CHANGESOURCES ####### CHANGESOURCES
# -*- section ChangeSources -*-
# the 'change_source' setting tells the buildmaster how it should find out # the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. # about source code changes. Here we point to the buildbot version of a python hello-world project.
from buildbot.changes.pb import PBChangeSource
# c['change_source'] = []
# with open("users") as file:
# userinfo = json.read(file)
# for user in userinfo:
# prefix = userinfo[user]["prefix"]
# c.['change_source'].append(PBChangeSource(user=user, passwd="BRiR6XcT7x3$", prefix=prefix))
c['change_source'] = [ c['change_source'] = [
PBChangeSource(user="ietfdb", passwd=ietfdb_svn_hook_pw), changes.PBChangeSource(user='ietfdb', passwd=buildbot_passwords.ietfdb_svn_hook_pw),
] ]
####### SCHEDULERS ####### SCHEDULERS
# -*- section Schedulers -*-
# Configure the Schedulers, which decide how to react to incoming changes. In this # Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'runtests' build # case, just kick off a 'runtests' build
from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.timed import Nightly
from buildbot.changes import filter
c['schedulers'] = [ c['schedulers'] = [
# Branch schedulers # Branch schedulers
SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"], schedulers.SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"],
change_filter=filter.ChangeFilter(branch='trunk')), change_filter=util.ChangeFilter(branch='trunk')),
SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"], schedulers.SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"],
change_filter=filter.ChangeFilter(branch='trunk')), change_filter=util.ChangeFilter(branch='trunk')),
SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"], # schedulers.SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"],
change_filter=filter.ChangeFilter(branch='trunk')), # change_filter=util.ChangeFilter(branch='trunk')),
# #
AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"], schedulers.AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"],
change_filter=filter.ChangeFilter(branch_re='branch/.*')), change_filter=util.ChangeFilter(branch_re='branch/.*')),
AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"], # schedulers.AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"],
change_filter=filter.ChangeFilter(branch_re='branch/.*')), # change_filter=util.ChangeFilter(branch_re='branch/.*')),
AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"], # schedulers.AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"],
change_filter=filter.ChangeFilter(branch_re='branch/.*')), # change_filter=util.ChangeFilter(branch_re='branch/.*')),
# #
AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"], schedulers.AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"],
change_filter=filter.ChangeFilter(branch_re='personal/.*')), change_filter=util.ChangeFilter(branch_re='personal/.*')),
AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"], schedulers.AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"],
change_filter=filter.ChangeFilter(branch_re='personal/.*')), change_filter=util.ChangeFilter(branch_re='personal/.*')),
# Periodic Schedulers # Periodic Schedulers
Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],), schedulers.Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],),
Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],), schedulers.Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],),
Nightly(name="crawler", hour=[8,9,10,11,12,13,14,15,16,17], minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],), schedulers.Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
# Force schedulers # schedulers.Force schedulers
ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]), schedulers.ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]),
ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]), schedulers.ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]),
ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]), # schedulers.ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]),
ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]), schedulers.ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]),
# #
ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]), schedulers.ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]),
ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]), schedulers.ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]),
ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]), # schedulers.ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]),
# #
ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]), schedulers.ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]),
ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]), schedulers.ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]),
] ]
####### BUILDERS ####### BUILDERS
# -*- section Builders -*-
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them. Note that any particular build will # what steps, and which workers can execute them. Note that any particular build will
# only take place on one slave. # only take place on one worker.
from buildbot.process.factory import BuildFactory
from buildbot.steps.source.svn import SVN
from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand
from buildbot.steps.python import PyFlakes
from buildbot.steps.python_twisted import RemovePYCs
#
from buildbot.process.properties import Property, Interpolate
from buildbot.config import BuilderConfig
#### Custom subclassed builder #### Custom subclassed builder
class TestCrawlerShellCommand(WarningCountingShellCommand): factory = util.BuildFactory()
name = "testcrawl" # check out the source
haltOnFailure = 1 factory.addStep(steps.Git(repourl='git://github.com/buildbot/hello-world.git', mode='incremental'))
flunkOnFailure = 1 # run the tests (note that this will require that 'trial' is installed)
descriptionDone = ["test crawler"] factory.addStep(steps.ShellCommand(command=["trial", "hello"],
command=["bin/test-crawl"] env={"PYTHONPATH": "."}))
warningPatterns = {
"exceptions": "^(Traceback| File| |.*Error|.*Exception)",
"failed": " FAIL ",
"warnings": " WARN",
"slow": " SLOW",
"invalid_html": " invalid html:",
}
logline = "^ *(?P<elapsed>\d+:\d+:\d+) +(?P<pages>\d+) +(?P<queue>\d+) +(?P<result>\d+) +(?P<runtime>\d+.\d+)s +(?P<message>.+)"
def setTestResults(self, **kwargs):
"""
Called by subclasses to set the relevant statistics; this actually
adds to any statistics already present
"""
for kw in kwargs:
value = kwargs[kw]
if value.isdigit():
# Counter
value = int(value)
value += self.step_status.getStatistic(kw, 0)
elif re.search("^[0-9]+\.[0-9]+$", value):
# Runtime
value = float(value)
value += self.step_status.getStatistic(kw, 0)
else:
# This is a percentage, and we can't add them
pass
self.step_status.setStatistic(kw, value)
def createSummary(self, log):
"""
Match log lines against warningPattern.
Warnings are collected into another log for this step, and the
build-wide 'warnings-count' is updated."""
warnings = {}
wregex = {}
regex_class = re.compile("").__class__
if not isinstance(self.logline, regex_class):
self.logline = re.compile(self.logline)
for key in self.warningPatterns:
warnings[key] = []
pattern = self.warningPatterns[key]
if not isinstance(pattern, regex_class):
wregex[key] = re.compile(pattern)
else:
wregex[key] = pattern
# Count matches to the various warning patterns
for line in log.getText().split("\n"):
for key in wregex:
match = re.search(wregex[key], line)
if match:
warnings[key].append(line)
if re.search(self.logline, line):
last_line = line
# If there were any warnings, make the log if lines with warnings
# available
for key in warnings:
if len(warnings[key]) > 0:
self.addCompleteLog("%s (%d)" % (key, len(warnings[key])),
"\n".join(warnings[key]) + "\n")
self.step_status.setStatistic(key, len(warnings[key]))
self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand")
match = re.search(self.logline, last_line)
for key in ['elapsed', 'pages']:
info = match.group(key)
self.step_status.setStatistic(key, info)
self.setProperty(key, info, "TestCrawlerShellCommand")
def describe(self, done=False):
description = WarningCountingShellCommand.describe(self, done)
if done:
description = description[:] # make a private copy
for name in ["time", "elapsed", "pages", "failed", "warnings", "slow", "invalid_html", ]:
if name in self.step_status.statistics:
value = self.step_status.getStatistic(name)
displayName = name.replace('_', ' ')
# special case. Mph.
if type(value) is float: # this is run-time
description.append('%s: %.2fs' % (displayName, value))
elif type(value) is int:
description.append('%s: %d' % (displayName, value))
else:
description.append('%s: %s' % (displayName, value))
return description
class UnitTest(WarningCountingShellCommand):
name = "test"
warnOnFailure = 1
description = ["testing"]
descriptionDone = ["test"]
command = ["python", "-m", "unittest", "discover"]
regexPatterns = {
"tests": "Ran (\d+) tests in [0-9.]+s",
"time": "Ran \d+ tests in ([0-9.]+)s",
"skipped": "(?:OK|FAILED).*skipped=(\d+)",
"failed": "FAILED.*failures=(\d+)",
"errors": "FAILED.*errors=(\d+)",
"template_coverage":" +Template coverage: +([0-9.]+%)",
"url_coverage": " +Url coverage: +([0-9.]+%)",
"code_coverage": " +Code coverage: +([0-9.]+%)",
}
def setTestResults(self, **kwargs):
"""
Called by subclasses to set the relevant statistics; this actually
adds to any statistics already present
"""
for kw in kwargs:
value = kwargs[kw]
if value.isdigit():
# Counter
value = int(value)
value += self.step_status.getStatistic(kw, 0)
elif re.search("^[0-9]+\.[0-9]+$", value):
# Runtime
value = float(value)
value += self.step_status.getStatistic(kw, 0)
else:
# This is a percentage, and we can't add them
pass
self.step_status.setStatistic(kw, value)
def createSummary(self, log):
info = {}
for line in log.getText().split("\n"):
for key in self.regexPatterns:
regex = self.regexPatterns[key]
match = re.search(regex, line)
if match:
info[key] = match.group(1)
self.setTestResults(**info)
def describe(self, done=False):
description = WarningCountingShellCommand.describe(self, done)
if done:
description = description[:] # make a private copy
self.step_status.statistics["passed"] = (
self.step_status.getStatistic("tests",0) -
self.step_status.getStatistic("skipped",0) -
self.step_status.getStatistic("failed",0) -
self.step_status.getStatistic("errors",0))
for name in ["time", "tests", "passed", "skipped", "failed", "errors", "template_coverage", "url_coverage", "code_coverage", ]:
if name in self.step_status.statistics:
value = self.step_status.getStatistic(name)
displayName = name.replace('_', ' ')
# special case. Mph.
if displayName == 'template coverage':
displayName = 'templ. coverage'
if type(value) is float: # this is run-time
description.append('%s: %.2fs' % (displayName, value))
elif type(value) is int:
description.append('%s: %d' % (displayName, value))
else:
description.append('%s: %s' % (displayName, value))
return description
## Set up builders
c['builders'] = [] c['builders'] = []
# -*- section Builder_Run_pyflakes -*- # -*- section Builder_Run_pyflakes -*-
factory = BuildFactory() factory = util.BuildFactory()
factory.addStep(SVN( factory.addStep(steps.SVN(
username='buildbot@tools.ietf.org', username='buildbot@tools.ietf.org',
descriptionDone="svn update", descriptionDone="svn update",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="seting up settings_local.py", descriptionDone="seting up settings_local.py",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["cp", "/home/buildbot/settings_local.py", "./"], command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
)) ))
factory.addStep(PyFlakes( factory.addStep(steps.PyFlakes(
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["ietf/manage.py", "pyflakes", "--verbosity=0"], command=["ietf/manage.py", "pyflakes", "--verbosity=0"],
)) ))
# This should be the last action # This should be the last action
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="mark as passed", descriptionDone="mark as passed",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ], "propset", "--revprop", "-r", util.Property('got_revision'), "test:pyflakes", "passed" ],
)) ))
c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk", c['builders'].append(util.BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk",
slavenames=["datatracker_lin_py27_1", ])) workernames=["datatracker_lin_py36_1", ]))
c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch", c['builders'].append(util.BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch",
slavenames=["datatracker_lin_py27_2", ])) workernames=["datatracker_lin_py36_2", ]))
c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal", c['builders'].append(util.BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal",
slavenames=["datatracker_lin_py27_3", ])) workernames=["datatracker_lin_py36_3", ]))
# -*- section Builder_TestSuite -*- # -*- section Builder_TestSuite -*-
factory = BuildFactory() factory = util.BuildFactory()
factory.addStep(SVN( factory.addStep(steps.SVN(
username='buildbot@tools.ietf.org', username='buildbot@tools.ietf.org',
descriptionDone="svn update", descriptionDone="svn update",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
)) ))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="remove tmp-* dirs", descriptionDone="remove tmp-* dirs",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["rm", "-rf", "tmp-*/"], command=["rm", "-rf", "tmp-*/"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="install requirements", descriptionDone="install requirements",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["pip", "install", "-r", "requirements.txt"], command=["pip", "install", "-r", "requirements.txt"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="copy settings_local.py", descriptionDone="copy settings_local.py",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["cp", "/home/buildbot/settings_local.py", "./"], command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="collect static files", descriptionDone="collect static files",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=False, haltOnFailure=False,
flunkOnFailure=False, flunkOnFailure=False,
command=["ietf/manage.py", "collectstatic", "--noinput", ], command=["ietf/manage.py", "collectstatic", "--noinput", ],
)) ))
factory.addStep(UnitTest( factory.addStep(custom_steps.DjangoTest(
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
)) ))
# This should come after tests # This should come after tests
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="mark as passed", descriptionDone="mark as passed",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ], "propset", "--revprop", "-r", util.Property('got_revision'), "test:unittest", "passed" ],
)) ))
c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk", c['builders'].append(util.BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
slavenames=["datatracker_lin_py27_1", ])) workernames=["datatracker_lin_py36_1", ]))
c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch", c['builders'].append(util.BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
slavenames=["datatracker_lin_py27_2", ])) workernames=["datatracker_lin_py36_2", ]))
c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal", c['builders'].append(util.BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
slavenames=["datatracker_lin_py27_3", ])) workernames=["datatracker_lin_py36_3", ]))
# -*- section Builder_TestSuiteOSX -*-
factory = BuildFactory()
factory.addStep(ShellCommand(
descriptionDone="svn cleanup",
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=False,
flunkOnFailure=False,
command=["svn", "cleanup"],
))
factory.addStep(SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
factory.addStep(ShellCommand(
descriptionDone="install requirements",
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["pip", "install", "-r", "requirements.txt"],
))
factory.addStep(ShellCommand(
descriptionDone="copy settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["cp", "/Users/buildbot/settings_local.py", "./"],
))
factory.addStep(ShellCommand(
descriptionDone="collect static files",
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=False,
flunkOnFailure=False,
command=["ietf/manage.py", "collectstatic", "--noinput", ],
))
factory.addStep(UnitTest(
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--skip-coverage", "--verbosity=2", ],
))
c['builders'].append(BuilderConfig(name="Test Suite (OS X)", factory=factory, category="1. trunk",
slavenames=["datatracker_osx_py27_4"]))
c['builders'].append(BuilderConfig(name="[branch] Test Suite (OS X)", factory=factory, category="2. branch",
slavenames=["datatracker_osx_py27_4"]))
# -*- section Builder_TestCrawler -*- # -*- section Builder_TestCrawler -*-
factory = BuildFactory() factory = util.BuildFactory()
factory.addStep(SVN( factory.addStep(steps.SVN(
username='buildbot@tools.ietf.org', username='buildbot@tools.ietf.org',
descriptionDone="svn update", descriptionDone="svn update",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
)) ))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="install requirements", descriptionDone="install requirements",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["pip", "install", "-r", "requirements.txt"], command=["pip", "install", "-r", "requirements.txt"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="copy settings_local.py", descriptionDone="copy settings_local.py",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["cp", "/home/buildbot/settings_local.py", "./"], command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="run migrations", descriptionDone="run migrations",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["ietf/manage.py", "migrate"], command=["ietf/manage.py", "migrate"],
)) ))
factory.addStep(TestCrawlerShellCommand( factory.addStep(custom_steps.TestCrawlerShellCommand(
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"], command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"],
)) ))
# This should be the last action # This should be the last action
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="mark as passed", descriptionDone="mark as passed",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ], "propset", "--revprop", "-r", util.Property('got_revision'), "test:crawler", "passed" ],
)) ))
c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk", c['builders'].append(util.BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk",
slavenames=["datatracker_lin_py27_6", ])) workernames=["datatracker_lin_py36_6", ]))
# -*- section Builder_Verify_Old_Libs -*- # -*- section Builder_Verify_Minimum_Libs -*-
# This build runs pip install --upgrade, to make sure that we install the earliest version of # This build runs pip install --upgrade, to make sure that we install the earliest version of
# all dependencies, in order to get an indication if/when an incompatibility turns up with a new # all dependencies, in order to get an indication if/when an incompatibility turns up with a new
@ -494,65 +260,65 @@ c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, categor
# change the external test conditions and produce spurious errors because of version changes in # change the external test conditions and produce spurious errors because of version changes in
# dependencies. # dependencies.
factory = BuildFactory() factory = util.BuildFactory()
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="remove tweaked requirements", descriptionDone="remove tweaked requirements",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=False, haltOnFailure=False,
flunkOnFailure=False, flunkOnFailure=False,
command=["rm", "requirements.txt"], command=["rm", "requirements.txt"],
)) ))
factory.addStep(SVN( factory.addStep(steps.SVN(
username='buildbot@tools.ietf.org', username='buildbot@tools.ietf.org',
descriptionDone="svn update", descriptionDone="svn update",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
alwaysUseLatest=True, alwaysUseLatest=True,
haltOnFailure=True, haltOnFailure=True,
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
)) ))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="edit requirements", descriptionDone="edit requirements",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"], command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="install/upgrade requirements", descriptionDone="install/upgrade requirements",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["pip", "install", "--upgrade", "-r", "requirements.txt"], command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="seting up settings_local.py", descriptionDone="seting up settings_local.py",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["cp", "/home/buildbot/settings_local.py", "./"], command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="list installed pyton modules", descriptionDone="list installed pyton modules",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["pip", "freeze"], command=["pip", "freeze"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="collect static files", descriptionDone="collect static files",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=False, haltOnFailure=False,
flunkOnFailure=False, flunkOnFailure=False,
command=["ietf/manage.py", "collectstatic", "--noinput", ], command=["ietf/manage.py", "collectstatic", "--noinput", ],
)) ))
factory.addStep(UnitTest( factory.addStep(custom_steps.DjangoTest(
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
)) ))
c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk", c['builders'].append(util.BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk",
slavenames=["datatracker_lin_py27_5"])) workernames=["datatracker_lin_py36_5", ]))
# -*- section Builder_Dependencies -*- # -*- section Builder_Veryfy_Latest_Libs -*-
# This build runs pip install --upgrade, to make sure that we install the latest version of all # This build runs pip install --upgrade, to make sure that we install the latest version of all
# dependencies, in order to get an indication if/when an incompatibility turns up with a new # dependencies, in order to get an indication if/when an incompatibility turns up with a new
@ -560,134 +326,99 @@ c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory,
# change the external test conditions and produce spurious errors because of version changes in # change the external test conditions and produce spurious errors because of version changes in
# dependencies. # dependencies.
factory = BuildFactory() factory = util.BuildFactory()
factory.addStep(SVN( factory.addStep(steps.SVN(
username='buildbot@tools.ietf.org', username='buildbot@tools.ietf.org',
descriptionDone="svn update", descriptionDone="svn update",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
alwaysUseLatest=True, alwaysUseLatest=True,
haltOnFailure=True, haltOnFailure=True,
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
)) ))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="install/upgrade requirements", descriptionDone="install/upgrade requirements",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["pip", "install", "--upgrade", "-r", "requirements.txt"], command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="seting up settings_local.py", descriptionDone="seting up settings_local.py",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["cp", "/home/buildbot/settings_local.py", "./"], command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="list installed pyton modules", descriptionDone="list installed pyton modules",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["pip", "freeze"], command=["pip", "freeze"],
)) ))
factory.addStep(ShellCommand( factory.addStep(steps.ShellCommand(
descriptionDone="collect static files", descriptionDone="collect static files",
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=False, haltOnFailure=False,
flunkOnFailure=False, flunkOnFailure=False,
command=["ietf/manage.py", "collectstatic", "--noinput", ], command=["ietf/manage.py", "collectstatic", "--noinput", ],
)) ))
factory.addStep(UnitTest( factory.addStep(custom_steps.DjangoTest(
workdir=Interpolate('build/%(src::branch)s'), workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True, haltOnFailure=True,
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
)) ))
c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk", c['builders'].append(util.BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk",
slavenames=["datatracker_lin_py27_5"])) workernames=["datatracker_lin_py36_5", ]))
####### BUILDBOT SERVICES
####### STATUS TARGETS # 'services' is a list of BuildbotService items like reporter targets. The
# -*- section StatusTargets -*- # status of each build will be pushed to these targets. buildbot/reporters/*.py
# has a variety to choose from, like IRC bots.
# 'status' is a list of Status Targets. The results of each build will be c['services'] = []
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.
c['status'] = []
from buildbot.status import html, mail
from buildbot.status.web import authz, auth
authz_cfg=authz.Authz(
# change any of these to True to enable; see the manual for more
# options
auth=auth.BasicAuth([("ietfdb","ietfdb")]),
gracefulShutdown = False,
forceBuild = 'auth', # use this to test your slave once it is set up
forceAllBuilds = False,
pingBuilder = False,
stopBuild = 'auth',
stopAllBuilds = False,
cancelPendingBuild = 'auth',
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
# A second web status with slightly different rendering
from twisted.python import log
def changelinkfilter(html, project):
log.msg(" * changelinkfilter(html='%s', project='%s')" % (html, project))
return html
import jinja2, os
trac_template_loaders = [jinja2.FileSystemLoader(os.path.join(os.getcwd(), 'trac_view'))]
c['status'].append(html.WebStatus(http_port=8011, jinja_loaders=trac_template_loaders,
authz=authz_cfg))
# Email notifications
from zope.interface import implements
from buildbot import interfaces
class UsernameIsEmailAddress():
"This IEmailLookup provider assumes that the svn username is a valid email address."
implements(interfaces.IEmailLookup)
def getAddress(self, name):
return name
c['status'].append(mail.MailNotifier(
fromaddr='buildbot@tools.ietf.org',
sendToInterestedUsers=False,
extraRecipients=['henrik@levkowetz.com',],
mode="problem",
))
c['status'].append(mail.MailNotifier(
fromaddr='buildbot@tools.ietf.org',
lookup=UsernameIsEmailAddress(),
mode="problem",
))
####### PROJECT IDENTITY ####### PROJECT IDENTITY
# -*- section Project -*-
# the 'title' string will appear at the top of this buildbot # the 'title' string will appear at the top of this buildbot installation's
# installation's html.WebStatus home page (linked to the # home pages (linked to the 'titleURL').
# 'titleURL') and is embedded in the title of the waterfall HTML page.
c['title'] = "IETF Datatracker" c['title'] = "Buildbot: IETF Datatracker"
c['titleURL'] = "https://datatracker.ietf.org/" c['titleURL'] = "https://datatracker.ietf.org/"
# the 'buildbotURL' string should point to the location where the buildbot's # the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This # internal web server is visible. This typically uses the port number set in
# typically uses the port number set in the Waterfall 'status' entry, but # the 'www' entry below, but with an externally-visible host name which the
# with an externally-visible host name which the buildbot cannot figure out # buildbot cannot figure out without some help.
# without some help.
c['buildbotURL'] = "http://dunkelfelder.tools.ietf.org:8010/"
# minimalistic config to activate new web UI
c['www'] = {
'port': 8010,
'plugins': {
'waterfall_view': True,
'console_view': True,
'grid_view': True,
},
'default_page': 'waterfall_view',
'debug': True,
'auth': util.UserPasswordAuth({"ietfdb": "ietfdb"}),
}
c['buildbotURL'] = "http://zinfandel.tools.ietf.org:8010/"
####### DB URL ####### DB URL
c['db'] = { c['db'] = {
# This specifies what database buildbot uses to store its state. You can leave # This specifies what database buildbot uses to store its state.
# this at its default for all but the largest installations. # It's easy to start with sqlite, but it's recommended to switch to a dedicated
# database, such as PostgreSQL or MySQL, for use in production environments.
# http://docs.buildbot.net/current/manual/configuration/global.html#database-specification
'db_url' : "sqlite:///state.sqlite", 'db_url' : "sqlite:///state.sqlite",
} }
####### MISC. SETTINGS
c['buildbotNetUsageData'] = 'full'

3
buildbot/restart-masters Executable file
View file

@ -0,0 +1,3 @@
#!/bin/bash
for d in masters/*; do ( cd $d; . env/bin/activate; buildbot restart; ); done

3
buildbot/restart-workers Executable file
View file

@ -0,0 +1,3 @@
#!/bin/bash
for d in workers/*; do ( cd $d; . env/bin/activate; buildbot-worker restart; ); done

180
buildbot/setup-buildbot-workers Executable file
View file

@ -0,0 +1,180 @@
#!/bin/bash
# -*- indent-with-tabs: 0 -*-
version=0.10
program=${0##*/}
progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi
# ----------------------------------------------------------------------
function usage() {
# Possible sections:
# NAME
# SYNOPSIS
# CONFIGURATION [Normally only in Section 4]
# DESCRIPTION
# OPTIONS [Normally only in Sections 1, 8]
# EXIT STATUS [Normally only in Sections 1, 8]
# RETURN VALUE [Normally only in Sections 2, 3]
# ERRORS [Typically only in Sections 2, 3]
# ENVIRONMENT
# FILES
# VERSIONS [Normally only in Sections 2, 3]
# CONFORMING TO
# NOTES
# BUGS
# EXAMPLE
# SEE ALSO
cat <<EOF
NAME
$program - set up buildbot workers
SYNOPSIS
$program [OPTIONS]
DESCRIPTION
Set up environment and buildbot worker files after checking
out the buildbot directory tree from the repository.
EOF
echo -e "OPTIONS"
if [ "$(uname)" = "Linux" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi
cat <<EOF
AUTHOR
Written by Henrik Levkowetz, <henrik@levkowetz.com>. Repository:
https://svn.tools.ietf.org/svn/tools/ietfdb/trunk/buildbot
COPYRIGHT
Copyright 2020 the IETF Trust. All rights reserved.
Redistribution and use in source and binary forms, with or
without modification, are permitted provided that the conditions
laid out in the 3-clause BSD license is followed.
License text: https://opensource.org/licenses/BSD-3-Clause
EOF
}
# ----------------------------------------------------------------------
function die() {
echo -e "\n$program: error: $*" >&2
exit 1
}
function err() {
echo -e "${red}$*${reset}" >&2
}
function note() {
if [ -n "$VERBOSE" ]; then echo -e "\n$*"; fi
}
# ----------------------------------------------------------------------
function version() {
echo -e "$program $version"
}
# ----------------------------------------------------------------------
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
# ----------------------------------------------------------------------
# Option parsing
# Options
shortopts=a:hp:s:vV
longopts=admin:,help,server:,passwd:,python:,verbose,version
# Default values
read -d ' ' <<< $(who -m)
user=$REPLY
name=$(getent passwd $user | cut -d ':' -f 5 | cut -d ',' -f 1)
server='zinfandel.tools.ietf.org'
pass='' # must be set on the command line
python=python3.6
if [ "$(uname)" = "Linux" ]; then
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
files="$@"
eval set -- "$args"
sed="sed -r"
else
# Darwin, BSDs
args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
files="$@"
set -- $args
sed="sed -E"
fi
while true ; do
case "$1" in
-a| --admin) admin="$2"; shift;; # "Name <email>" of buildbot admin
-h| --help) usage; exit;; # Show this help, then exit
-p| --passwd) pass=$2; shift;; # Worker password
--python) python=$2; shift;; # Python version to use (e.g., 'python3.6')
-s| --server) server=$2; shift;; # Set the server fqdn
-v| --verbose) VERBOSE=1;; # Be more talkative
-V| --version) version; exit;; # Show program version, then exit
--) shift; break;;
*) die "Internal error, inconsistent option specification: '$1'";;
esac
shift
done
# ----------------------------------------------------------------------
# The program itself
dir=$(dirname $(realpath $0))
if [ -d "$dir/slaves" ]; then
path="$dir/slaves"
else
path="$dir/workers"
fi
for worker in $path/*; do
(
cd $worker;
pwd
if [ ! -d ./env ]; then
echo "Setting up virtual environment"
# Change python version to match deployment version
python3.6 -m venv env
fi
. env/bin/activate
pip install buildbot-worker
if [ ! -f ./buildbot.tac ]; then
pwfile=$dir/${worker##*/}_pw
echo "Looking for pwfile: $pwfile"
[ -f "$pwfile" ] && pass=$(< $pwfile)
[ -z "$pass" ] && die "Missing parameter: worker password"
buildbot-worker create-worker $PWD $server ${PWD##*/} $pass
fi
if ! grep -q "$name" ./info/admin; then
read -p "Expected $name in $PWD/info/admin, but found $(<./info/admin) -- change it [Y/n]?"
if [ "$REPLY" = "Y" -o "$REPLY" = "y" ]; then
if [ -z "$admin" ]; then
read -p "Admin (Your Name <your@email.example>): "
admin=$REPLY
fi
echo "Setting up ./info/admin"
echo "$admin" > ./info/admin
echo "Setting up ./info/host"
echo "$(uname -s -n -r) $(python --version)" > ./info/host
fi
fi
buildbot-worker stop
buildbot-worker start
)
done

1
buildbot/workers/datatracker_lin_py36_1/info/admin Normal file
View file

@ -0,0 +1 @@
Henrik Levkowetz <henrik@levkowetz.com>

1
buildbot/workers/datatracker_lin_py36_1/info/host Normal file
View file

@ -0,0 +1 @@
Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9

1
buildbot/workers/datatracker_lin_py36_2/info/admin Normal file
View file

@ -0,0 +1 @@
Henrik Levkowetz <henrik@levkowetz.com>

1
buildbot/workers/datatracker_lin_py36_2/info/host Normal file
View file

@ -0,0 +1 @@
Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9

1
buildbot/workers/datatracker_lin_py36_3/info/admin Normal file
View file

@ -0,0 +1 @@
Henrik Levkowetz <henrik@levkowetz.com>

1
buildbot/workers/datatracker_lin_py36_3/info/host Normal file
View file

@ -0,0 +1 @@
Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9

1
buildbot/workers/datatracker_lin_py36_4/info/admin Normal file
View file

@ -0,0 +1 @@
Henrik Levkowetz <henrik@levkowetz.com>

1
buildbot/workers/datatracker_lin_py36_4/info/host Normal file
View file

@ -0,0 +1 @@
Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9

1
buildbot/workers/datatracker_lin_py36_5/info/admin Normal file
View file

@ -0,0 +1 @@
Henrik Levkowetz <henrik@levkowetz.com>

1
buildbot/workers/datatracker_lin_py36_5/info/host Normal file
View file

@ -0,0 +1 @@
Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9

1
buildbot/workers/datatracker_lin_py36_6/info/admin Normal file
View file

@ -0,0 +1 @@
Henrik Levkowetz <henrik@levkowetz.com>

1
buildbot/workers/datatracker_lin_py36_6/info/host Normal file
View file

@ -0,0 +1 @@
Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9