altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Added a buildbot master config file for buildbot 2.7, and a passwords template.

Browse source 
- Legacy-Id: 17658
This commit is contained in:
buildbot 2020-04-20 11:53:58 +00:00
parent cdaa2bc437
commit 4f9a112645
3 changed files with 455 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
buildbot/masters/datatracker/buildbot_passwords.py.template Normal file
View file

@ -0,0 +1,21 @@
# Copyright The IETF Trust 2015-2020, All Rights Reserved
# -*- coding: utf-8 -*-
# Uncomment and set passwords below to match those set for the workers, then
# save as "buildbot_passwords.py"
# datatracker_lin_py27_1_pw = "<password>"
# datatracker_lin_py27_2_pw = "<password>"
# datatracker_lin_py27_3_pw = "<password>"
# datatracker_osx_py27_4_pw = "<password>"
# datatracker_lin_py27_5_pw = "<password>"
# datatracker_lin_py27_6_pw = "<password>"
# ietfdb_svn_hook_pw = "<password>"
#
# datatracker_lin_py36_1_pw = "<password>"
# datatracker_lin_py36_2_pw = "<password>"
# datatracker_lin_py36_3_pw = "<password>"
# datatracker_lin_py36_4_pw = "<password>"
# datatracker_lin_py36_5_pw = "<password>"
# datatracker_lin_py36_6_pw = "<password>"

4
buildbot/masters/datatracker/custom_steps.py
View file

@ -66,6 +66,7 @@ class TestCrawlerShellCommand(steps.WarningCountingShellCommand):
wregex[key] = pattern
# Count matches to the various warning patterns
last_line = None
for line in log.getText().split("\n"):
for key in wregex:
match = re.search(wregex[key], line)
@ -83,6 +84,7 @@ class TestCrawlerShellCommand(steps.WarningCountingShellCommand):
self.step_status.setStatistic(key, len(warnings[key]))
self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand")
if last_line:
match = re.search(self.logline, last_line)
for key in ['elapsed', 'pages']:
info = match.group(key)
@ -113,7 +115,7 @@ class DjangoTest(steps.WarningCountingShellCommand):
warnOnFailure = 1
description = ["testing"]
descriptionDone = ["test"]
command = ["python", "-m", "unittest", "discover"]
command = ["manage.py", "test", ]
regexPatterns = {
"tests": "Ran (\d+) tests in [0-9.]+s",

426
buildbot/masters/datatracker/master.buildbot27.cfg Normal file
View file

@ -0,0 +1,426 @@
# -*- python -*-
# ex: set filetype=python:
from os.path import expanduser as expandtilde
from buildbot.plugins import worker, changes, schedulers, util, steps
import buildbot_passwords
import custom_steps
# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
####### SETTINGS
# For miscellaneous settings, see MISC. SETTINGS at the bottom of the file
####### WORKERS
# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password. The same
# worker name and password must be configured on the worker.
c['workers'] = [
worker.Worker("datatracker_lin_py36_1", buildbot_passwords.datatracker_lin_py36_1_pw),
worker.Worker("datatracker_lin_py36_2", buildbot_passwords.datatracker_lin_py36_2_pw),
worker.Worker("datatracker_lin_py36_3", buildbot_passwords.datatracker_lin_py36_3_pw),
# worker.Worker("datatracker_lin_py36_4", buildbot_passwords.datatracker_lin_py36_4_pw),
worker.Worker("datatracker_lin_py36_5", buildbot_passwords.datatracker_lin_py36_5_pw),
worker.Worker("datatracker_lin_py36_6", buildbot_passwords.datatracker_lin_py36_6_pw),
]
# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': 9989}}
####### CHANGESOURCES
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot version of a python hello-world project.
c['change_source'] = [
changes.PBChangeSource(user='ietfdb', passwd=buildbot_passwords.ietfdb_svn_hook_pw),
]
####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'runtests' build
c['schedulers'] = [
# Branch schedulers
schedulers.SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"],
change_filter=util.ChangeFilter(branch='trunk')),
schedulers.SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"],
change_filter=util.ChangeFilter(branch='trunk')),
# schedulers.SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"],
# change_filter=util.ChangeFilter(branch='trunk')),
#
schedulers.AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"],
change_filter=util.ChangeFilter(branch_re='branch/.*')),
# schedulers.AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"],
# change_filter=util.ChangeFilter(branch_re='branch/.*')),
# schedulers.AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"],
# change_filter=util.ChangeFilter(branch_re='branch/.*')),
#
schedulers.AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"],
change_filter=util.ChangeFilter(branch_re='personal/.*')),
schedulers.AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"],
change_filter=util.ChangeFilter(branch_re='personal/.*')),
# Periodic Schedulers
schedulers.Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],),
schedulers.Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],),
schedulers.Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
# schedulers.Force schedulers
schedulers.ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]),
schedulers.ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]),
# schedulers.ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]),
schedulers.ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]),
#
schedulers.ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]),
schedulers.ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]),
# schedulers.ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]),
#
schedulers.ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]),
schedulers.ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]),
]
####### BUILDERS
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them. Note that any particular build will
# only take place on one worker.
#### Custom subclassed builder
factory = util.BuildFactory()
# check out the source
factory.addStep(steps.Git(repourl='git://github.com/buildbot/hello-world.git', mode='incremental'))
# run the tests (note that this will require that 'trial' is installed)
factory.addStep(steps.ShellCommand(command=["trial", "hello"],
env={"PYTHONPATH": "."}))
c['builders'] = []
# -*- section Builder_Run_pyflakes -*-
factory = util.BuildFactory()
factory.addStep(steps.SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(steps.ShellCommand(
descriptionDone="seting up settings_local.py",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
))
factory.addStep(steps.PyFlakes(
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["ietf/manage.py", "pyflakes", "--verbosity=0"],
))
# This should be the last action
factory.addStep(steps.ShellCommand(
descriptionDone="mark as passed",
workdir=util.Interpolate('build/%(src::branch)s'),
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", util.Property('got_revision'), "test:pyflakes", "passed" ],
))
c['builders'].append(util.BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk",
workernames=["datatracker_lin_py36_1", ]))
c['builders'].append(util.BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch",
workernames=["datatracker_lin_py36_2", ]))
c['builders'].append(util.BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal",
workernames=["datatracker_lin_py36_3", ]))
# -*- section Builder_TestSuite -*-
factory = util.BuildFactory()
factory.addStep(steps.SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
factory.addStep(steps.ShellCommand(
descriptionDone="remove tmp-* dirs",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["rm", "-rf", "tmp-*/"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="install requirements",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["pip", "install", "-r", "requirements.txt"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="copy settings_local.py",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="collect static files",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=False,
flunkOnFailure=False,
command=["ietf/manage.py", "collectstatic", "--noinput", ],
))
factory.addStep(custom_steps.DjangoTest(
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
want_stderr=True,
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
))
# This should come after tests
factory.addStep(steps.ShellCommand(
descriptionDone="mark as passed",
workdir=util.Interpolate('build/%(src::branch)s'),
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", util.Property('got_revision'), "test:unittest", "passed" ],
))
c['builders'].append(util.BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
workernames=["datatracker_lin_py36_1", ]))
c['builders'].append(util.BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
workernames=["datatracker_lin_py36_2", ]))
c['builders'].append(util.BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
workernames=["datatracker_lin_py36_3", ]))
# -*- section Builder_TestCrawler -*-
factory = util.BuildFactory()
factory.addStep(steps.SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
factory.addStep(steps.ShellCommand(
descriptionDone="install requirements",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["pip", "install", "-r", "requirements.txt"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="copy settings_local.py",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="run migrations",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["ietf/manage.py", "migrate"],
))
factory.addStep(custom_steps.TestCrawlerShellCommand(
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
want_stderr=True,
command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"],
))
# This should be the last action
factory.addStep(steps.ShellCommand(
descriptionDone="mark as passed",
workdir=util.Interpolate('build/%(src::branch)s'),
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", util.Property('got_revision'), "test:crawler", "passed" ],
))
c['builders'].append(util.BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk",
workernames=["datatracker_lin_py36_6", ]))
# -*- section Builder_Verify_Minimum_Libs -*-
# This build runs pip install --upgrade, to make sure that we install the earliest version of
# all dependencies, in order to get an indication if/when an incompatibility turns up with a new
# version of a dependency. The other test suite builders *don't* use --upgrade, in order to not
# change the external test conditions and produce spurious errors because of version changes in
# dependencies.
factory = util.BuildFactory()
factory.addStep(steps.ShellCommand(
descriptionDone="remove tweaked requirements",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=False,
flunkOnFailure=False,
command=["rm", "requirements.txt"],
))
factory.addStep(steps.SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
workdir=util.Interpolate('build/%(src::branch)s'),
alwaysUseLatest=True,
haltOnFailure=True,
repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
factory.addStep(steps.ShellCommand(
descriptionDone="edit requirements",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="install/upgrade requirements",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="seting up settings_local.py",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="list installed pyton modules",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["pip", "freeze"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="collect static files",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=False,
flunkOnFailure=False,
command=["ietf/manage.py", "collectstatic", "--noinput", ],
))
factory.addStep(custom_steps.DjangoTest(
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
))
c['builders'].append(util.BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk",
workernames=["datatracker_lin_py36_5", ]))
# -*- section Builder_Veryfy_Latest_Libs -*-
# This build runs pip install --upgrade, to make sure that we install the latest version of all
# dependencies, in order to get an indication if/when an incompatibility turns up with a new
# version of a dependency. The other test suite builders *don't* use --upgrade, in order to not
# change the external test conditions and produce spurious errors because of version changes in
# dependencies.
factory = util.BuildFactory()
factory.addStep(steps.SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
workdir=util.Interpolate('build/%(src::branch)s'),
alwaysUseLatest=True,
haltOnFailure=True,
repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
factory.addStep(steps.ShellCommand(
descriptionDone="install/upgrade requirements",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="seting up settings_local.py",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="list installed pyton modules",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["pip", "freeze"],
))
factory.addStep(steps.ShellCommand(
descriptionDone="collect static files",
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=False,
flunkOnFailure=False,
command=["ietf/manage.py", "collectstatic", "--noinput", ],
))
factory.addStep(custom_steps.DjangoTest(
workdir=util.Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
))
c['builders'].append(util.BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk",
workernames=["datatracker_lin_py36_5", ]))
####### BUILDBOT SERVICES
# 'services' is a list of BuildbotService items like reporter targets. The
# status of each build will be pushed to these targets. buildbot/reporters/*.py
# has a variety to choose from, like IRC bots.
c['services'] = []
####### PROJECT IDENTITY
# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').
c['title'] = "Buildbot: IETF Datatracker"
c['titleURL'] = "https://datatracker.ietf.org/"
# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.
c['buildbotURL'] = "http://dunkelfelder.tools.ietf.org:8010/"
# minimalistic config to activate new web UI
c['www'] = {
'port': 8010,
'plugins': {
'waterfall_view': True,
'console_view': True,
'grid_view': True,
},
'default_page': 'waterfall_view',
'debug': True,
'auth': util.UserPasswordAuth({"ietfdb": "ietfdb"}),
}
####### DB URL
c['db'] = {
# This specifies what database buildbot uses to store its state.
# It's easy to start with sqlite, but it's recommended to switch to a dedicated
# database, such as PostgreSQL or MySQL, for use in production environments.
# http://docs.buildbot.net/current/manual/configuration/global.html#database-specification
'db_url' : "sqlite:///state.sqlite",
}
####### MISC. SETTINGS
c['buildbotNetUsageData'] = 'full'