altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Updated the buildbot config with additional builders, database loading as part of test-crawler run, tweaked test-crawler start-time and other tweaks

Browse source 
- Legacy-Id: 17765
This commit is contained in:
buildbot 2020-05-09 05:27:58 +00:00
parent 695b6e0e86
commit 73144a6782
1 changed files with 56 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
buildbot/masters/datatracker/master.cfg
View file

@ -20,19 +20,16 @@ c = BuildmasterConfig = {}
# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave
c['slaves'] = [
BuildSlave("datatracker_lin_py27_1", datatracker_lin_py27_1_pw),
BuildSlave("datatracker_lin_py27_2", datatracker_lin_py27_2_pw),
BuildSlave("datatracker_lin_py27_3", datatracker_lin_py27_3_pw),
BuildSlave("datatracker_osx_py27_4", datatracker_osx_py27_4_pw),
BuildSlave("datatracker_lin_py27_5", datatracker_lin_py27_5_pw),
BuildSlave("datatracker_lin_py27_6", datatracker_lin_py27_6_pw),
#
BuildSlave("datatracker_lin_py36_1", datatracker_lin_py36_1_pw),
BuildSlave("datatracker_lin_py36_2", datatracker_lin_py36_2_pw),
BuildSlave("datatracker_lin_py36_3", datatracker_lin_py36_3_pw),
BuildSlave("datatracker_lin_py36_4", datatracker_lin_py36_4_pw),
BuildSlave("datatracker_lin_py36_5", datatracker_lin_py36_5_pw),
BuildSlave("datatracker_lin_py36_6", datatracker_lin_py36_6_pw),
BuildSlave("dunkelfelder_lin_py36_1", dunkelfelder_lin_py36_1_pw),
BuildSlave("dunkelfelder_lin_py36_2", dunkelfelder_lin_py36_2_pw),
BuildSlave("dunkelfelder_lin_py36_3", dunkelfelder_lin_py36_3_pw),
BuildSlave("dunkelfelder_lin_py36_4", dunkelfelder_lin_py36_4_pw),
BuildSlave("dornfelder_lin_py36_1", dornfelder_lin_py36_1_pw),
BuildSlave("dornfelder_lin_py36_2", dornfelder_lin_py36_2_pw),
BuildSlave("dornfelder_lin_py36_3", dornfelder_lin_py36_3_pw),
BuildSlave("dornfelder_lin_py36_4", dornfelder_lin_py36_4_pw),
]
# 'protocols' contains information about protocols which master will use for
@ -93,7 +90,7 @@ c['schedulers'] = [
# Periodic Schedulers
Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],),
Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],),
Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
Nightly(name="crawler", hour=23, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
# Force schedulers
ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]),
@ -311,6 +308,7 @@ c['builders'] = []
# -*- section Builder_Run_pyflakes -*-
factory = BuildFactory()
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
@ -320,7 +318,13 @@ factory.addStep(SVN(
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="install requirements",
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
usePTY=False,
command=["pip", "install", "-r", "requirements.txt"],
))
factory.addStep(ShellCommand(
descriptionDone="seting up settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
@ -338,21 +342,23 @@ factory.addStep(PyFlakes(
factory.addStep(ShellCommand(
descriptionDone="mark as passed",
workdir=Interpolate('build/%(src::branch)s'),
flunkOnFailure=False,
usePTY=False,
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ],
))
c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk",
slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ]))
slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch",
slavenames=["datatracker_lin_py36_2", ]))
slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal",
slavenames=["datatracker_lin_py36_3", ]))
slavenames=["dunkelfelder_lin_py36_2",]))
# -*- section Builder_TestSuite -*-
factory = BuildFactory()
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
@ -362,7 +368,7 @@ factory.addStep(SVN(
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
descriptionDone="remove tmp-* dirs",
workdir=Interpolate('build/%(src::branch)s'),
@ -377,7 +383,6 @@ factory.addStep(ShellCommand(
usePTY=False,
command=["pip", "install", "-r", "requirements.txt"],
))
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="copy settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
@ -403,22 +408,32 @@ factory.addStep(UnitTest(
factory.addStep(ShellCommand(
descriptionDone="mark as passed",
workdir=Interpolate('build/%(src::branch)s'),
flunkOnFailure=False,
usePTY=False,
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ],
))
c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ]))
slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
slavenames=["datatracker_lin_py36_2", ]))
slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
slavenames=["datatracker_lin_py36_3", ]))
slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
# -*- section Builder_TestCrawler -*-
factory = BuildFactory()
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="update database",
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
usePTY=False,
timeout=3600, # 1 hour
command=["docker/updatedb", "-q"],
))
factory.addStep(SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
@ -428,7 +443,7 @@ factory.addStep(SVN(
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
descriptionDone="install requirements",
workdir=Interpolate('build/%(src::branch)s'),
@ -436,7 +451,6 @@ factory.addStep(ShellCommand(
usePTY=False,
command=["pip", "install", "-r", "requirements.txt"],
))
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="copy settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
@ -451,6 +465,14 @@ factory.addStep(ShellCommand(
usePTY=False,
command=["ietf/manage.py", "migrate"],
))
# This will not only do a prelimnary sanity check, but also patch libs as needed:
factory.addStep(ShellCommand(
descriptionDone="run django checks",
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
usePTY=False,
command=["ietf/manage.py", "check"],
))
factory.addStep(TestCrawlerShellCommand(
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
@ -461,13 +483,14 @@ factory.addStep(TestCrawlerShellCommand(
factory.addStep(ShellCommand(
descriptionDone="mark as passed",
workdir=Interpolate('build/%(src::branch)s'),
flunkOnFailure=False,
usePTY=False,
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ],
))
c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk",
slavenames=["datatracker_lin_py36_6", ]))
slavenames=["dunkelfelder_lin_py36_4", ]))
# -*- section Builder_Verify_Old_Libs -*-
@ -479,6 +502,7 @@ c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, categor
# dependencies.
factory = BuildFactory()
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="remove tweaked requirements",
workdir=Interpolate('build/%(src::branch)s'),
@ -497,7 +521,7 @@ factory.addStep(SVN(
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
descriptionDone="edit requirements",
workdir=Interpolate('build/%(src::branch)s'),
@ -512,7 +536,6 @@ factory.addStep(ShellCommand(
usePTY=False,
command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
))
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="seting up settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
@ -542,10 +565,10 @@ factory.addStep(UnitTest(
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
))
c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk",
slavenames=["datatracker_lin_py36_5", ]))
slavenames=["dornfelder_lin_py36_3", ]))
# -*- section Builder_Dependencies -*-
# -*- section Verify_Latest_Libs -*-
# This build runs pip install --upgrade, to make sure that we install the latest version of all
# dependencies, in order to get an indication if/when an incompatibility turns up with a new
@ -554,6 +577,7 @@ c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory,
# dependencies.
factory = BuildFactory()
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
@ -564,7 +588,7 @@ factory.addStep(SVN(
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
descriptionDone="install/upgrade requirements",
workdir=Interpolate('build/%(src::branch)s'),
@ -572,7 +596,6 @@ factory.addStep(ShellCommand(
usePTY=False,
command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
))
factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="seting up settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
@ -603,7 +626,7 @@ factory.addStep(UnitTest(
))
c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk",
slavenames=["datatracker_lin_py36_5", ]))
slavenames=["dornfelder_lin_py36_3", ]))
####### STATUS TARGETS
@ -673,7 +696,7 @@ c['status'].append(mail.MailNotifier(
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.
c['title'] = "IETF Datatracker"
c['title'] = "Buildbot: IETF Datatracker"
c['titleURL'] = "https://datatracker.ietf.org/"
# the 'buildbotURL' string should point to the location where the buildbot's