From 73144a6782403b7e629e91e1cb6572bae7797951 Mon Sep 17 00:00:00 2001
From: buildbot <buildbot@tools.ietf.org>
Date: Sat, 9 May 2020 05:27:58 +0000
Subject: [PATCH] Updated the buildbot config with additional builders,
database loading as part of test-crawler run, tweaked test-crawler start-time
and other tweaks - Legacy-Id: 17765
---
buildbot/masters/datatracker/master.cfg | 89 ++++++++++++++++---------
1 file changed, 56 insertions(+), 33 deletions(-)
diff --git a/buildbot/masters/datatracker/master.cfg b/buildbot/masters/datatracker/master.cfg
index dace7868d..ff5e5489e 100644
--- a/buildbot/masters/datatracker/master.cfg
+++ b/buildbot/masters/datatracker/master.cfg
@@ -20,19 +20,16 @@ c = BuildmasterConfig = {}
# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave
c['slaves'] = [
- BuildSlave("datatracker_lin_py27_1", datatracker_lin_py27_1_pw),
- BuildSlave("datatracker_lin_py27_2", datatracker_lin_py27_2_pw),
- BuildSlave("datatracker_lin_py27_3", datatracker_lin_py27_3_pw),
- BuildSlave("datatracker_osx_py27_4", datatracker_osx_py27_4_pw),
- BuildSlave("datatracker_lin_py27_5", datatracker_lin_py27_5_pw),
- BuildSlave("datatracker_lin_py27_6", datatracker_lin_py27_6_pw),
#
- BuildSlave("datatracker_lin_py36_1", datatracker_lin_py36_1_pw),
- BuildSlave("datatracker_lin_py36_2", datatracker_lin_py36_2_pw),
- BuildSlave("datatracker_lin_py36_3", datatracker_lin_py36_3_pw),
- BuildSlave("datatracker_lin_py36_4", datatracker_lin_py36_4_pw),
- BuildSlave("datatracker_lin_py36_5", datatracker_lin_py36_5_pw),
- BuildSlave("datatracker_lin_py36_6", datatracker_lin_py36_6_pw),
+ BuildSlave("dunkelfelder_lin_py36_1", dunkelfelder_lin_py36_1_pw),
+ BuildSlave("dunkelfelder_lin_py36_2", dunkelfelder_lin_py36_2_pw),
+ BuildSlave("dunkelfelder_lin_py36_3", dunkelfelder_lin_py36_3_pw),
+ BuildSlave("dunkelfelder_lin_py36_4", dunkelfelder_lin_py36_4_pw),
+
+ BuildSlave("dornfelder_lin_py36_1", dornfelder_lin_py36_1_pw),
+ BuildSlave("dornfelder_lin_py36_2", dornfelder_lin_py36_2_pw),
+ BuildSlave("dornfelder_lin_py36_3", dornfelder_lin_py36_3_pw),
+ BuildSlave("dornfelder_lin_py36_4", dornfelder_lin_py36_4_pw),
]
# 'protocols' contains information about protocols which master will use for
@@ -93,7 +90,7 @@ c['schedulers'] = [
# Periodic Schedulers
Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],),
Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],),
- Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
+ Nightly(name="crawler", hour=23, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
# Force schedulers
ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]),
@@ -311,6 +308,7 @@ c['builders'] = []
# -*- section Builder_Run_pyflakes -*-
factory = BuildFactory()
+factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
@@ -320,7 +318,13 @@ factory.addStep(SVN(
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
-factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
+factory.addStep(ShellCommand(
+ descriptionDone="install requirements",
+ workdir=Interpolate('build/%(src::branch)s'),
+ haltOnFailure=True,
+ usePTY=False,
+ command=["pip", "install", "-r", "requirements.txt"],
+ ))
factory.addStep(ShellCommand(
descriptionDone="seting up settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
@@ -338,21 +342,23 @@ factory.addStep(PyFlakes(
factory.addStep(ShellCommand(
descriptionDone="mark as passed",
workdir=Interpolate('build/%(src::branch)s'),
+ flunkOnFailure=False,
usePTY=False,
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ],
))
c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk",
- slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ]))
+ slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch",
- slavenames=["datatracker_lin_py36_2", ]))
+ slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal",
- slavenames=["datatracker_lin_py36_3", ]))
+ slavenames=["dunkelfelder_lin_py36_2",]))
# -*- section Builder_TestSuite -*-
factory = BuildFactory()
+factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
@@ -362,7 +368,7 @@ factory.addStep(SVN(
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
-factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
+factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
descriptionDone="remove tmp-* dirs",
workdir=Interpolate('build/%(src::branch)s'),
@@ -377,7 +383,6 @@ factory.addStep(ShellCommand(
usePTY=False,
command=["pip", "install", "-r", "requirements.txt"],
))
-factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="copy settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
@@ -403,22 +408,32 @@ factory.addStep(UnitTest(
factory.addStep(ShellCommand(
descriptionDone="mark as passed",
workdir=Interpolate('build/%(src::branch)s'),
+ flunkOnFailure=False,
usePTY=False,
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ],
))
c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
- slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ]))
+ slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ]))
c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
- slavenames=["datatracker_lin_py36_2", ]))
+ slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
- slavenames=["datatracker_lin_py36_3", ]))
+ slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ]))
# -*- section Builder_TestCrawler -*-
factory = BuildFactory()
+factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
+factory.addStep(ShellCommand(
+ descriptionDone="update database",
+ workdir=Interpolate('build/%(src::branch)s'),
+ haltOnFailure=True,
+ usePTY=False,
+ timeout=3600, # 1 hour
+ command=["docker/updatedb", "-q"],
+ ))
factory.addStep(SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
@@ -428,7 +443,7 @@ factory.addStep(SVN(
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
-factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
+factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
descriptionDone="install requirements",
workdir=Interpolate('build/%(src::branch)s'),
@@ -436,7 +451,6 @@ factory.addStep(ShellCommand(
usePTY=False,
command=["pip", "install", "-r", "requirements.txt"],
))
-factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="copy settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
@@ -451,6 +465,14 @@ factory.addStep(ShellCommand(
usePTY=False,
command=["ietf/manage.py", "migrate"],
))
+# This will not only do a prelimnary sanity check, but also patch libs as needed:
+factory.addStep(ShellCommand(
+ descriptionDone="run django checks",
+ workdir=Interpolate('build/%(src::branch)s'),
+ haltOnFailure=True,
+ usePTY=False,
+ command=["ietf/manage.py", "check"],
+ ))
factory.addStep(TestCrawlerShellCommand(
workdir=Interpolate('build/%(src::branch)s'),
haltOnFailure=True,
@@ -461,13 +483,14 @@ factory.addStep(TestCrawlerShellCommand(
factory.addStep(ShellCommand(
descriptionDone="mark as passed",
workdir=Interpolate('build/%(src::branch)s'),
+ flunkOnFailure=False,
usePTY=False,
command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
"propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ],
))
c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk",
- slavenames=["datatracker_lin_py36_6", ]))
+ slavenames=["dunkelfelder_lin_py36_4", ]))
# -*- section Builder_Verify_Old_Libs -*-
@@ -479,6 +502,7 @@ c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, categor
# dependencies.
factory = BuildFactory()
+factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="remove tweaked requirements",
workdir=Interpolate('build/%(src::branch)s'),
@@ -497,7 +521,7 @@ factory.addStep(SVN(
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
-factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
+factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
descriptionDone="edit requirements",
workdir=Interpolate('build/%(src::branch)s'),
@@ -512,7 +536,6 @@ factory.addStep(ShellCommand(
usePTY=False,
command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
))
-factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="seting up settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
@@ -542,10 +565,10 @@ factory.addStep(UnitTest(
command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
))
c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk",
- slavenames=["datatracker_lin_py36_5", ]))
+ slavenames=["dornfelder_lin_py36_3", ]))
-# -*- section Builder_Dependencies -*-
+# -*- section Verify_Latest_Libs -*-
# This build runs pip install --upgrade, to make sure that we install the latest version of all
# dependencies, in order to get an indication if/when an incompatibility turns up with a new
@@ -554,6 +577,7 @@ c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory,
# dependencies.
factory = BuildFactory()
+factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(SVN(
username='buildbot@tools.ietf.org',
descriptionDone="svn update",
@@ -564,7 +588,7 @@ factory.addStep(SVN(
repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')],
))
-factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s')))
+factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False))
factory.addStep(ShellCommand(
descriptionDone="install/upgrade requirements",
workdir=Interpolate('build/%(src::branch)s'),
@@ -572,7 +596,6 @@ factory.addStep(ShellCommand(
usePTY=False,
command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
))
-factory.addStep(SetPropertiesFromEnv(variables=['HOME',]))
factory.addStep(ShellCommand(
descriptionDone="seting up settings_local.py",
workdir=Interpolate('build/%(src::branch)s'),
@@ -603,7 +626,7 @@ factory.addStep(UnitTest(
))
c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk",
- slavenames=["datatracker_lin_py36_5", ]))
+ slavenames=["dornfelder_lin_py36_3", ]))
####### STATUS TARGETS
@@ -673,7 +696,7 @@ c['status'].append(mail.MailNotifier(
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.
-c['title'] = "IETF Datatracker"
+c['title'] = "Buildbot: IETF Datatracker"
c['titleURL'] = "https://datatracker.ietf.org/"
# the 'buildbotURL' string should point to the location where the buildbot's