diff --git a/buildbot/masters/datatracker/master.cfg b/buildbot/masters/datatracker/master.cfg index dace7868d..ff5e5489e 100644 --- a/buildbot/masters/datatracker/master.cfg +++ b/buildbot/masters/datatracker/master.cfg @@ -20,19 +20,16 @@ c = BuildmasterConfig = {} # slave name and password must be configured on the slave. from buildbot.buildslave import BuildSlave c['slaves'] = [ - BuildSlave("datatracker_lin_py27_1", datatracker_lin_py27_1_pw), - BuildSlave("datatracker_lin_py27_2", datatracker_lin_py27_2_pw), - BuildSlave("datatracker_lin_py27_3", datatracker_lin_py27_3_pw), - BuildSlave("datatracker_osx_py27_4", datatracker_osx_py27_4_pw), - BuildSlave("datatracker_lin_py27_5", datatracker_lin_py27_5_pw), - BuildSlave("datatracker_lin_py27_6", datatracker_lin_py27_6_pw), # - BuildSlave("datatracker_lin_py36_1", datatracker_lin_py36_1_pw), - BuildSlave("datatracker_lin_py36_2", datatracker_lin_py36_2_pw), - BuildSlave("datatracker_lin_py36_3", datatracker_lin_py36_3_pw), - BuildSlave("datatracker_lin_py36_4", datatracker_lin_py36_4_pw), - BuildSlave("datatracker_lin_py36_5", datatracker_lin_py36_5_pw), - BuildSlave("datatracker_lin_py36_6", datatracker_lin_py36_6_pw), + BuildSlave("dunkelfelder_lin_py36_1", dunkelfelder_lin_py36_1_pw), + BuildSlave("dunkelfelder_lin_py36_2", dunkelfelder_lin_py36_2_pw), + BuildSlave("dunkelfelder_lin_py36_3", dunkelfelder_lin_py36_3_pw), + BuildSlave("dunkelfelder_lin_py36_4", dunkelfelder_lin_py36_4_pw), + + BuildSlave("dornfelder_lin_py36_1", dornfelder_lin_py36_1_pw), + BuildSlave("dornfelder_lin_py36_2", dornfelder_lin_py36_2_pw), + BuildSlave("dornfelder_lin_py36_3", dornfelder_lin_py36_3_pw), + BuildSlave("dornfelder_lin_py36_4", dornfelder_lin_py36_4_pw), ] # 'protocols' contains information about protocols which master will use for @@ -93,7 +90,7 @@ c['schedulers'] = [ # Periodic Schedulers Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],), Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],), - Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],), + Nightly(name="crawler", hour=23, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],), # Force schedulers ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]), @@ -311,6 +308,7 @@ c['builders'] = [] # -*- section Builder_Run_pyflakes -*- factory = BuildFactory() +factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) factory.addStep(SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", @@ -320,7 +318,13 @@ factory.addStep(SVN( repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) +factory.addStep(ShellCommand( + descriptionDone="install requirements", + workdir=Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + usePTY=False, + command=["pip", "install", "-r", "requirements.txt"], + )) factory.addStep(ShellCommand( descriptionDone="seting up settings_local.py", workdir=Interpolate('build/%(src::branch)s'), @@ -338,21 +342,23 @@ factory.addStep(PyFlakes( factory.addStep(ShellCommand( descriptionDone="mark as passed", workdir=Interpolate('build/%(src::branch)s'), + flunkOnFailure=False, usePTY=False, command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", "propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ], )) c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk", - slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ])) + slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ])) c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch", - slavenames=["datatracker_lin_py36_2", ])) + slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ])) c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal", - slavenames=["datatracker_lin_py36_3", ])) + slavenames=["dunkelfelder_lin_py36_2",])) # -*- section Builder_TestSuite -*- factory = BuildFactory() +factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) factory.addStep(SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", @@ -362,7 +368,7 @@ factory.addStep(SVN( repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) +factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False)) factory.addStep(ShellCommand( descriptionDone="remove tmp-* dirs", workdir=Interpolate('build/%(src::branch)s'), @@ -377,7 +383,6 @@ factory.addStep(ShellCommand( usePTY=False, command=["pip", "install", "-r", "requirements.txt"], )) -factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) factory.addStep(ShellCommand( descriptionDone="copy settings_local.py", workdir=Interpolate('build/%(src::branch)s'), @@ -403,22 +408,32 @@ factory.addStep(UnitTest( factory.addStep(ShellCommand( descriptionDone="mark as passed", workdir=Interpolate('build/%(src::branch)s'), + flunkOnFailure=False, usePTY=False, command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", "propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ], )) c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk", - slavenames=["datatracker_lin_py36_1", "datatracker_lin_py36_4", ])) + slavenames=["dunkelfelder_lin_py36_1", "dornfelder_lin_py36_1", ])) c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch", - slavenames=["datatracker_lin_py36_2", ])) + slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ])) c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal", - slavenames=["datatracker_lin_py36_3", ])) + slavenames=["dunkelfelder_lin_py36_2", "dornfelder_lin_py36_2", ])) # -*- section Builder_TestCrawler -*- factory = BuildFactory() +factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) +factory.addStep(ShellCommand( + descriptionDone="update database", + workdir=Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + usePTY=False, + timeout=3600, # 1 hour + command=["docker/updatedb", "-q"], + )) factory.addStep(SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", @@ -428,7 +443,7 @@ factory.addStep(SVN( repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) +factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False)) factory.addStep(ShellCommand( descriptionDone="install requirements", workdir=Interpolate('build/%(src::branch)s'), @@ -436,7 +451,6 @@ factory.addStep(ShellCommand( usePTY=False, command=["pip", "install", "-r", "requirements.txt"], )) -factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) factory.addStep(ShellCommand( descriptionDone="copy settings_local.py", workdir=Interpolate('build/%(src::branch)s'), @@ -451,6 +465,14 @@ factory.addStep(ShellCommand( usePTY=False, command=["ietf/manage.py", "migrate"], )) +# This will not only do a prelimnary sanity check, but also patch libs as needed: +factory.addStep(ShellCommand( + descriptionDone="run django checks", + workdir=Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + usePTY=False, + command=["ietf/manage.py", "check"], + )) factory.addStep(TestCrawlerShellCommand( workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, @@ -461,13 +483,14 @@ factory.addStep(TestCrawlerShellCommand( factory.addStep(ShellCommand( descriptionDone="mark as passed", workdir=Interpolate('build/%(src::branch)s'), + flunkOnFailure=False, usePTY=False, command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", "propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ], )) c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk", - slavenames=["datatracker_lin_py36_6", ])) + slavenames=["dunkelfelder_lin_py36_4", ])) # -*- section Builder_Verify_Old_Libs -*- @@ -479,6 +502,7 @@ c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, categor # dependencies. factory = BuildFactory() +factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) factory.addStep(ShellCommand( descriptionDone="remove tweaked requirements", workdir=Interpolate('build/%(src::branch)s'), @@ -497,7 +521,7 @@ factory.addStep(SVN( repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) +factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False)) factory.addStep(ShellCommand( descriptionDone="edit requirements", workdir=Interpolate('build/%(src::branch)s'), @@ -512,7 +536,6 @@ factory.addStep(ShellCommand( usePTY=False, command=["pip", "install", "--upgrade", "-r", "requirements.txt"], )) -factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) factory.addStep(ShellCommand( descriptionDone="seting up settings_local.py", workdir=Interpolate('build/%(src::branch)s'), @@ -542,10 +565,10 @@ factory.addStep(UnitTest( command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], )) c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk", - slavenames=["datatracker_lin_py36_5", ])) + slavenames=["dornfelder_lin_py36_3", ])) -# -*- section Builder_Dependencies -*- +# -*- section Verify_Latest_Libs -*- # This build runs pip install --upgrade, to make sure that we install the latest version of all # dependencies, in order to get an indication if/when an incompatibility turns up with a new @@ -554,6 +577,7 @@ c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, # dependencies. factory = BuildFactory() +factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) factory.addStep(SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", @@ -564,7 +588,7 @@ factory.addStep(SVN( repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) +factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'), usePTY=False)) factory.addStep(ShellCommand( descriptionDone="install/upgrade requirements", workdir=Interpolate('build/%(src::branch)s'), @@ -572,7 +596,6 @@ factory.addStep(ShellCommand( usePTY=False, command=["pip", "install", "--upgrade", "-r", "requirements.txt"], )) -factory.addStep(SetPropertiesFromEnv(variables=['HOME',])) factory.addStep(ShellCommand( descriptionDone="seting up settings_local.py", workdir=Interpolate('build/%(src::branch)s'), @@ -603,7 +626,7 @@ factory.addStep(UnitTest( )) c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk", - slavenames=["datatracker_lin_py36_5", ])) + slavenames=["dornfelder_lin_py36_3", ])) ####### STATUS TARGETS @@ -673,7 +696,7 @@ c['status'].append(mail.MailNotifier( # installation's html.WebStatus home page (linked to the # 'titleURL') and is embedded in the title of the waterfall HTML page. -c['title'] = "IETF Datatracker" +c['title'] = "Buildbot: IETF Datatracker" c['titleURL'] = "https://datatracker.ietf.org/" # the 'buildbotURL' string should point to the location where the buildbot's