diff --git a/buildbot/masters/datatracker/buildbot_passwords.py.template b/buildbot/masters/datatracker/buildbot_passwords.py.template new file mode 100644 index 000000000..2aaea1238 --- /dev/null +++ b/buildbot/masters/datatracker/buildbot_passwords.py.template @@ -0,0 +1,21 @@ +# Copyright The IETF Trust 2015-2020, All Rights Reserved +# -*- coding: utf-8 -*- + +# Uncomment and set passwords below to match those set for the workers, then +# save as "buildbot_passwords.py" + +# datatracker_lin_py27_1_pw = "" +# datatracker_lin_py27_2_pw = "" +# datatracker_lin_py27_3_pw = "" +# datatracker_osx_py27_4_pw = "" +# datatracker_lin_py27_5_pw = "" +# datatracker_lin_py27_6_pw = "" +# ietfdb_svn_hook_pw = "" +# +# datatracker_lin_py36_1_pw = "" +# datatracker_lin_py36_2_pw = "" +# datatracker_lin_py36_3_pw = "" +# datatracker_lin_py36_4_pw = "" +# datatracker_lin_py36_5_pw = "" +# datatracker_lin_py36_6_pw = "" + diff --git a/buildbot/masters/datatracker/custom_steps.py b/buildbot/masters/datatracker/custom_steps.py index 5e1c6294b..593d29b2d 100644 --- a/buildbot/masters/datatracker/custom_steps.py +++ b/buildbot/masters/datatracker/custom_steps.py @@ -66,6 +66,7 @@ class TestCrawlerShellCommand(steps.WarningCountingShellCommand): wregex[key] = pattern # Count matches to the various warning patterns + last_line = None for line in log.getText().split("\n"): for key in wregex: match = re.search(wregex[key], line) @@ -83,11 +84,12 @@ class TestCrawlerShellCommand(steps.WarningCountingShellCommand): self.step_status.setStatistic(key, len(warnings[key])) self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand") - match = re.search(self.logline, last_line) - for key in ['elapsed', 'pages']: - info = match.group(key) - self.step_status.setStatistic(key, info) - self.setProperty(key, info, "TestCrawlerShellCommand") + if last_line: + match = re.search(self.logline, last_line) + for key in ['elapsed', 'pages']: + info = match.group(key) + self.step_status.setStatistic(key, info) + self.setProperty(key, info, "TestCrawlerShellCommand") def describe(self, done=False): description = steps.WarningCountingShellCommand.describe(self, done) @@ -113,7 +115,7 @@ class DjangoTest(steps.WarningCountingShellCommand): warnOnFailure = 1 description = ["testing"] descriptionDone = ["test"] - command = ["python", "-m", "unittest", "discover"] + command = ["manage.py", "test", ] regexPatterns = { "tests": "Ran (\d+) tests in [0-9.]+s", diff --git a/buildbot/masters/datatracker/master.buildbot27.cfg b/buildbot/masters/datatracker/master.buildbot27.cfg new file mode 100644 index 000000000..e4b1535b7 --- /dev/null +++ b/buildbot/masters/datatracker/master.buildbot27.cfg @@ -0,0 +1,426 @@ +# -*- python -*- +# ex: set filetype=python: + +from os.path import expanduser as expandtilde +from buildbot.plugins import worker, changes, schedulers, util, steps +import buildbot_passwords +import custom_steps + +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory. + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +####### SETTINGS + +# For miscellaneous settings, see MISC. SETTINGS at the bottom of the file + +####### WORKERS + +# The 'workers' list defines the set of recognized workers. Each element is +# a Worker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. +c['workers'] = [ + worker.Worker("datatracker_lin_py36_1", buildbot_passwords.datatracker_lin_py36_1_pw), + worker.Worker("datatracker_lin_py36_2", buildbot_passwords.datatracker_lin_py36_2_pw), + worker.Worker("datatracker_lin_py36_3", buildbot_passwords.datatracker_lin_py36_3_pw), +# worker.Worker("datatracker_lin_py36_4", buildbot_passwords.datatracker_lin_py36_4_pw), + worker.Worker("datatracker_lin_py36_5", buildbot_passwords.datatracker_lin_py36_5_pw), + worker.Worker("datatracker_lin_py36_6", buildbot_passwords.datatracker_lin_py36_6_pw), +] + +# 'protocols' contains information about protocols which master will use for +# communicating with workers. You must define at least 'port' option that workers +# could connect to your master with this protocol. +# 'port' must match the value configured into the workers (with their +# --master option) +c['protocols'] = {'pb': {'port': 9989}} + +####### CHANGESOURCES + +# the 'change_source' setting tells the buildmaster how it should find out +# about source code changes. Here we point to the buildbot version of a python hello-world project. + +c['change_source'] = [ + changes.PBChangeSource(user='ietfdb', passwd=buildbot_passwords.ietfdb_svn_hook_pw), +] + +####### SCHEDULERS + +# Configure the Schedulers, which decide how to react to incoming changes. In this +# case, just kick off a 'runtests' build + +c['schedulers'] = [ + # Branch schedulers + schedulers.SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"], + change_filter=util.ChangeFilter(branch='trunk')), + schedulers.SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"], + change_filter=util.ChangeFilter(branch='trunk')), +# schedulers.SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"], +# change_filter=util.ChangeFilter(branch='trunk')), + # + schedulers.AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"], + change_filter=util.ChangeFilter(branch_re='branch/.*')), +# schedulers.AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"], +# change_filter=util.ChangeFilter(branch_re='branch/.*')), +# schedulers.AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"], +# change_filter=util.ChangeFilter(branch_re='branch/.*')), + # + schedulers.AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"], + change_filter=util.ChangeFilter(branch_re='personal/.*')), + schedulers.AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"], + change_filter=util.ChangeFilter(branch_re='personal/.*')), + # Periodic Schedulers + schedulers.Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],), + schedulers.Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],), + schedulers.Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],), + + # schedulers.Force schedulers + schedulers.ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]), + schedulers.ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]), +# schedulers.ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]), + schedulers.ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]), + # + schedulers.ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]), + schedulers.ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]), +# schedulers.ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]), + # + schedulers.ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]), + schedulers.ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]), + +] + +####### BUILDERS + +# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: +# what steps, and which workers can execute them. Note that any particular build will +# only take place on one worker. + + +#### Custom subclassed builder + +factory = util.BuildFactory() +# check out the source +factory.addStep(steps.Git(repourl='git://github.com/buildbot/hello-world.git', mode='incremental')) +# run the tests (note that this will require that 'trial' is installed) +factory.addStep(steps.ShellCommand(command=["trial", "hello"], + env={"PYTHONPATH": "."})) + +c['builders'] = [] + +# -*- section Builder_Run_pyflakes -*- + +factory = util.BuildFactory() +factory.addStep(steps.SVN( + username='buildbot@tools.ietf.org', + descriptionDone="svn update", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="seting up settings_local.py", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], + )) +factory.addStep(steps.PyFlakes( + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["ietf/manage.py", "pyflakes", "--verbosity=0"], + )) +# This should be the last action +factory.addStep(steps.ShellCommand( + descriptionDone="mark as passed", + workdir=util.Interpolate('build/%(src::branch)s'), + command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", + "propset", "--revprop", "-r", util.Property('got_revision'), "test:pyflakes", "passed" ], + )) + +c['builders'].append(util.BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk", + workernames=["datatracker_lin_py36_1", ])) +c['builders'].append(util.BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch", + workernames=["datatracker_lin_py36_2", ])) +c['builders'].append(util.BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal", + workernames=["datatracker_lin_py36_3", ])) + +# -*- section Builder_TestSuite -*- + +factory = util.BuildFactory() +factory.addStep(steps.SVN( + username='buildbot@tools.ietf.org', + descriptionDone="svn update", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], + )) +factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) +factory.addStep(steps.ShellCommand( + descriptionDone="remove tmp-* dirs", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["rm", "-rf", "tmp-*/"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="install requirements", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["pip", "install", "-r", "requirements.txt"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="copy settings_local.py", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="collect static files", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=False, + flunkOnFailure=False, + command=["ietf/manage.py", "collectstatic", "--noinput", ], + )) +factory.addStep(custom_steps.DjangoTest( + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + want_stderr=True, + command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], + )) +# This should come after tests +factory.addStep(steps.ShellCommand( + descriptionDone="mark as passed", + workdir=util.Interpolate('build/%(src::branch)s'), + command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", + "propset", "--revprop", "-r", util.Property('got_revision'), "test:unittest", "passed" ], + )) + +c['builders'].append(util.BuilderConfig(name="Test Suite", factory=factory, category="1. trunk", + workernames=["datatracker_lin_py36_1", ])) +c['builders'].append(util.BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch", + workernames=["datatracker_lin_py36_2", ])) +c['builders'].append(util.BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal", + workernames=["datatracker_lin_py36_3", ])) + + +# -*- section Builder_TestCrawler -*- + +factory = util.BuildFactory() +factory.addStep(steps.SVN( + username='buildbot@tools.ietf.org', + descriptionDone="svn update", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], + )) +factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) +factory.addStep(steps.ShellCommand( + descriptionDone="install requirements", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["pip", "install", "-r", "requirements.txt"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="copy settings_local.py", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="run migrations", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["ietf/manage.py", "migrate"], + )) +factory.addStep(custom_steps.TestCrawlerShellCommand( + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + want_stderr=True, + command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"], + )) +# This should be the last action +factory.addStep(steps.ShellCommand( + descriptionDone="mark as passed", + workdir=util.Interpolate('build/%(src::branch)s'), + command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", + "propset", "--revprop", "-r", util.Property('got_revision'), "test:crawler", "passed" ], + )) + +c['builders'].append(util.BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk", + workernames=["datatracker_lin_py36_6", ])) + + +# -*- section Builder_Verify_Minimum_Libs -*- + +# This build runs pip install --upgrade, to make sure that we install the earliest version of +# all dependencies, in order to get an indication if/when an incompatibility turns up with a new +# version of a dependency. The other test suite builders *don't* use --upgrade, in order to not +# change the external test conditions and produce spurious errors because of version changes in +# dependencies. + +factory = util.BuildFactory() +factory.addStep(steps.ShellCommand( + descriptionDone="remove tweaked requirements", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=False, + flunkOnFailure=False, + command=["rm", "requirements.txt"], + )) +factory.addStep(steps.SVN( + username='buildbot@tools.ietf.org', + descriptionDone="svn update", + workdir=util.Interpolate('build/%(src::branch)s'), + alwaysUseLatest=True, + haltOnFailure=True, + repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], + )) +factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) +factory.addStep(steps.ShellCommand( + descriptionDone="edit requirements", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="install/upgrade requirements", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["pip", "install", "--upgrade", "-r", "requirements.txt"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="seting up settings_local.py", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="list installed pyton modules", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["pip", "freeze"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="collect static files", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=False, + flunkOnFailure=False, + command=["ietf/manage.py", "collectstatic", "--noinput", ], + )) +factory.addStep(custom_steps.DjangoTest( + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], + )) +c['builders'].append(util.BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk", + workernames=["datatracker_lin_py36_5", ])) + + +# -*- section Builder_Veryfy_Latest_Libs -*- + +# This build runs pip install --upgrade, to make sure that we install the latest version of all +# dependencies, in order to get an indication if/when an incompatibility turns up with a new +# version of a dependency. The other test suite builders *don't* use --upgrade, in order to not +# change the external test conditions and produce spurious errors because of version changes in +# dependencies. + +factory = util.BuildFactory() +factory.addStep(steps.SVN( + username='buildbot@tools.ietf.org', + descriptionDone="svn update", + workdir=util.Interpolate('build/%(src::branch)s'), + alwaysUseLatest=True, + haltOnFailure=True, + repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], + )) +factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) +factory.addStep(steps.ShellCommand( + descriptionDone="install/upgrade requirements", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["pip", "install", "--upgrade", "-r", "requirements.txt"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="seting up settings_local.py", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="list installed pyton modules", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["pip", "freeze"], + )) +factory.addStep(steps.ShellCommand( + descriptionDone="collect static files", + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=False, + flunkOnFailure=False, + command=["ietf/manage.py", "collectstatic", "--noinput", ], + )) +factory.addStep(custom_steps.DjangoTest( + workdir=util.Interpolate('build/%(src::branch)s'), + haltOnFailure=True, + command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], + )) + +c['builders'].append(util.BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk", + workernames=["datatracker_lin_py36_5", ])) + +####### BUILDBOT SERVICES + +# 'services' is a list of BuildbotService items like reporter targets. The +# status of each build will be pushed to these targets. buildbot/reporters/*.py +# has a variety to choose from, like IRC bots. + +c['services'] = [] + +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot installation's +# home pages (linked to the 'titleURL'). + +c['title'] = "Buildbot: IETF Datatracker" +c['titleURL'] = "https://datatracker.ietf.org/" + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server is visible. This typically uses the port number set in +# the 'www' entry below, but with an externally-visible host name which the +# buildbot cannot figure out without some help. + +c['buildbotURL'] = "http://dunkelfelder.tools.ietf.org:8010/" + +# minimalistic config to activate new web UI +c['www'] = { + 'port': 8010, + 'plugins': { + 'waterfall_view': True, + 'console_view': True, + 'grid_view': True, + }, + 'default_page': 'waterfall_view', + 'debug': True, + 'auth': util.UserPasswordAuth({"ietfdb": "ietfdb"}), +} + + +####### DB URL + +c['db'] = { + # This specifies what database buildbot uses to store its state. + # It's easy to start with sqlite, but it's recommended to switch to a dedicated + # database, such as PostgreSQL or MySQL, for use in production environments. + # http://docs.buildbot.net/current/manual/configuration/global.html#database-specification + 'db_url' : "sqlite:///state.sqlite", +} + +####### MISC. SETTINGS + +c['buildbotNetUsageData'] = 'full' +