Added a buildbot master config file for buildbot 2.7, and a passwords template.

- Legacy-Id: 17658

buildbot/masters/datatracker/buildbot_passwords.py.template

@@ -0,0 +1,21 @@
+# Copyright The IETF Trust 2015-2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+# Uncomment and set passwords below to match those set for the workers, then
+# save as "buildbot_passwords.py"
+
+# datatracker_lin_py27_1_pw   = "<password>"
+# datatracker_lin_py27_2_pw   = "<password>"
+# datatracker_lin_py27_3_pw   = "<password>"
+# datatracker_osx_py27_4_pw   = "<password>"
+# datatracker_lin_py27_5_pw   = "<password>"
+# datatracker_lin_py27_6_pw   = "<password>"
+# ietfdb_svn_hook_pw          = "<password>"
+# 
+# datatracker_lin_py36_1_pw   = "<password>"
+# datatracker_lin_py36_2_pw   = "<password>"
+# datatracker_lin_py36_3_pw   = "<password>"
+# datatracker_lin_py36_4_pw   = "<password>"
+# datatracker_lin_py36_5_pw   = "<password>"
+# datatracker_lin_py36_6_pw   = "<password>"
+

buildbot/masters/datatracker/custom_steps.py

@@ -66,6 +66,7 @@ class TestCrawlerShellCommand(steps.WarningCountingShellCommand):
                 wregex[key] = pattern
 
         # Count matches to the various warning patterns
+        last_line = None
         for line in log.getText().split("\n"):
             for key in wregex:
                 match = re.search(wregex[key], line)
@@ -83,11 +84,12 @@ class TestCrawlerShellCommand(steps.WarningCountingShellCommand):
                 self.step_status.setStatistic(key, len(warnings[key]))
             self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand")
 
-        match = re.search(self.logline, last_line)
+        if last_line:
-        for key in ['elapsed', 'pages']:
+            match = re.search(self.logline, last_line)
-            info = match.group(key)
+            for key in ['elapsed', 'pages']:
-            self.step_status.setStatistic(key, info)
+                info = match.group(key)
-            self.setProperty(key, info, "TestCrawlerShellCommand")
+                self.step_status.setStatistic(key, info)
+                self.setProperty(key, info, "TestCrawlerShellCommand")
 
     def describe(self, done=False):
         description = steps.WarningCountingShellCommand.describe(self, done)
@@ -113,7 +115,7 @@ class DjangoTest(steps.WarningCountingShellCommand):
     warnOnFailure = 1
     description = ["testing"]
     descriptionDone = ["test"]
-    command = ["python", "-m", "unittest", "discover"]
+    command = ["manage.py", "test", ]
 
     regexPatterns = {
         "tests":            "Ran (\d+) tests in [0-9.]+s",

buildbot/masters/datatracker/master.buildbot27.cfg

@@ -0,0 +1,426 @@
+# -*- python -*-
+# ex: set filetype=python:
+
+from os.path import expanduser as expandtilde
+from buildbot.plugins import worker, changes, schedulers, util, steps
+import buildbot_passwords
+import custom_steps
+
+# This is a sample buildmaster config file. It must be installed as
+# 'master.cfg' in your buildmaster's base directory.
+
+# This is the dictionary that the buildmaster pays attention to. We also use
+# a shorter alias to save typing.
+c = BuildmasterConfig = {}
+
+####### SETTINGS
+
+# For miscellaneous settings, see MISC. SETTINGS at the bottom of the file
+
+####### WORKERS
+
+# The 'workers' list defines the set of recognized workers. Each element is
+# a Worker object, specifying a unique worker name and password.  The same
+# worker name and password must be configured on the worker.
+c['workers'] = [
+    worker.Worker("datatracker_lin_py36_1", buildbot_passwords.datatracker_lin_py36_1_pw),
+    worker.Worker("datatracker_lin_py36_2", buildbot_passwords.datatracker_lin_py36_2_pw),
+    worker.Worker("datatracker_lin_py36_3", buildbot_passwords.datatracker_lin_py36_3_pw),
+#    worker.Worker("datatracker_lin_py36_4", buildbot_passwords.datatracker_lin_py36_4_pw),
+    worker.Worker("datatracker_lin_py36_5", buildbot_passwords.datatracker_lin_py36_5_pw),
+    worker.Worker("datatracker_lin_py36_6", buildbot_passwords.datatracker_lin_py36_6_pw),
+]
+
+# 'protocols' contains information about protocols which master will use for
+# communicating with workers. You must define at least 'port' option that workers
+# could connect to your master with this protocol.
+# 'port' must match the value configured into the workers (with their
+# --master option)
+c['protocols'] = {'pb': {'port': 9989}}
+
+####### CHANGESOURCES
+
+# the 'change_source' setting tells the buildmaster how it should find out
+# about source code changes.  Here we point to the buildbot version of a python hello-world project.
+
+c['change_source'] = [
+    changes.PBChangeSource(user='ietfdb', passwd=buildbot_passwords.ietfdb_svn_hook_pw),
+]
+
+####### SCHEDULERS
+
+# Configure the Schedulers, which decide how to react to incoming changes.  In this
+# case, just kick off a 'runtests' build
+
+c['schedulers'] = [
+    # Branch schedulers
+    schedulers.SingleBranchScheduler(name="pyflakes",      treeStableTimer=10,     builderNames=["Check PyFlakes"],
+        change_filter=util.ChangeFilter(branch='trunk')),
+    schedulers.SingleBranchScheduler(name="lin_test",      treeStableTimer=60*5,   builderNames=["Test Suite"],
+        change_filter=util.ChangeFilter(branch='trunk')),
+#    schedulers.SingleBranchScheduler(name="osx_test",      treeStableTimer=60*5,   builderNames=["Test Suite (OS X)"],
+#        change_filter=util.ChangeFilter(branch='trunk')),
+    #
+    schedulers.AnyBranchScheduler(name="pyflakes_branch",  treeStableTimer=10,     builderNames=["[branch] Check PyFlakes"],
+        change_filter=util.ChangeFilter(branch_re='branch/.*')),
+#    schedulers.AnyBranchScheduler(name="lin_test_branch",  treeStableTimer=60*5,   builderNames=["[branch] Test Suite"],
+#        change_filter=util.ChangeFilter(branch_re='branch/.*')),
+#    schedulers.AnyBranchScheduler(name="osx_test_branch",  treeStableTimer=60*5,   builderNames=["[branch] Test Suite (OS X)"],
+#        change_filter=util.ChangeFilter(branch_re='branch/.*')),
+    #
+    schedulers.AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10,     builderNames=["[personal] Check PyFlakes"],
+        change_filter=util.ChangeFilter(branch_re='personal/.*')),
+    schedulers.AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5,   builderNames=["[personal] Test Suite"],
+        change_filter=util.ChangeFilter(branch_re='personal/.*')),
+    # Periodic Schedulers
+    schedulers.Nightly(name="lin_test_old_libs",  hour=16, minute=12, branch="trunk",  builderNames=["Verify Minimum Libs"],),
+    schedulers.Nightly(name="lin_test_libs",      hour=16, minute=42, branch="trunk",  builderNames=["Verify Latest Libs"],),
+    schedulers.Nightly(name="crawler",            hour=9,  minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],),
+
+    # schedulers.Force schedulers
+    schedulers.ForceScheduler(name="force_pyflakes",                               builderNames=["Check PyFlakes"]),
+    schedulers.ForceScheduler(name="force_lin_test",                               builderNames=["Test Suite"]),
+#    schedulers.ForceScheduler(name="force_osx_test",                               builderNames=["Test Suite (OS X)"]),
+    schedulers.ForceScheduler(name="force_test_crawler",                           builderNames=["Test-Crawler"]),
+    #
+    schedulers.ForceScheduler(name="force_pyflakes_branch",                        builderNames=["[branch] Check PyFlakes"]),
+    schedulers.ForceScheduler(name="force_lin_test_branch",                        builderNames=["[branch] Test Suite"]),
+#    schedulers.ForceScheduler(name="force_osx_test_branch",                        builderNames=["[branch] Test Suite (OS X)"]),
+    #
+    schedulers.ForceScheduler(name="force_pyflakes_personal",                      builderNames=["[personal] Check PyFlakes"]),
+    schedulers.ForceScheduler(name="force_lin_test_personal",                      builderNames=["[personal] Test Suite"]),
+
+]
+
+####### BUILDERS
+
+# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
+# what steps, and which workers can execute them.  Note that any particular build will
+# only take place on one worker.
+
+
+#### Custom subclassed builder
+
+factory = util.BuildFactory()
+# check out the source
+factory.addStep(steps.Git(repourl='git://github.com/buildbot/hello-world.git', mode='incremental'))
+# run the tests (note that this will require that 'trial' is installed)
+factory.addStep(steps.ShellCommand(command=["trial", "hello"],
+                                   env={"PYTHONPATH": "."}))
+
+c['builders'] = []
+
+# -*- section Builder_Run_pyflakes -*-
+
+factory = util.BuildFactory()
+factory.addStep(steps.SVN(
+                username='buildbot@tools.ietf.org',
+                descriptionDone="svn update",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
+                descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="seting up settings_local.py",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
+            ))
+factory.addStep(steps.PyFlakes(
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["ietf/manage.py", "pyflakes", "--verbosity=0"],
+            ))
+# This should be the last action
+factory.addStep(steps.ShellCommand(
+                descriptionDone="mark as passed",
+                workdir=util.Interpolate('build/%(src::branch)s'), 
+                command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
+                    "propset", "--revprop", "-r", util.Property('got_revision'), "test:pyflakes", "passed" ],
+            ))
+
+c['builders'].append(util.BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk",
+                workernames=["datatracker_lin_py36_1", ]))
+c['builders'].append(util.BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch",
+                workernames=["datatracker_lin_py36_2", ]))
+c['builders'].append(util.BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal",
+                workernames=["datatracker_lin_py36_3", ]))
+
+# -*- section Builder_TestSuite -*-
+
+factory = util.BuildFactory()
+factory.addStep(steps.SVN(
+                username='buildbot@tools.ietf.org',
+                descriptionDone="svn update",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
+                descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
+            ))
+factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="remove tmp-* dirs",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["rm", "-rf", "tmp-*/"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="install requirements",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["pip", "install", "-r", "requirements.txt"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="copy settings_local.py",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="collect static files",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=False,
+                flunkOnFailure=False,
+                command=["ietf/manage.py", "collectstatic", "--noinput", ],
+            ))
+factory.addStep(custom_steps.DjangoTest(
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                want_stderr=True,
+                command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
+            ))
+# This should come after tests
+factory.addStep(steps.ShellCommand(
+                descriptionDone="mark as passed",
+                workdir=util.Interpolate('build/%(src::branch)s'), 
+                command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
+                    "propset", "--revprop", "-r", util.Property('got_revision'), "test:unittest", "passed" ],
+            ))
+
+c['builders'].append(util.BuilderConfig(name="Test Suite", factory=factory, category="1. trunk",
+        workernames=["datatracker_lin_py36_1", ]))
+c['builders'].append(util.BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch",
+        workernames=["datatracker_lin_py36_2", ]))
+c['builders'].append(util.BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal",
+        workernames=["datatracker_lin_py36_3", ]))
+
+
+# -*- section Builder_TestCrawler -*-
+
+factory = util.BuildFactory()
+factory.addStep(steps.SVN(
+                username='buildbot@tools.ietf.org',
+                descriptionDone="svn update",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
+                descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
+            ))
+factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="install requirements",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["pip", "install", "-r", "requirements.txt"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="copy settings_local.py",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="run migrations",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["ietf/manage.py", "migrate"], 
+            ))
+factory.addStep(custom_steps.TestCrawlerShellCommand(
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                want_stderr=True,
+                command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"],
+            ))
+# This should be the last action
+factory.addStep(steps.ShellCommand(
+                descriptionDone="mark as passed",
+                workdir=util.Interpolate('build/%(src::branch)s'), 
+                command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive",
+                    "propset", "--revprop", "-r", util.Property('got_revision'), "test:crawler", "passed" ],
+            ))
+
+c['builders'].append(util.BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk",
+        workernames=["datatracker_lin_py36_6", ]))
+
+
+# -*- section Builder_Verify_Minimum_Libs -*-
+
+# This build runs pip install --upgrade, to make sure that we install the earliest version of
+# all dependencies, in order to get an indication if/when an incompatibility turns up with a new
+# version of a dependency.  The other test suite builders *don't* use --upgrade, in order to not
+# change the external test conditions and produce spurious errors because of version changes in
+# dependencies.
+
+factory = util.BuildFactory()
+factory.addStep(steps.ShellCommand(
+                descriptionDone="remove tweaked requirements",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=False,
+                flunkOnFailure=False,
+                command=["rm", "requirements.txt"],
+            ))
+factory.addStep(steps.SVN(
+                username='buildbot@tools.ietf.org',
+                descriptionDone="svn update",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                alwaysUseLatest=True,
+                haltOnFailure=True,
+                repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
+                descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
+            ))
+factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="edit requirements",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="install/upgrade requirements",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="seting up settings_local.py",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="list installed pyton modules",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["pip", "freeze"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="collect static files",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=False,
+                flunkOnFailure=False,
+                command=["ietf/manage.py", "collectstatic", "--noinput", ],
+            ))
+factory.addStep(custom_steps.DjangoTest(
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
+            ))
+c['builders'].append(util.BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk",
+        workernames=["datatracker_lin_py36_5", ]))
+
+
+# -*- section Builder_Veryfy_Latest_Libs -*-
+
+# This build runs pip install --upgrade, to make sure that we install the latest version of all
+# dependencies, in order to get an indication if/when an incompatibility turns up with a new
+# version of a dependency.  The other test suite builders *don't* use --upgrade, in order to not
+# change the external test conditions and produce spurious errors because of version changes in
+# dependencies.
+
+factory = util.BuildFactory()
+factory.addStep(steps.SVN(
+                username='buildbot@tools.ietf.org',
+                descriptionDone="svn update",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                alwaysUseLatest=True,
+                haltOnFailure=True,
+                repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'),
+                descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')],
+            ))
+factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s')))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="install/upgrade requirements",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["pip", "install", "--upgrade", "-r", "requirements.txt"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="seting up settings_local.py",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["cp", expandtilde("~/settings_local.py"), "./ietf/"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="list installed pyton modules",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["pip", "freeze"],
+            ))
+factory.addStep(steps.ShellCommand(
+                descriptionDone="collect static files",
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=False,
+                flunkOnFailure=False,
+                command=["ietf/manage.py", "collectstatic", "--noinput", ],
+            ))
+factory.addStep(custom_steps.DjangoTest(
+                workdir=util.Interpolate('build/%(src::branch)s'),
+                haltOnFailure=True,
+                command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ],
+            ))
+
+c['builders'].append(util.BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk",
+        workernames=["datatracker_lin_py36_5", ]))
+
+####### BUILDBOT SERVICES
+
+# 'services' is a list of BuildbotService items like reporter targets. The
+# status of each build will be pushed to these targets. buildbot/reporters/*.py
+# has a variety to choose from, like IRC bots.
+
+c['services'] = []
+
+####### PROJECT IDENTITY
+
+# the 'title' string will appear at the top of this buildbot installation's
+# home pages (linked to the 'titleURL').
+
+c['title'] = "Buildbot: IETF Datatracker"
+c['titleURL'] = "https://datatracker.ietf.org/"
+
+# the 'buildbotURL' string should point to the location where the buildbot's
+# internal web server is visible. This typically uses the port number set in
+# the 'www' entry below, but with an externally-visible host name which the
+# buildbot cannot figure out without some help.
+
+c['buildbotURL'] = "http://dunkelfelder.tools.ietf.org:8010/"
+
+# minimalistic config to activate new web UI
+c['www'] = {
+    'port': 8010,
+    'plugins': {
+        'waterfall_view':   True,
+        'console_view':     True,
+        'grid_view':        True,
+    },
+    'default_page': 'waterfall_view',
+    'debug': True,
+    'auth': util.UserPasswordAuth({"ietfdb": "ietfdb"}),
+}
+
+
+####### DB URL
+
+c['db'] = {
+    # This specifies what database buildbot uses to store its state.
+    # It's easy to start with sqlite, but it's recommended to switch to a dedicated
+    # database, such as PostgreSQL or MySQL, for use in production environments.
+    # http://docs.buildbot.net/current/manual/configuration/global.html#database-specification
+    'db_url' : "sqlite:///state.sqlite",
+}
+
+####### MISC. SETTINGS
+
+c['buildbotNetUsageData'] = 'full'
+