From cdaa2bc437bbbb07ebf3d27bbea6d5d4d329a8cb Mon Sep 17 00:00:00 2001 From: buildbot Date: Sun, 19 Apr 2020 13:21:49 +0000 Subject: [PATCH] Updated buildbot from 0.8 to 2.7, transitioned to Python 3, and added some convenience scripts. - Legacy-Id: 17657 --- buildbot/masters/datatracker/custom_steps.py | 181 +++++ buildbot/masters/datatracker/master.cfg | 691 ++++++------------ buildbot/restart-masters | 3 + buildbot/restart-workers | 3 + buildbot/setup-buildbot-workers | 180 +++++ .../workers/datatracker_lin_py36_1/info/admin | 1 + .../workers/datatracker_lin_py36_1/info/host | 1 + .../workers/datatracker_lin_py36_2/info/admin | 1 + .../workers/datatracker_lin_py36_2/info/host | 1 + .../workers/datatracker_lin_py36_3/info/admin | 1 + .../workers/datatracker_lin_py36_3/info/host | 1 + .../workers/datatracker_lin_py36_4/info/admin | 1 + .../workers/datatracker_lin_py36_4/info/host | 1 + .../workers/datatracker_lin_py36_5/info/admin | 1 + .../workers/datatracker_lin_py36_5/info/host | 1 + .../workers/datatracker_lin_py36_6/info/admin | 1 + .../workers/datatracker_lin_py36_6/info/host | 1 + 17 files changed, 590 insertions(+), 480 deletions(-) create mode 100644 buildbot/masters/datatracker/custom_steps.py create mode 100755 buildbot/restart-masters create mode 100755 buildbot/restart-workers create mode 100755 buildbot/setup-buildbot-workers create mode 100644 buildbot/workers/datatracker_lin_py36_1/info/admin create mode 100644 buildbot/workers/datatracker_lin_py36_1/info/host create mode 100644 buildbot/workers/datatracker_lin_py36_2/info/admin create mode 100644 buildbot/workers/datatracker_lin_py36_2/info/host create mode 100644 buildbot/workers/datatracker_lin_py36_3/info/admin create mode 100644 buildbot/workers/datatracker_lin_py36_3/info/host create mode 100644 buildbot/workers/datatracker_lin_py36_4/info/admin create mode 100644 buildbot/workers/datatracker_lin_py36_4/info/host create mode 100644 buildbot/workers/datatracker_lin_py36_5/info/admin create mode 100644 buildbot/workers/datatracker_lin_py36_5/info/host create mode 100644 buildbot/workers/datatracker_lin_py36_6/info/admin create mode 100644 buildbot/workers/datatracker_lin_py36_6/info/host diff --git a/buildbot/masters/datatracker/custom_steps.py b/buildbot/masters/datatracker/custom_steps.py new file mode 100644 index 000000000..5e1c6294b --- /dev/null +++ b/buildbot/masters/datatracker/custom_steps.py @@ -0,0 +1,181 @@ +# Copyright The IETF Trust 2015-2020, All Rights Reserved +# -*- coding: utf-8 -*- + +import re + +from buildbot.plugins import steps + +class TestCrawlerShellCommand(steps.WarningCountingShellCommand): + name = "testcrawl" + haltOnFailure = 1 + flunkOnFailure = 1 + descriptionDone = ["test crawler"] + command=["bin/test-crawl"] + + warningPatterns = { + "exceptions": "^(Traceback| File| |.*Error|.*Exception)", + "failed": " FAIL ", + "warnings": " WARN", + "slow": " SLOW", + "invalid_html": " invalid html:", + } + + logline = "^ *(?P\d+:\d+:\d+) +(?P\d+) +(?P\d+) +(?P\d+) +(?P\d+.\d+)s +(?P.+)" + + def setTestResults(self, **kwargs): + """ + Called by subclasses to set the relevant statistics; this actually + adds to any statistics already present + """ + for kw in kwargs: + value = kwargs[kw] + if value.isdigit(): + # Counter + value = int(value) + value += self.step_status.getStatistic(kw, 0) + elif re.search("^[0-9]+\.[0-9]+$", value): + # Runtime + value = float(value) + value += self.step_status.getStatistic(kw, 0) + else: + # This is a percentage, and we can't add them + pass + self.step_status.setStatistic(kw, value) + + def createSummary(self, log): + """ + Match log lines against warningPattern. + + Warnings are collected into another log for this step, and the + build-wide 'warnings-count' is updated.""" + + warnings = {} + wregex = {} + + regex_class = re.compile("").__class__ + + if not isinstance(self.logline, regex_class): + self.logline = re.compile(self.logline) + + for key in self.warningPatterns: + warnings[key] = [] + pattern = self.warningPatterns[key] + if not isinstance(pattern, regex_class): + wregex[key] = re.compile(pattern) + else: + wregex[key] = pattern + + # Count matches to the various warning patterns + for line in log.getText().split("\n"): + for key in wregex: + match = re.search(wregex[key], line) + if match: + warnings[key].append(line) + if re.search(self.logline, line): + last_line = line + + # If there were any warnings, make the log if lines with warnings + # available + for key in warnings: + if len(warnings[key]) > 0: + self.addCompleteLog("%s (%d)" % (key, len(warnings[key])), + "\n".join(warnings[key]) + "\n") + self.step_status.setStatistic(key, len(warnings[key])) + self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand") + + match = re.search(self.logline, last_line) + for key in ['elapsed', 'pages']: + info = match.group(key) + self.step_status.setStatistic(key, info) + self.setProperty(key, info, "TestCrawlerShellCommand") + + def describe(self, done=False): + description = steps.WarningCountingShellCommand.describe(self, done) + if done: + description = description[:] # make a private copy + for name in ["time", "elapsed", "pages", "failed", "warnings", "slow", "invalid_html", ]: + if name in self.step_status.statistics: + value = self.step_status.getStatistic(name) + displayName = name.replace('_', ' ') + # special case. Mph. + if type(value) is float: # this is run-time + description.append('%s: %.2fs' % (displayName, value)) + elif type(value) is int: + description.append('%s: %d' % (displayName, value)) + else: + description.append('%s: %s' % (displayName, value)) + return description + + +class DjangoTest(steps.WarningCountingShellCommand): + + name = "test" + warnOnFailure = 1 + description = ["testing"] + descriptionDone = ["test"] + command = ["python", "-m", "unittest", "discover"] + + regexPatterns = { + "tests": "Ran (\d+) tests in [0-9.]+s", + "time": "Ran \d+ tests in ([0-9.]+)s", + "skipped": "(?:OK|FAILED).*skipped=(\d+)", + "failed": "FAILED.*failures=(\d+)", + "errors": "FAILED.*errors=(\d+)", + "template_coverage":" +Template coverage: +([0-9.]+%)", + "url_coverage": " +Url coverage: +([0-9.]+%)", + "code_coverage": " +Code coverage: +([0-9.]+%)", + } + + def setTestResults(self, **kwargs): + """ + Called by subclasses to set the relevant statistics; this actually + adds to any statistics already present + """ + for kw in kwargs: + value = kwargs[kw] + if value.isdigit(): + # Counter + value = int(value) + value += self.step_status.getStatistic(kw, 0) + elif re.search("^[0-9]+\.[0-9]+$", value): + # Runtime + value = float(value) + value += self.step_status.getStatistic(kw, 0) + else: + # This is a percentage, and we can't add them + pass + self.step_status.setStatistic(kw, value) + + def createSummary(self, log): + info = {} + for line in log.getText().split("\n"): + for key in self.regexPatterns: + regex = self.regexPatterns[key] + match = re.search(regex, line) + if match: + info[key] = match.group(1) + self.setTestResults(**info) + + def describe(self, done=False): + description = steps.WarningCountingShellCommand.describe(self, done) + if done: + description = description[:] # make a private copy + self.step_status.statistics["passed"] = ( + self.step_status.getStatistic("tests",0) - + self.step_status.getStatistic("skipped",0) - + self.step_status.getStatistic("failed",0) - + self.step_status.getStatistic("errors",0)) + for name in ["time", "tests", "passed", "skipped", "failed", "errors", "template_coverage", "url_coverage", "code_coverage", ]: + if name in self.step_status.statistics: + value = self.step_status.getStatistic(name) + displayName = name.replace('_', ' ') + # special case. Mph. + if displayName == 'template coverage': + displayName = 'templ. coverage' + if type(value) is float: # this is run-time + description.append('%s: %.2fs' % (displayName, value)) + elif type(value) is int: + description.append('%s: %d' % (displayName, value)) + else: + description.append('%s: %s' % (displayName, value)) + return description diff --git a/buildbot/masters/datatracker/master.cfg b/buildbot/masters/datatracker/master.cfg index 0ec44ff47..46e40aca1 100644 --- a/buildbot/masters/datatracker/master.cfg +++ b/buildbot/masters/datatracker/master.cfg @@ -1,492 +1,258 @@ # -*- python -*- -# ex: set syntax=python: -import re -from buildbot_passwords import * +# ex: set filetype=python: +from os.path import expanduser as expandtilde +from buildbot.plugins import worker, changes, schedulers, util, steps +import buildbot_passwords +import custom_steps + +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory. # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. c = BuildmasterConfig = {} -# -*- section BuildSlaves -*- +####### SETTINGS -# The 'slaves' list defines the set of recognized buildslaves. Each element is -# a BuildSlave object, specifying a unique slave name and password. The same -# slave name and password must be configured on the slave. -from buildbot.buildslave import BuildSlave -c['slaves'] = [ - BuildSlave("datatracker_lin_py27_1", datatracker_lin_py27_1_pw), - BuildSlave("datatracker_lin_py27_2", datatracker_lin_py27_2_pw), - BuildSlave("datatracker_lin_py27_3", datatracker_lin_py27_3_pw), - BuildSlave("datatracker_osx_py27_4", datatracker_osx_py27_4_pw), - BuildSlave("datatracker_lin_py27_5", datatracker_lin_py27_5_pw), - BuildSlave("datatracker_lin_py27_6", datatracker_lin_py27_6_pw), +# For miscellaneous settings, see MISC. SETTINGS at the bottom of the file + +####### WORKERS + +# The 'workers' list defines the set of recognized workers. Each element is +# a Worker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. +c['workers'] = [ + worker.Worker("datatracker_lin_py36_1", buildbot_passwords.datatracker_lin_py36_1_pw), + worker.Worker("datatracker_lin_py36_2", buildbot_passwords.datatracker_lin_py36_2_pw), + worker.Worker("datatracker_lin_py36_3", buildbot_passwords.datatracker_lin_py36_3_pw), + worker.Worker("datatracker_lin_py36_4", buildbot_passwords.datatracker_lin_py36_4_pw), + worker.Worker("datatracker_lin_py36_5", buildbot_passwords.datatracker_lin_py36_5_pw), + worker.Worker("datatracker_lin_py36_6", buildbot_passwords.datatracker_lin_py36_6_pw), ] # 'protocols' contains information about protocols which master will use for -# communicating with slaves. -# You must define at least 'port' option that slaves could connect to your master -# with this protocol. -# 'port' must match the value configured into the buildslaves (with their +# communicating with workers. You must define at least 'port' option that workers +# could connect to your master with this protocol. +# 'port' must match the value configured into the workers (with their # --master option) -c['protocols'] = {'pb': {'host':'zinfandel.tools.ietf.org', 'port': 9989}} +c['protocols'] = {'pb': {'port': 9989}} ####### CHANGESOURCES -# -*- section ChangeSources -*- # the 'change_source' setting tells the buildmaster how it should find out -# about source code changes. +# about source code changes. Here we point to the buildbot version of a python hello-world project. -from buildbot.changes.pb import PBChangeSource -# c['change_source'] = [] -# with open("users") as file: -# userinfo = json.read(file) -# for user in userinfo: -# prefix = userinfo[user]["prefix"] -# c.['change_source'].append(PBChangeSource(user=user, passwd="BRiR6XcT7x3$", prefix=prefix)) c['change_source'] = [ - PBChangeSource(user="ietfdb", passwd=ietfdb_svn_hook_pw), + changes.PBChangeSource(user='ietfdb', passwd=buildbot_passwords.ietfdb_svn_hook_pw), ] ####### SCHEDULERS -# -*- section Schedulers -*- # Configure the Schedulers, which decide how to react to incoming changes. In this # case, just kick off a 'runtests' build -from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler -from buildbot.schedulers.forcesched import ForceScheduler -from buildbot.schedulers.timed import Nightly -from buildbot.changes import filter c['schedulers'] = [ # Branch schedulers - SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"], - change_filter=filter.ChangeFilter(branch='trunk')), - SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"], - change_filter=filter.ChangeFilter(branch='trunk')), - SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"], - change_filter=filter.ChangeFilter(branch='trunk')), + schedulers.SingleBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["Check PyFlakes"], + change_filter=util.ChangeFilter(branch='trunk')), + schedulers.SingleBranchScheduler(name="lin_test", treeStableTimer=60*5, builderNames=["Test Suite"], + change_filter=util.ChangeFilter(branch='trunk')), +# schedulers.SingleBranchScheduler(name="osx_test", treeStableTimer=60*5, builderNames=["Test Suite (OS X)"], +# change_filter=util.ChangeFilter(branch='trunk')), # - AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"], - change_filter=filter.ChangeFilter(branch_re='branch/.*')), - AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"], - change_filter=filter.ChangeFilter(branch_re='branch/.*')), - AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"], - change_filter=filter.ChangeFilter(branch_re='branch/.*')), + schedulers.AnyBranchScheduler(name="pyflakes_branch", treeStableTimer=10, builderNames=["[branch] Check PyFlakes"], + change_filter=util.ChangeFilter(branch_re='branch/.*')), +# schedulers.AnyBranchScheduler(name="lin_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite"], +# change_filter=util.ChangeFilter(branch_re='branch/.*')), +# schedulers.AnyBranchScheduler(name="osx_test_branch", treeStableTimer=60*5, builderNames=["[branch] Test Suite (OS X)"], +# change_filter=util.ChangeFilter(branch_re='branch/.*')), # - AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"], - change_filter=filter.ChangeFilter(branch_re='personal/.*')), - AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"], - change_filter=filter.ChangeFilter(branch_re='personal/.*')), + schedulers.AnyBranchScheduler(name="pyflakes_personal",treeStableTimer=10, builderNames=["[personal] Check PyFlakes"], + change_filter=util.ChangeFilter(branch_re='personal/.*')), + schedulers.AnyBranchScheduler(name="lin_test_personal",treeStableTimer=60*5, builderNames=["[personal] Test Suite"], + change_filter=util.ChangeFilter(branch_re='personal/.*')), # Periodic Schedulers - Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],), - Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],), - Nightly(name="crawler", hour=[8,9,10,11,12,13,14,15,16,17], minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],), + schedulers.Nightly(name="lin_test_old_libs", hour=16, minute=12, branch="trunk", builderNames=["Verify Minimum Libs"],), + schedulers.Nightly(name="lin_test_libs", hour=16, minute=42, branch="trunk", builderNames=["Verify Latest Libs"],), + schedulers.Nightly(name="crawler", hour=9, minute=00, branch="trunk", onlyIfChanged=True, builderNames=["Test-Crawler"],), - # Force schedulers - ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]), - ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]), - ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]), - ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]), + # schedulers.Force schedulers + schedulers.ForceScheduler(name="force_pyflakes", builderNames=["Check PyFlakes"]), + schedulers.ForceScheduler(name="force_lin_test", builderNames=["Test Suite"]), +# schedulers.ForceScheduler(name="force_osx_test", builderNames=["Test Suite (OS X)"]), + schedulers.ForceScheduler(name="force_test_crawler", builderNames=["Test-Crawler"]), # - ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]), - ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]), - ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]), + schedulers.ForceScheduler(name="force_pyflakes_branch", builderNames=["[branch] Check PyFlakes"]), + schedulers.ForceScheduler(name="force_lin_test_branch", builderNames=["[branch] Test Suite"]), +# schedulers.ForceScheduler(name="force_osx_test_branch", builderNames=["[branch] Test Suite (OS X)"]), # - ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]), - ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]), + schedulers.ForceScheduler(name="force_pyflakes_personal", builderNames=["[personal] Check PyFlakes"]), + schedulers.ForceScheduler(name="force_lin_test_personal", builderNames=["[personal] Test Suite"]), + ] ####### BUILDERS -# -*- section Builders -*- # The 'builders' list defines the Builders, which tell Buildbot how to perform a build: -# what steps, and which slaves can execute them. Note that any particular build will -# only take place on one slave. +# what steps, and which workers can execute them. Note that any particular build will +# only take place on one worker. -from buildbot.process.factory import BuildFactory -from buildbot.steps.source.svn import SVN -from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand -from buildbot.steps.python import PyFlakes -from buildbot.steps.python_twisted import RemovePYCs -# -from buildbot.process.properties import Property, Interpolate -from buildbot.config import BuilderConfig #### Custom subclassed builder -class TestCrawlerShellCommand(WarningCountingShellCommand): - name = "testcrawl" - haltOnFailure = 1 - flunkOnFailure = 1 - descriptionDone = ["test crawler"] - command=["bin/test-crawl"] - - warningPatterns = { - "exceptions": "^(Traceback| File| |.*Error|.*Exception)", - "failed": " FAIL ", - "warnings": " WARN", - "slow": " SLOW", - "invalid_html": " invalid html:", - } - - logline = "^ *(?P\d+:\d+:\d+) +(?P\d+) +(?P\d+) +(?P\d+) +(?P\d+.\d+)s +(?P.+)" - - def setTestResults(self, **kwargs): - """ - Called by subclasses to set the relevant statistics; this actually - adds to any statistics already present - """ - for kw in kwargs: - value = kwargs[kw] - if value.isdigit(): - # Counter - value = int(value) - value += self.step_status.getStatistic(kw, 0) - elif re.search("^[0-9]+\.[0-9]+$", value): - # Runtime - value = float(value) - value += self.step_status.getStatistic(kw, 0) - else: - # This is a percentage, and we can't add them - pass - self.step_status.setStatistic(kw, value) - - def createSummary(self, log): - """ - Match log lines against warningPattern. - - Warnings are collected into another log for this step, and the - build-wide 'warnings-count' is updated.""" - - warnings = {} - wregex = {} - - regex_class = re.compile("").__class__ - - if not isinstance(self.logline, regex_class): - self.logline = re.compile(self.logline) - - for key in self.warningPatterns: - warnings[key] = [] - pattern = self.warningPatterns[key] - if not isinstance(pattern, regex_class): - wregex[key] = re.compile(pattern) - else: - wregex[key] = pattern - - # Count matches to the various warning patterns - for line in log.getText().split("\n"): - for key in wregex: - match = re.search(wregex[key], line) - if match: - warnings[key].append(line) - if re.search(self.logline, line): - last_line = line - - # If there were any warnings, make the log if lines with warnings - # available - for key in warnings: - if len(warnings[key]) > 0: - self.addCompleteLog("%s (%d)" % (key, len(warnings[key])), - "\n".join(warnings[key]) + "\n") - self.step_status.setStatistic(key, len(warnings[key])) - self.setProperty(key, len(warnings[key]), "TestCrawlerShellCommand") - - match = re.search(self.logline, last_line) - for key in ['elapsed', 'pages']: - info = match.group(key) - self.step_status.setStatistic(key, info) - self.setProperty(key, info, "TestCrawlerShellCommand") - - def describe(self, done=False): - description = WarningCountingShellCommand.describe(self, done) - if done: - description = description[:] # make a private copy - for name in ["time", "elapsed", "pages", "failed", "warnings", "slow", "invalid_html", ]: - if name in self.step_status.statistics: - value = self.step_status.getStatistic(name) - displayName = name.replace('_', ' ') - # special case. Mph. - if type(value) is float: # this is run-time - description.append('%s: %.2fs' % (displayName, value)) - elif type(value) is int: - description.append('%s: %d' % (displayName, value)) - else: - description.append('%s: %s' % (displayName, value)) - return description - - -class UnitTest(WarningCountingShellCommand): - - name = "test" - warnOnFailure = 1 - description = ["testing"] - descriptionDone = ["test"] - command = ["python", "-m", "unittest", "discover"] - - regexPatterns = { - "tests": "Ran (\d+) tests in [0-9.]+s", - "time": "Ran \d+ tests in ([0-9.]+)s", - "skipped": "(?:OK|FAILED).*skipped=(\d+)", - "failed": "FAILED.*failures=(\d+)", - "errors": "FAILED.*errors=(\d+)", - "template_coverage":" +Template coverage: +([0-9.]+%)", - "url_coverage": " +Url coverage: +([0-9.]+%)", - "code_coverage": " +Code coverage: +([0-9.]+%)", - } - - def setTestResults(self, **kwargs): - """ - Called by subclasses to set the relevant statistics; this actually - adds to any statistics already present - """ - for kw in kwargs: - value = kwargs[kw] - if value.isdigit(): - # Counter - value = int(value) - value += self.step_status.getStatistic(kw, 0) - elif re.search("^[0-9]+\.[0-9]+$", value): - # Runtime - value = float(value) - value += self.step_status.getStatistic(kw, 0) - else: - # This is a percentage, and we can't add them - pass - self.step_status.setStatistic(kw, value) - - def createSummary(self, log): - info = {} - for line in log.getText().split("\n"): - for key in self.regexPatterns: - regex = self.regexPatterns[key] - match = re.search(regex, line) - if match: - info[key] = match.group(1) - self.setTestResults(**info) - - def describe(self, done=False): - description = WarningCountingShellCommand.describe(self, done) - if done: - description = description[:] # make a private copy - self.step_status.statistics["passed"] = ( - self.step_status.getStatistic("tests",0) - - self.step_status.getStatistic("skipped",0) - - self.step_status.getStatistic("failed",0) - - self.step_status.getStatistic("errors",0)) - for name in ["time", "tests", "passed", "skipped", "failed", "errors", "template_coverage", "url_coverage", "code_coverage", ]: - if name in self.step_status.statistics: - value = self.step_status.getStatistic(name) - displayName = name.replace('_', ' ') - # special case. Mph. - if displayName == 'template coverage': - displayName = 'templ. coverage' - if type(value) is float: # this is run-time - description.append('%s: %.2fs' % (displayName, value)) - elif type(value) is int: - description.append('%s: %d' % (displayName, value)) - else: - description.append('%s: %s' % (displayName, value)) - return description - - -## Set up builders +factory = util.BuildFactory() +# check out the source +factory.addStep(steps.Git(repourl='git://github.com/buildbot/hello-world.git', mode='incremental')) +# run the tests (note that this will require that 'trial' is installed) +factory.addStep(steps.ShellCommand(command=["trial", "hello"], + env={"PYTHONPATH": "."})) c['builders'] = [] # -*- section Builder_Run_pyflakes -*- -factory = BuildFactory() -factory.addStep(SVN( +factory = util.BuildFactory() +factory.addStep(steps.SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, - repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], + repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="seting up settings_local.py", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, - command=["cp", "/home/buildbot/settings_local.py", "./"], + command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], )) -factory.addStep(PyFlakes( - workdir=Interpolate('build/%(src::branch)s'), +factory.addStep(steps.PyFlakes( + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["ietf/manage.py", "pyflakes", "--verbosity=0"], )) # This should be the last action -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="mark as passed", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", - "propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ], + "propset", "--revprop", "-r", util.Property('got_revision'), "test:pyflakes", "passed" ], )) -c['builders'].append(BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk", - slavenames=["datatracker_lin_py27_1", ])) -c['builders'].append(BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch", - slavenames=["datatracker_lin_py27_2", ])) -c['builders'].append(BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal", - slavenames=["datatracker_lin_py27_3", ])) +c['builders'].append(util.BuilderConfig(name="Check PyFlakes", factory=factory, category="1. trunk", + workernames=["datatracker_lin_py36_1", ])) +c['builders'].append(util.BuilderConfig(name="[branch] Check PyFlakes", factory=factory, category="2. branch", + workernames=["datatracker_lin_py36_2", ])) +c['builders'].append(util.BuilderConfig(name="[personal] Check PyFlakes", factory=factory, category="3. personal", + workernames=["datatracker_lin_py36_3", ])) # -*- section Builder_TestSuite -*- -factory = BuildFactory() -factory.addStep(SVN( +factory = util.BuildFactory() +factory.addStep(steps.SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, - repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], + repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) -factory.addStep(ShellCommand( +factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) +factory.addStep(steps.ShellCommand( descriptionDone="remove tmp-* dirs", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["rm", "-rf", "tmp-*/"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="install requirements", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["pip", "install", "-r", "requirements.txt"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="copy settings_local.py", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, - command=["cp", "/home/buildbot/settings_local.py", "./"], + command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="collect static files", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=False, flunkOnFailure=False, command=["ietf/manage.py", "collectstatic", "--noinput", ], )) -factory.addStep(UnitTest( - workdir=Interpolate('build/%(src::branch)s'), +factory.addStep(custom_steps.DjangoTest( + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], )) # This should come after tests -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="mark as passed", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", - "propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ], + "propset", "--revprop", "-r", util.Property('got_revision'), "test:unittest", "passed" ], )) -c['builders'].append(BuilderConfig(name="Test Suite", factory=factory, category="1. trunk", - slavenames=["datatracker_lin_py27_1", ])) -c['builders'].append(BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch", - slavenames=["datatracker_lin_py27_2", ])) -c['builders'].append(BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal", - slavenames=["datatracker_lin_py27_3", ])) - -# -*- section Builder_TestSuiteOSX -*- - -factory = BuildFactory() -factory.addStep(ShellCommand( - descriptionDone="svn cleanup", - workdir=Interpolate('build/%(src::branch)s'), - haltOnFailure=False, - flunkOnFailure=False, - command=["svn", "cleanup"], - )) -factory.addStep(SVN( - username='buildbot@tools.ietf.org', - descriptionDone="svn update", - workdir=Interpolate('build/%(src::branch)s'), - haltOnFailure=True, - repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], - )) -factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) -factory.addStep(ShellCommand( - descriptionDone="install requirements", - workdir=Interpolate('build/%(src::branch)s'), - haltOnFailure=True, - command=["pip", "install", "-r", "requirements.txt"], - )) -factory.addStep(ShellCommand( - descriptionDone="copy settings_local.py", - workdir=Interpolate('build/%(src::branch)s'), - haltOnFailure=True, - command=["cp", "/Users/buildbot/settings_local.py", "./"], - )) -factory.addStep(ShellCommand( - descriptionDone="collect static files", - workdir=Interpolate('build/%(src::branch)s'), - haltOnFailure=False, - flunkOnFailure=False, - command=["ietf/manage.py", "collectstatic", "--noinput", ], - )) -factory.addStep(UnitTest( - workdir=Interpolate('build/%(src::branch)s'), - haltOnFailure=True, - command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--skip-coverage", "--verbosity=2", ], - )) - -c['builders'].append(BuilderConfig(name="Test Suite (OS X)", factory=factory, category="1. trunk", - slavenames=["datatracker_osx_py27_4"])) -c['builders'].append(BuilderConfig(name="[branch] Test Suite (OS X)", factory=factory, category="2. branch", - slavenames=["datatracker_osx_py27_4"])) +c['builders'].append(util.BuilderConfig(name="Test Suite", factory=factory, category="1. trunk", + workernames=["datatracker_lin_py36_1", ])) +c['builders'].append(util.BuilderConfig(name="[branch] Test Suite", factory=factory, category="2. branch", + workernames=["datatracker_lin_py36_2", ])) +c['builders'].append(util.BuilderConfig(name="[personal] Test Suite", factory=factory, category="3. personal", + workernames=["datatracker_lin_py36_3", ])) # -*- section Builder_TestCrawler -*- -factory = BuildFactory() -factory.addStep(SVN( +factory = util.BuildFactory() +factory.addStep(steps.SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, - repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], + repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) -factory.addStep(ShellCommand( +factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) +factory.addStep(steps.ShellCommand( descriptionDone="install requirements", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["pip", "install", "-r", "requirements.txt"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="copy settings_local.py", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, - command=["cp", "/home/buildbot/settings_local.py", "./"], + command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="run migrations", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["ietf/manage.py", "migrate"], )) -factory.addStep(TestCrawlerShellCommand( - workdir=Interpolate('build/%(src::branch)s'), +factory.addStep(custom_steps.TestCrawlerShellCommand( + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"], )) # This should be the last action -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="mark as passed", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), command=["svn", "--username=buildbot@tools.ietf.org", "--non-interactive", - "propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ], + "propset", "--revprop", "-r", util.Property('got_revision'), "test:crawler", "passed" ], )) -c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk", - slavenames=["datatracker_lin_py27_6", ])) +c['builders'].append(util.BuilderConfig(name="Test-Crawler", factory=factory, category="1. trunk", + workernames=["datatracker_lin_py36_6", ])) -# -*- section Builder_Verify_Old_Libs -*- +# -*- section Builder_Verify_Minimum_Libs -*- # This build runs pip install --upgrade, to make sure that we install the earliest version of # all dependencies, in order to get an indication if/when an incompatibility turns up with a new @@ -494,65 +260,65 @@ c['builders'].append(BuilderConfig(name="Test-Crawler", factory=factory, categor # change the external test conditions and produce spurious errors because of version changes in # dependencies. -factory = BuildFactory() -factory.addStep(ShellCommand( +factory = util.BuildFactory() +factory.addStep(steps.ShellCommand( descriptionDone="remove tweaked requirements", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=False, flunkOnFailure=False, command=["rm", "requirements.txt"], )) -factory.addStep(SVN( +factory.addStep(steps.SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), alwaysUseLatest=True, haltOnFailure=True, - repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], + repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) -factory.addStep(ShellCommand( +factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) +factory.addStep(steps.ShellCommand( descriptionDone="edit requirements", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["sed", "-i", "-e", "s/>=/==/", "requirements.txt"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="install/upgrade requirements", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["pip", "install", "--upgrade", "-r", "requirements.txt"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="seting up settings_local.py", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, - command=["cp", "/home/buildbot/settings_local.py", "./"], + command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="list installed pyton modules", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["pip", "freeze"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="collect static files", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=False, flunkOnFailure=False, command=["ietf/manage.py", "collectstatic", "--noinput", ], )) -factory.addStep(UnitTest( - workdir=Interpolate('build/%(src::branch)s'), +factory.addStep(custom_steps.DjangoTest( + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], )) -c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk", - slavenames=["datatracker_lin_py27_5"])) +c['builders'].append(util.BuilderConfig(name="Verify Minimum Libs", factory=factory, category="1. trunk", + workernames=["datatracker_lin_py36_5", ])) -# -*- section Builder_Dependencies -*- +# -*- section Builder_Veryfy_Latest_Libs -*- # This build runs pip install --upgrade, to make sure that we install the latest version of all # dependencies, in order to get an indication if/when an incompatibility turns up with a new @@ -560,134 +326,99 @@ c['builders'].append(BuilderConfig(name="Verify Minimum Libs", factory=factory, # change the external test conditions and produce spurious errors because of version changes in # dependencies. -factory = BuildFactory() -factory.addStep(SVN( +factory = util.BuildFactory() +factory.addStep(steps.SVN( username='buildbot@tools.ietf.org', descriptionDone="svn update", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), alwaysUseLatest=True, haltOnFailure=True, - repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), - descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], + repourl=util.Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[util.Interpolate('%(src::branch)s %(src::revision)s')], )) -factory.addStep(RemovePYCs(workdir=Interpolate('build/%(src::branch)s'))) -factory.addStep(ShellCommand( +factory.addStep(steps.RemovePYCs(workdir=util.Interpolate('build/%(src::branch)s'))) +factory.addStep(steps.ShellCommand( descriptionDone="install/upgrade requirements", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["pip", "install", "--upgrade", "-r", "requirements.txt"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="seting up settings_local.py", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, - command=["cp", "/home/buildbot/settings_local.py", "./"], + command=["cp", expandtilde("~/settings_local.py"), "./ietf/"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="list installed pyton modules", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["pip", "freeze"], )) -factory.addStep(ShellCommand( +factory.addStep(steps.ShellCommand( descriptionDone="collect static files", - workdir=Interpolate('build/%(src::branch)s'), + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=False, flunkOnFailure=False, command=["ietf/manage.py", "collectstatic", "--noinput", ], )) -factory.addStep(UnitTest( - workdir=Interpolate('build/%(src::branch)s'), +factory.addStep(custom_steps.DjangoTest( + workdir=util.Interpolate('build/%(src::branch)s'), haltOnFailure=True, command=["ietf/manage.py", "test", "--settings=settings_sqlitetest", "--verbosity=2", ], )) -c['builders'].append(BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk", - slavenames=["datatracker_lin_py27_5"])) +c['builders'].append(util.BuilderConfig(name="Verify Latest Libs", factory=factory, category="1. trunk", + workernames=["datatracker_lin_py36_5", ])) +####### BUILDBOT SERVICES -####### STATUS TARGETS -# -*- section StatusTargets -*- +# 'services' is a list of BuildbotService items like reporter targets. The +# status of each build will be pushed to these targets. buildbot/reporters/*.py +# has a variety to choose from, like IRC bots. -# 'status' is a list of Status Targets. The results of each build will be -# pushed to these targets. buildbot/status/*.py has a variety to choose from, -# including web pages, email senders, and IRC bots. - -c['status'] = [] - -from buildbot.status import html, mail -from buildbot.status.web import authz, auth - -authz_cfg=authz.Authz( - # change any of these to True to enable; see the manual for more - # options - auth=auth.BasicAuth([("ietfdb","ietfdb")]), - gracefulShutdown = False, - forceBuild = 'auth', # use this to test your slave once it is set up - forceAllBuilds = False, - pingBuilder = False, - stopBuild = 'auth', - stopAllBuilds = False, - cancelPendingBuild = 'auth', -) -c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) - -# A second web status with slightly different rendering -from twisted.python import log -def changelinkfilter(html, project): - log.msg(" * changelinkfilter(html='%s', project='%s')" % (html, project)) - return html - -import jinja2, os -trac_template_loaders = [jinja2.FileSystemLoader(os.path.join(os.getcwd(), 'trac_view'))] -c['status'].append(html.WebStatus(http_port=8011, jinja_loaders=trac_template_loaders, - authz=authz_cfg)) - -# Email notifications -from zope.interface import implements -from buildbot import interfaces - -class UsernameIsEmailAddress(): - "This IEmailLookup provider assumes that the svn username is a valid email address." - implements(interfaces.IEmailLookup) - def getAddress(self, name): - return name - -c['status'].append(mail.MailNotifier( - fromaddr='buildbot@tools.ietf.org', - sendToInterestedUsers=False, - extraRecipients=['henrik@levkowetz.com',], - mode="problem", - )) - -c['status'].append(mail.MailNotifier( - fromaddr='buildbot@tools.ietf.org', - lookup=UsernameIsEmailAddress(), - mode="problem", - )) +c['services'] = [] ####### PROJECT IDENTITY -# -*- section Project -*- -# the 'title' string will appear at the top of this buildbot -# installation's html.WebStatus home page (linked to the -# 'titleURL') and is embedded in the title of the waterfall HTML page. +# the 'title' string will appear at the top of this buildbot installation's +# home pages (linked to the 'titleURL'). -c['title'] = "IETF Datatracker" +c['title'] = "Buildbot: IETF Datatracker" c['titleURL'] = "https://datatracker.ietf.org/" # the 'buildbotURL' string should point to the location where the buildbot's -# internal web server (usually the html.WebStatus page) is visible. This -# typically uses the port number set in the Waterfall 'status' entry, but -# with an externally-visible host name which the buildbot cannot figure out -# without some help. +# internal web server is visible. This typically uses the port number set in +# the 'www' entry below, but with an externally-visible host name which the +# buildbot cannot figure out without some help. + +c['buildbotURL'] = "http://dunkelfelder.tools.ietf.org:8010/" + +# minimalistic config to activate new web UI +c['www'] = { + 'port': 8010, + 'plugins': { + 'waterfall_view': True, + 'console_view': True, + 'grid_view': True, + }, + 'default_page': 'waterfall_view', + 'debug': True, + 'auth': util.UserPasswordAuth({"ietfdb": "ietfdb"}), +} -c['buildbotURL'] = "http://zinfandel.tools.ietf.org:8010/" ####### DB URL c['db'] = { - # This specifies what database buildbot uses to store its state. You can leave - # this at its default for all but the largest installations. + # This specifies what database buildbot uses to store its state. + # It's easy to start with sqlite, but it's recommended to switch to a dedicated + # database, such as PostgreSQL or MySQL, for use in production environments. + # http://docs.buildbot.net/current/manual/configuration/global.html#database-specification 'db_url' : "sqlite:///state.sqlite", } + +####### MISC. SETTINGS + +c['buildbotNetUsageData'] = 'full' + diff --git a/buildbot/restart-masters b/buildbot/restart-masters new file mode 100755 index 000000000..be827cf40 --- /dev/null +++ b/buildbot/restart-masters @@ -0,0 +1,3 @@ +#!/bin/bash + +for d in masters/*; do ( cd $d; . env/bin/activate; buildbot restart; ); done diff --git a/buildbot/restart-workers b/buildbot/restart-workers new file mode 100755 index 000000000..4415c7d1e --- /dev/null +++ b/buildbot/restart-workers @@ -0,0 +1,3 @@ +#!/bin/bash + +for d in workers/*; do ( cd $d; . env/bin/activate; buildbot-worker restart; ); done diff --git a/buildbot/setup-buildbot-workers b/buildbot/setup-buildbot-workers new file mode 100755 index 000000000..026ce5e25 --- /dev/null +++ b/buildbot/setup-buildbot-workers @@ -0,0 +1,180 @@ +#!/bin/bash +# -*- indent-with-tabs: 0 -*- + +version=0.10 +program=${0##*/} +progdir=${0%/*} +if [ "$progdir" = "$program" ]; then progdir="."; fi + +# ---------------------------------------------------------------------- +function usage() { +# Possible sections: +# NAME +# SYNOPSIS +# CONFIGURATION [Normally only in Section 4] +# DESCRIPTION +# OPTIONS [Normally only in Sections 1, 8] +# EXIT STATUS [Normally only in Sections 1, 8] +# RETURN VALUE [Normally only in Sections 2, 3] +# ERRORS [Typically only in Sections 2, 3] +# ENVIRONMENT +# FILES +# VERSIONS [Normally only in Sections 2, 3] +# CONFORMING TO +# NOTES +# BUGS +# EXAMPLE +# SEE ALSO + + cat <. Repository: + https://svn.tools.ietf.org/svn/tools/ietfdb/trunk/buildbot + +COPYRIGHT + Copyright 2020 the IETF Trust. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, are permitted provided that the conditions + laid out in the 3-clause BSD license is followed. + + License text: https://opensource.org/licenses/BSD-3-Clause + +EOF + +} + +# ---------------------------------------------------------------------- + +function die() { + echo -e "\n$program: error: $*" >&2 + exit 1 +} + +function err() { + echo -e "${red}$*${reset}" >&2 +} + +function note() { + if [ -n "$VERBOSE" ]; then echo -e "\n$*"; fi +} + +# ---------------------------------------------------------------------- +function version() { + echo -e "$program $version" +} + +# ---------------------------------------------------------------------- +trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR + + +# ---------------------------------------------------------------------- +# Option parsing + +# Options +shortopts=a:hp:s:vV +longopts=admin:,help,server:,passwd:,python:,verbose,version + +# Default values +read -d ' ' <<< $(who -m) +user=$REPLY +name=$(getent passwd $user | cut -d ':' -f 5 | cut -d ',' -f 1) +server='zinfandel.tools.ietf.org' +pass='' # must be set on the command line +python=python3.6 + +if [ "$(uname)" = "Linux" ]; then + args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@") + if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi + files="$@" + eval set -- "$args" + sed="sed -r" +else + # Darwin, BSDs + args=$(getopt -o$shortopts $SV $*) + if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi + files="$@" + set -- $args + sed="sed -E" +fi + +while true ; do + case "$1" in + -a| --admin) admin="$2"; shift;; # "Name " of buildbot admin + -h| --help) usage; exit;; # Show this help, then exit + -p| --passwd) pass=$2; shift;; # Worker password + --python) python=$2; shift;; # Python version to use (e.g., 'python3.6') + -s| --server) server=$2; shift;; # Set the server fqdn + -v| --verbose) VERBOSE=1;; # Be more talkative + -V| --version) version; exit;; # Show program version, then exit + --) shift; break;; + *) die "Internal error, inconsistent option specification: '$1'";; + esac + shift +done + +# ---------------------------------------------------------------------- +# The program itself + +dir=$(dirname $(realpath $0)) +if [ -d "$dir/slaves" ]; then + path="$dir/slaves" +else + path="$dir/workers" +fi + +for worker in $path/*; do + ( + cd $worker; + pwd + if [ ! -d ./env ]; then + echo "Setting up virtual environment" + # Change python version to match deployment version + python3.6 -m venv env + fi + . env/bin/activate + pip install buildbot-worker + if [ ! -f ./buildbot.tac ]; then + pwfile=$dir/${worker##*/}_pw + echo "Looking for pwfile: $pwfile" + [ -f "$pwfile" ] && pass=$(< $pwfile) + [ -z "$pass" ] && die "Missing parameter: worker password" + buildbot-worker create-worker $PWD $server ${PWD##*/} $pass + fi + if ! grep -q "$name" ./info/admin; then + read -p "Expected $name in $PWD/info/admin, but found $(<./info/admin) -- change it [Y/n]?" + if [ "$REPLY" = "Y" -o "$REPLY" = "y" ]; then + if [ -z "$admin" ]; then + read -p "Admin (Your Name ): " + admin=$REPLY + fi + echo "Setting up ./info/admin" + echo "$admin" > ./info/admin + echo "Setting up ./info/host" + echo "$(uname -s -n -r) $(python --version)" > ./info/host + fi + fi + buildbot-worker stop + buildbot-worker start + ) +done \ No newline at end of file diff --git a/buildbot/workers/datatracker_lin_py36_1/info/admin b/buildbot/workers/datatracker_lin_py36_1/info/admin new file mode 100644 index 000000000..064f019af --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_1/info/admin @@ -0,0 +1 @@ +Henrik Levkowetz diff --git a/buildbot/workers/datatracker_lin_py36_1/info/host b/buildbot/workers/datatracker_lin_py36_1/info/host new file mode 100644 index 000000000..0d0764050 --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_1/info/host @@ -0,0 +1 @@ +Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9 diff --git a/buildbot/workers/datatracker_lin_py36_2/info/admin b/buildbot/workers/datatracker_lin_py36_2/info/admin new file mode 100644 index 000000000..064f019af --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_2/info/admin @@ -0,0 +1 @@ +Henrik Levkowetz diff --git a/buildbot/workers/datatracker_lin_py36_2/info/host b/buildbot/workers/datatracker_lin_py36_2/info/host new file mode 100644 index 000000000..0d0764050 --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_2/info/host @@ -0,0 +1 @@ +Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9 diff --git a/buildbot/workers/datatracker_lin_py36_3/info/admin b/buildbot/workers/datatracker_lin_py36_3/info/admin new file mode 100644 index 000000000..064f019af --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_3/info/admin @@ -0,0 +1 @@ +Henrik Levkowetz diff --git a/buildbot/workers/datatracker_lin_py36_3/info/host b/buildbot/workers/datatracker_lin_py36_3/info/host new file mode 100644 index 000000000..0d0764050 --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_3/info/host @@ -0,0 +1 @@ +Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9 diff --git a/buildbot/workers/datatracker_lin_py36_4/info/admin b/buildbot/workers/datatracker_lin_py36_4/info/admin new file mode 100644 index 000000000..064f019af --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_4/info/admin @@ -0,0 +1 @@ +Henrik Levkowetz diff --git a/buildbot/workers/datatracker_lin_py36_4/info/host b/buildbot/workers/datatracker_lin_py36_4/info/host new file mode 100644 index 000000000..0d0764050 --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_4/info/host @@ -0,0 +1 @@ +Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9 diff --git a/buildbot/workers/datatracker_lin_py36_5/info/admin b/buildbot/workers/datatracker_lin_py36_5/info/admin new file mode 100644 index 000000000..064f019af --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_5/info/admin @@ -0,0 +1 @@ +Henrik Levkowetz diff --git a/buildbot/workers/datatracker_lin_py36_5/info/host b/buildbot/workers/datatracker_lin_py36_5/info/host new file mode 100644 index 000000000..0d0764050 --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_5/info/host @@ -0,0 +1 @@ +Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9 diff --git a/buildbot/workers/datatracker_lin_py36_6/info/admin b/buildbot/workers/datatracker_lin_py36_6/info/admin new file mode 100644 index 000000000..064f019af --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_6/info/admin @@ -0,0 +1 @@ +Henrik Levkowetz diff --git a/buildbot/workers/datatracker_lin_py36_6/info/host b/buildbot/workers/datatracker_lin_py36_6/info/host new file mode 100644 index 000000000..0d0764050 --- /dev/null +++ b/buildbot/workers/datatracker_lin_py36_6/info/host @@ -0,0 +1 @@ +Linux dunkelfelder 3.16.0-4-amd64 Python 3.6.9