From cedb1150389727a8c780f5b4d06b99277abf1384 Mon Sep 17 00:00:00 2001 From: buildbot Date: Tue, 17 Mar 2015 16:15:04 +0000 Subject: [PATCH] Added exception capture and error message when the --release switch to coverage_changes specify a release for which there aren't any data. Also tweaked the diff output for code coverage changes. - Legacy-Id: 9239 --- buildbot/masters/datatracker/master.cfg | 206 ++++++++++++++++++ .../management/commands/coverage_changes.py | 11 +- 2 files changed, 212 insertions(+), 5 deletions(-) create mode 100644 buildbot/masters/datatracker/master.cfg diff --git a/buildbot/masters/datatracker/master.cfg b/buildbot/masters/datatracker/master.cfg new file mode 100644 index 000000000..72b2f5e25 --- /dev/null +++ b/buildbot/masters/datatracker/master.cfg @@ -0,0 +1,206 @@ +# -*- python -*- +# ex: set syntax=python: + +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory. + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +####### BUILDSLAVES + +# The 'slaves' list defines the set of recognized buildslaves. Each element is +# a BuildSlave object, specifying a unique slave name and password. The same +# slave name and password must be configured on the slave. +from buildbot.buildslave import BuildSlave +c['slaves'] = [ + BuildSlave("datatracker_py27_1", "yWQaUzdJk8oL"), + BuildSlave("datatracker_py27_2", "nm+=+knVGycN"), + BuildSlave("datatracker_py27_3", "WZgYahJ4twXR"), +] + +# 'protocols' contains information about protocols which master will use for +# communicating with slaves. +# You must define at least 'port' option that slaves could connect to your master +# with this protocol. +# 'port' must match the value configured into the buildslaves (with their +# --master option) +c['protocols'] = {'pb': {'host':'zinfandel.tools.ietf.org', 'port': 9989}} + +####### CHANGESOURCES + +# the 'change_source' setting tells the buildmaster how it should find out +# about source code changes. + +from buildbot.changes.pb import PBChangeSource +# c['change_source'] = [] +# with open("users") as file: +# userinfo = json.read(file) +# for user in userinfo: +# prefix = userinfo[user]["prefix"] +# c.['change_source'].append(PBChangeSource(user=user, passwd="BRiR6XcT7x3$", prefix=prefix)) +c['change_source'] = [ + PBChangeSource(user="ietfdb", passwd="BRiR6XcT7x3$"), +] + +####### SCHEDULERS + +# Configure the Schedulers, which decide how to react to incoming changes. In this +# case, just kick off a 'runtests' build + +from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler +from buildbot.schedulers.forcesched import ForceScheduler +from buildbot.changes import filter +c['schedulers'] = [ + AnyBranchScheduler(name="pyflakes", treeStableTimer=10, builderNames=["run_pyflakes"]), + AnyBranchScheduler(name="test", treeStableTimer=60*10, builderNames=["run_tests"]), + SingleBranchScheduler(name="crawler", treeStableTimer=60*60*4, builderNames=["run_crawler"], + change_filter=filter.ChangeFilter(branch='trunk')), +] + +####### BUILDERS + +# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: +# what steps, and which slaves can execute them. Note that any particular build will +# only take place on one slave. + +from buildbot.process.factory import BuildFactory +from buildbot.steps.source.svn import SVN +from buildbot.steps.shell import ShellCommand +from buildbot.steps.python import PyFlakes +from buildbot.process.properties import Property, Interpolate +from buildbot.config import BuilderConfig + + +c['builders'] = [] + +# Run pyflakes +factory = BuildFactory() +factory.addStep(SVN( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], + )) +factory.addStep(ShellCommand( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + command=["pip", "install", "-r", "requirements.txt"])) +factory.addStep(PyFlakes( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + command=["ietf/manage.py", "pyflakes", "--verbosity=0"])) +# This should be the last action +factory.addStep(ShellCommand(descriptionDone="mark as passed", + workdir=Interpolate('build/%(src::branch)s'), + command=["svn", "--username=buildbot@tools.ietf.org", "--password=ax+u#ikxabx7", + "--no-auth-cache", "--non-interactive", + "propset", "--revprop", "-r", Property('got_revision'), "test:pyflakes", "passed" ])) + +c['builders'].append(BuilderConfig(name="run_pyflakes", factory=factory, + slavenames=["datatracker_py27_1", "datatracker_py27_2", "datatracker_py27_3"])) + +# Run tests +factory = BuildFactory() +factory.addStep(SVN( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'), + descriptionSuffix=[Interpolate('%(src::branch)s %(src::revision)s')], + )) +factory.addStep(ShellCommand( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + command=["pip", "install", "-r", "requirements.txt"])) +factory.addStep(ShellCommand( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + command=["ietf/manage.py", "test", "--settings=settings_sqlitetest"])) +# This should be the last action +factory.addStep(ShellCommand(descriptionDone="mark as passed", + workdir=Interpolate('build/%(src::branch)s'), + command=["svn", "--username=buildbot@tools.ietf.org", "--password=ax+u#ikxabx7", + "--no-auth-cache", "--non-interactive", + "propset", "--revprop", "-r", Property('got_revision'), "test:unittest", "passed" ])) + +c['builders'].append(BuilderConfig(name="run_tests", factory=factory, + slavenames=["datatracker_py27_1", "datatracker_py27_2", "datatracker_py27_3"])) + +# Run test-crawler +factory = BuildFactory() +factory.addStep(SVN( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + repourl=Interpolate('https://svn.tools.ietf.org/svn/tools/ietfdb/%(src::branch:~trunk)s'))) +factory.addStep(ShellCommand( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + command=["pip", "install", "-r", "requirements.txt"])) +factory.addStep(ShellCommand( + workdir=Interpolate('build/%(src::branch)s'), haltOnFailure=True, + command=["bin/test-crawl", "--settings=ietf.settings_testcrawl"])) +# This should be the last action +factory.addStep(ShellCommand(descriptionDone="mark as passed", + workdir=Interpolate('build/%(src::branch)s'), + command=["svn", "--username=buildbot@tools.ietf.org", "--password=ax+u#ikxabx7", + "--no-auth-cache", "--non-interactive", + "propset", "--revprop", "-r", Property('got_revision'), "test:crawler", "passed" ])) + +c['builders'].append(BuilderConfig(name="run_crawler", factory=factory, + slavenames=["datatracker_py27_1"])) + + +####### STATUS TARGETS + +# 'status' is a list of Status Targets. The results of each build will be +# pushed to these targets. buildbot/status/*.py has a variety to choose from, +# including web pages, email senders, and IRC bots. + +c['status'] = [] + +from buildbot.status import html +from buildbot.status.web import authz, auth + +authz_cfg=authz.Authz( + # change any of these to True to enable; see the manual for more + # options + auth.BasicAuth([("ietfdb","ietfdb")]), + gracefulShutdown = False, + forceBuild = 'auth', # use this to test your slave once it is set up + forceAllBuilds = False, + pingBuilder = False, + stopBuild = False, + stopAllBuilds = False, + cancelPendingBuild = False, +) +c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) + +# A second web status with slightly different rendering +from twisted.python import log +def changelinkfilter(html, project): + log.msg(" * changelinkfilter(html='%s', project='%s')" % (html, project)) + return html + +import jinja2, os +trac_template_loaders = [jinja2.FileSystemLoader(os.path.join(os.getcwd(), 'trac_view'))] +c['status'].append(html.WebStatus(http_port=8011, jinja_loaders=trac_template_loaders, + authz=authz_cfg)) + + +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot +# installation's html.WebStatus home page (linked to the +# 'titleURL') and is embedded in the title of the waterfall HTML page. + +c['title'] = "IETF Datatracker" +c['titleURL'] = "https://datatracker.ietf.org/" + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server (usually the html.WebStatus page) is visible. This +# typically uses the port number set in the Waterfall 'status' entry, but +# with an externally-visible host name which the buildbot cannot figure out +# without some help. + +c['buildbotURL'] = "http://zinfandel.tools.ietf.org:8010/" + +####### DB URL + +c['db'] = { + # This specifies what database buildbot uses to store its state. You can leave + # this at its default for all but the largest installations. + 'db_url' : "sqlite:///state.sqlite", +} diff --git a/ietf/utils/management/commands/coverage_changes.py b/ietf/utils/management/commands/coverage_changes.py index cbec9aa7a..2da9e350b 100644 --- a/ietf/utils/management/commands/coverage_changes.py +++ b/ietf/utils/management/commands/coverage_changes.py @@ -32,9 +32,10 @@ class Command(BaseCommand): try: data = json.load(file) except ValueError as e: - self.stderr.write("Failure to read json data from %s: %s" % (filename, e)) - exit(1) + raise CommandError("Failure to read json data from %s: %s" % (filename, e)) version = version or data["version"] + if not version in data: + raise CommandError("There is no data for version %s available in %s" % (version, filename)) return data[version], version def coverage_diff(self, master, latest, sections=','.join(valid_sections), release=None, **options): @@ -60,12 +61,12 @@ class Command(BaseCommand): if not key in mcoverage: mcoverage[key] = None if type(mcoverage[key]) is float or type(lcoverage[key]) is float: - mval = ("%8.2f" % mcoverage[key]) if mcoverage[key] else "-" - lval = ("%8.2f" % lcoverage[key]) if lcoverage[key] else "-" + mval = ("%5.1f" % (100*mcoverage[key])) if mcoverage[key] else "-" + lval = ("%5.1f %%" % (100*lcoverage[key])) if lcoverage[key] else "- " else: mval = mcoverage[key] lval = lcoverage[key] - if mval != lval: + if mcoverage[key] != lcoverage[key]: if not header_written: self.stdout.write(self.diff_line_format % ("\n%s"%section.capitalize(), mversion[:7], lversion[:7]))