import re
import base64
import subprocess
-import ConfigParser
+import configparser
from buildbot import locks
+from buildbot.changes import filter
+from buildbot.changes.gitpoller import GitPoller
+from buildbot.config import BuilderConfig
+from buildbot.plugins import schedulers
+from buildbot.plugins import steps
+from buildbot.plugins import util
+from buildbot.process.factory import BuildFactory
+from buildbot.process.properties import Property
+from buildbot.process.properties import WithProperties
+from buildbot.schedulers.basic import SingleBranchScheduler
+from buildbot.schedulers.forcesched import ForceScheduler
+from buildbot.steps.master import MasterShellCommand
+from buildbot.steps.shell import SetProperty
+from buildbot.steps.shell import ShellCommand
+from buildbot.steps.transfer import FileDownload
+from buildbot.steps.transfer import FileUpload
+from buildbot.steps.transfer import StringDownload
+from buildbot.worker import Worker
+
-ini = ConfigParser.ConfigParser()
+ini = configparser.ConfigParser()
ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
-buildbot_url = ini.get("general", "buildbot_url")
+buildbot_url = ini.get("phase2", "buildbot_url")
# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.
####### BUILDSLAVES
-# The 'slaves' list defines the set of recognized buildslaves. Each element is
-# a BuildSlave object, specifying a unique slave name and password. The same
+# The 'workers' list defines the set of recognized buildslaves. Each element is
+# a Worker object, specifying a unique slave name and password. The same
# slave name and password must be configured on the slave.
-from buildbot.buildslave import BuildSlave
slave_port = 9990
persistent = False
git_ssh = False
git_ssh_key = None
-if ini.has_option("general", "port"):
- slave_port = ini.getint("general", "port")
+if ini.has_option("phase2", "port"):
+ slave_port = ini.getint("phase2", "port")
-if ini.has_option("general", "persistent"):
- persistent = ini.getboolean("general", "persistent")
+if ini.has_option("phase2", "persistent"):
+ persistent = ini.getboolean("phase2", "persistent")
-if ini.has_option("general", "other_builds"):
- other_builds = ini.getint("general", "other_builds")
+if ini.has_option("phase2", "other_builds"):
+ other_builds = ini.getint("phase2", "other_builds")
-if ini.has_option("general", "expire"):
- tree_expire = ini.getint("general", "expire")
+if ini.has_option("phase2", "expire"):
+ tree_expire = ini.getint("phase2", "expire")
if ini.has_option("general", "git_ssh"):
git_ssh = ini.getboolean("general", "git_ssh")
else:
git_ssh = False
-c['slaves'] = []
+c['workers'] = []
max_builds = dict()
for section in ini.sections():
if section.startswith("slave "):
- if ini.has_option(section, "name") and ini.has_option(section, "password"):
+ if ini.has_option(section, "name") and ini.has_option(section, "password") and \
+ ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
name = ini.get(section, "name")
password = ini.get(section, "password")
+ sl_props = { 'shared_wd': False }
max_builds[name] = 1
+
if ini.has_option(section, "builds"):
max_builds[name] = ini.getint(section, "builds")
- c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
-# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
+ if max_builds[name] == 1:
+ sl_props['shared_wd'] = True
+
+ if ini.has_option(section, "shared_wd"):
+ sl_props['shared_wd'] = ini.getboolean(section, "shared_wd")
+ if sl_props['shared_wd'] and (max_builds != 1):
+ raise ValueError('max_builds must be 1 with shared workdir!')
+
+ c['workers'].append(Worker(name, password, max_builds = max_builds[name], properties = sl_props))
+
+# 'slavePortnum' defines the TCP port to listen on for connections from workers.
# This must match the value configured into the buildslaves (with their
# --master option)
-c['slavePortnum'] = slave_port
+c['protocols'] = {'pb': {'port': slave_port}}
# coalesce builds
-c['mergeRequests'] = True
+c['collapseRequests'] = True
# Reduce amount of backlog data
c['buildHorizon'] = 30
line = findarches.stdout.readline()
if not line:
break
- at = line.strip().split()
+ at = line.decode().strip().split()
arches.append(at)
archnames.append(at[0])
feeds = []
feedbranches = dict()
-from buildbot.changes.gitpoller import GitPoller
c['change_source'] = []
def parse_feed_entry(line):
# Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'basebuild' build
-def branch_change_filter(change):
- return change.branch == feedbranches[change.repository]
-
-from buildbot.schedulers.basic import SingleBranchScheduler
-from buildbot.schedulers.forcesched import ForceScheduler
-from buildbot.changes import filter
c['schedulers'] = []
c['schedulers'].append(SingleBranchScheduler(
- name="all",
- change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
- treeStableTimer=60,
- builderNames=archnames))
+ name = "all",
+ change_filter = filter.ChangeFilter(
+ filter_fn = lambda change: change.branch == feedbranches[change.repository]
+ ),
+ treeStableTimer = 60,
+ builderNames = archnames))
c['schedulers'].append(ForceScheduler(
- name="force",
- builderNames=archnames))
+ name = "force",
+ buttonName = "Force builds",
+ label = "Force build details",
+ builderNames = [ "00_force_build" ],
+
+ codebases = [
+ util.CodebaseParameter(
+ "",
+ label = "Repository",
+ branch = util.FixedParameter(name = "branch", default = ""),
+ revision = util.FixedParameter(name = "revision", default = ""),
+ repository = util.FixedParameter(name = "repository", default = ""),
+ project = util.FixedParameter(name = "project", default = "")
+ )
+ ],
+
+ reason = util.StringParameter(
+ name = "reason",
+ label = "Reason",
+ default = "Trigger build",
+ required = True,
+ size = 80
+ ),
+
+ properties = [
+ util.NestedParameter(
+ name="options",
+ label="Build Options",
+ layout="vertical",
+ fields=[
+ util.ChoiceStringParameter(
+ name = "architecture",
+ label = "Build architecture",
+ default = "all",
+ choices = [ "all" ] + archnames
+ )
+ ]
+ )
+ ]
+))
####### BUILDERS
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
-# what steps, and which slaves can execute them. Note that any particular build will
+# what steps, and which workers can execute them. Note that any particular build will
# only take place on one slave.
-from buildbot.process.factory import BuildFactory
-from buildbot.steps.source import Git
-from buildbot.steps.shell import ShellCommand
-from buildbot.steps.shell import SetProperty
-from buildbot.steps.transfer import FileUpload
-from buildbot.steps.transfer import FileDownload
-from buildbot.steps.transfer import StringDownload
-from buildbot.steps.master import MasterShellCommand
-from buildbot.process.properties import WithProperties
-
-
def GetDirectorySuffix(props):
- verpat = re.compile('^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
+ verpat = re.compile(r'^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
if props.hasProperty("release_version"):
m = verpat.match(props["release_version"])
if m is not None:
return ""
def GetNumJobs(props):
- if props.hasProperty("slavename") and props.hasProperty("nproc"):
- return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
+ if props.hasProperty("workername") and props.hasProperty("nproc"):
+ return ((int(props["nproc"]) / (max_builds[props["workername"]] + other_builds)) + 1)
else:
return 1
else:
return "/"
+def IsArchitectureSelected(target):
+ def CheckArchitectureProperty(step):
+ try:
+ options = step.getProperty("options")
+ if type(options) is dict:
+ selected_arch = options.get("architecture", "all")
+ if selected_arch != "all" and selected_arch != target:
+ return False
+ except KeyError:
+ pass
+
+ return True
+
+ return CheckArchitectureProperty
+
def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
try:
seckey = base64.b64decode(seckey)
return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
+def IsSharedWorkdir(step):
+ return bool(step.getProperty("shared_wd"))
+
c['builders'] = []
-dlLock = locks.SlaveLock("slave_dl")
+dlLock = locks.WorkerLock("slave_dl")
slaveNames = [ ]
-for slave in c['slaves']:
- slaveNames.append(slave.slavename)
+for slave in c['workers']:
+ slaveNames.append(slave.workername)
+
+force_factory = BuildFactory()
+
+c['builders'].append(BuilderConfig(
+ name = "00_force_build",
+ workernames = slaveNames,
+ factory = force_factory))
for arch in arches:
ts = arch[1].split('/')
factory = BuildFactory()
+ # setup shared work directory if required
+ factory.addStep(ShellCommand(
+ name = "sharedwd",
+ description = "Setting up shared work directory",
+ command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
+ workdir = ".",
+ haltOnFailure = True,
+ doStepIf = IsSharedWorkdir))
+
# find number of cores
factory.addStep(SetProperty(
name = "nproc",
# prepare workspace
factory.addStep(FileDownload(
- mastersrc = scripts_dir + '/cleanup-phase2.sh',
- slavedest = "cleanup.sh",
- mode = 0755))
+ mastersrc = scripts_dir + '/cleanup.sh',
+ workerdest = "../cleanup.sh",
+ mode = 0o755))
if not persistent:
factory.addStep(ShellCommand(
name = "cleanold",
description = "Cleaning previous builds",
- command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
+ command = ["./cleanup.sh", buildbot_url, WithProperties("%(workername)s"), WithProperties("%(buildername)s"), "full"],
+ workdir = ".",
haltOnFailure = True,
timeout = 2400))
factory.addStep(ShellCommand(
name = "cleanup",
description = "Cleaning work area",
- command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
+ command = ["./cleanup.sh", buildbot_url, WithProperties("%(workername)s"), WithProperties("%(buildername)s"), "single"],
+ workdir = ".",
haltOnFailure = True,
timeout = 2400))
elif tree_expire > 0:
factory.addStep(FileDownload(
mastersrc = scripts_dir + '/expire.sh',
- slavedest = "../expire.sh",
- mode = 0755))
+ workerdest = "../expire.sh",
+ mode = 0o755))
factory.addStep(ShellCommand(
name = "expire",
command = "rsync --checksum -av sdk_update/ sdk/ && rm -rf sdk_update",
haltOnFailure = True))
+ factory.addStep(ShellCommand(
+ name = "cleancmdlinks",
+ description = "Sanitizing host command symlinks",
+ command = "find sdk/staging_dir/host/bin/ -type l -exec sh -c 'case $(readlink {}) in /bin/*|/usr/bin/*) true;; /*) rm -vf {};; esac' \\;",
+ haltOnFailure = True))
+
factory.addStep(StringDownload(
name = "writeversionmk",
s = 'TOPDIR:=${CURDIR}\n\ninclude $(TOPDIR)/include/version.mk\n\nversion:\n\t@echo $(VERSION_NUMBER)\n',
- slavedest = "sdk/getversion.mk",
- mode = 0755))
+ workerdest = "sdk/getversion.mk",
+ mode = 0o755))
factory.addStep(SetProperty(
name = "getversion",
factory.addStep(StringDownload(
name = "dlkeybuildpub",
s = UsignSec2Pub(usign_key, usign_comment),
- slavedest = "sdk/key-build.pub",
- mode = 0600))
+ workerdest = "sdk/key-build.pub",
+ mode = 0o600))
factory.addStep(StringDownload(
name = "dlkeybuild",
s = "# fake private key",
- slavedest = "sdk/key-build",
- mode = 0600))
+ workerdest = "sdk/key-build",
+ mode = 0o600))
factory.addStep(StringDownload(
name = "dlkeybuilducert",
s = "# fake certificate",
- slavedest = "sdk/key-build.ucert",
- mode = 0600))
+ workerdest = "sdk/key-build.ucert",
+ mode = 0o600))
factory.addStep(ShellCommand(
name = "mkdldir",
factory.addStep(FileDownload(
mastersrc = scripts_dir + '/ccache.sh',
- slavedest = 'sdk/ccache.sh',
- mode = 0755))
+ workerdest = 'sdk/ccache.sh',
+ mode = 0o755))
factory.addStep(ShellCommand(
name = "prepccache",
factory.addStep(StringDownload(
name = "dlgitclonekey",
s = git_ssh_key,
- slavedest = "../git-clone.key",
- mode = 0600))
+ workerdest = "../git-clone.key",
+ mode = 0o600))
factory.addStep(ShellCommand(
name = "patchfeedsconf",
description = "Building packages",
workdir = "build/sdk",
timeout = 3600,
- command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y"],
+ command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="],
env = {'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)},
haltOnFailure = True))
))
factory.addStep(FileUpload(
- slavesrc = "sdk/sign.tar.gz",
+ workersrc = "sdk/sign.tar.gz",
masterdest = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
haltOnFailure = True
))
factory.addStep(FileDownload(
mastersrc = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
- slavedest = "sdk/sign.tar.gz",
+ workerdest = "sdk/sign.tar.gz",
haltOnFailure = True
))
description = "Uploading source archives",
workdir = "build/sdk",
command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates",
- WithProperties("--partial-dir=.~tmp~%s~%%(slavename)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
+ WithProperties("--partial-dir=.~tmp~%s~%%(workername)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
env={'RSYNC_PASSWORD': rsync_src_key},
haltOnFailure = False,
logEnviron = False
alwaysRun = True
))
- from buildbot.config import BuilderConfig
-
- c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
+ c['builders'].append(BuilderConfig(name=arch[0], workernames=slaveNames, factory=factory))
+ c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % arch[0], builderNames=[ arch[0] ]))
+ force_factory.addStep(steps.Trigger(
+ name = "trigger_%s" % arch[0],
+ description = "Triggering %s build" % arch[0],
+ schedulerNames = [ "trigger_%s" % arch[0] ],
+ set_properties = { "reason": Property("reason") },
+ doStepIf = IsArchitectureSelected(arch[0])
+ ))
####### STATUS arches
# pushed to these arches. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.
-c['status'] = []
-
-from buildbot.status import html
-from buildbot.status.web import authz, auth
-
-if ini.has_option("status", "bind"):
- if ini.has_option("status", "user") and ini.has_option("status", "password"):
- authz_cfg=authz.Authz(
- # change any of these to True to enable; see the manual for more
- # options
- auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
- gracefulShutdown = 'auth',
- forceBuild = 'auth', # use this to test your slave once it is set up
- forceAllBuilds = 'auth',
- pingBuilder = False,
- stopBuild = 'auth',
- stopAllBuilds = 'auth',
- cancelPendingBuild = 'auth',
+if ini.has_option("phase2", "status_bind"):
+ c['www'] = {
+ 'port': ini.get("phase2", "status_bind"),
+ 'plugins': {
+ 'waterfall_view': True,
+ 'console_view': True,
+ 'grid_view': True
+ }
+ }
+
+ if ini.has_option("phase2", "status_user") and ini.has_option("phase2", "status_password"):
+ c['www']['auth'] = util.UserPasswordAuth([
+ (ini.get("phase2", "status_user"), ini.get("phase2", "status_password"))
+ ])
+ c['www']['authz'] = util.Authz(
+ allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
+ roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[ini.get("phase2", "status_user")]) ]
)
- c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
- else:
- c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
####### PROJECT IDENTITY
# this at its default for all but the largest installations.
'db_url' : "sqlite:///state.sqlite",
}
+
+c['buildbotNetUsageData'] = None
projects / buildbot.git / blobdiff