2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
43 c['slavePortnum'] = 9989
46 c['mergeRequests'] = True
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
53 if ini.has_option("general", "expire"):
54 tree_expire = ini.getint("general", "expire")
56 repo_url = ini.get("repo", "url")
58 rsync_bin_url = ini.get("rsync", "binary_url")
59 rsync_bin_key = ini.get("rsync", "binary_password")
64 if ini.has_option("rsync", "source_url"):
65 rsync_src_url = ini.get("rsync", "source_url")
66 rsync_src_key = ini.get("rsync", "source_password")
69 gpg_comment = "Unattended build signature"
70 gpg_passfile = "/dev/null"
72 if ini.has_option("gpg", "keyid"):
73 gpg_keyid = ini.get("gpg", "keyid")
75 if ini.has_option("gpg", "comment"):
76 gpg_comment = ini.get("gpg", "comment")
78 if ini.has_option("gpg", "passfile"):
79 gpg_passfile = ini.get("gpg", "passfile")
85 if not os.path.isdir(home_dir+'/source.git'):
86 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
88 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
89 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
92 line = findtargets.stdout.readline()
95 ta = line.strip().split(' ')
99 # the 'change_source' setting tells the buildmaster how it should find out
100 # about source code changes. Here we point to the buildbot clone of pyflakes.
102 from buildbot.changes.gitpoller import GitPoller
103 c['change_source'] = []
104 c['change_source'].append(GitPoller(
106 workdir=home_dir+'/source.git', branch='master',
111 # Configure the Schedulers, which decide how to react to incoming changes. In this
112 # case, just kick off a 'basebuild' build
114 from buildbot.schedulers.basic import SingleBranchScheduler
115 from buildbot.schedulers.forcesched import ForceScheduler
116 from buildbot.changes import filter
118 c['schedulers'].append(SingleBranchScheduler(
120 change_filter=filter.ChangeFilter(branch='master'),
122 builderNames=targets))
124 c['schedulers'].append(ForceScheduler(
126 builderNames=targets))
130 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
131 # what steps, and which slaves can execute them. Note that any particular build will
132 # only take place on one slave.
134 from buildbot.process.factory import BuildFactory
135 from buildbot.steps.source import Git
136 from buildbot.steps.shell import ShellCommand
137 from buildbot.steps.shell import SetProperty
138 from buildbot.steps.transfer import FileUpload
139 from buildbot.steps.transfer import FileDownload
140 from buildbot.steps.master import MasterShellCommand
141 from buildbot.process.properties import WithProperties
145 "^tools/": "tools/clean",
146 "^toolchain/": "toolchain/clean",
147 "^target/linux/": "target/linux/clean",
148 "^(config|include)/": "dirclean"
151 def IsAffected(pattern):
152 def CheckAffected(change):
153 for request in change.build.requests:
154 for source in request.sources:
155 for change in source.changes:
156 for file in change.files:
157 if re.match(pattern, file):
165 dlLock = locks.SlaveLock("slave_dl")
167 checkBuiltin = re.sub('[\t\n ]+', ' ', """
169 local symbol op path file;
170 for file in $CHANGED_FILES; do
176 while read symbol op path; do
177 case "$symbol" in package-*)
178 symbol="${symbol##*(}";
179 symbol="${symbol%)}";
180 for file in $CHANGED_FILES; do
181 case "$file" in "package/$path/"*)
182 grep -qsx "$symbol=y" .config && return 0
186 done < tmp/.packagedeps;
192 class IfBuiltinShellCommand(ShellCommand):
193 def _quote(self, str):
194 if re.search("[^a-zA-Z0-9/_.-]", str):
195 return "'%s'" %(re.sub("'", "'\"'\"'", str))
198 def setCommand(self, command):
199 if not isinstance(command, (str, unicode)):
200 command = ' '.join(map(self._quote, command))
203 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
206 def setupEnvironment(self, cmd):
207 slaveEnv = self.slaveEnvironment
211 for request in self.build.requests:
212 for source in request.sources:
213 for change in source.changes:
214 for file in change.files:
215 changedFiles[file] = True
216 fullSlaveEnv = slaveEnv.copy()
217 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
218 cmd.args['env'] = fullSlaveEnv
222 for slave in c['slaves']:
223 slaveNames.append(slave.slavename)
225 for target in targets:
226 ts = target.split('/')
228 factory = BuildFactory()
230 # find number of cores
231 factory.addStep(SetProperty(
234 description = "Finding number of CPUs",
235 command = ["nproc"]))
237 # expire tree if needed
239 factory.addStep(FileDownload(
240 mastersrc = "expire.sh",
241 slavedest = "../expire.sh",
244 factory.addStep(ShellCommand(
246 description = "Checking for build tree expiry",
247 command = ["./expire.sh", str(tree_expire)],
249 haltOnFailure = True,
252 # check out the source
253 factory.addStep(Git(repourl=repo_url, mode='update'))
255 factory.addStep(ShellCommand(
257 description = "Remove tmp folder",
258 command=["rm", "-rf", "tmp/"]))
261 # factory.addStep(ShellCommand(
262 # name = "feedsconf",
263 # description = "Copy the feeds.conf",
264 # command='''cp ~/feeds.conf ./feeds.conf''' ))
267 factory.addStep(ShellCommand(
268 name = "rmfeedlinks",
269 description = "Remove feed symlinks",
270 command=["rm", "-rf", "package/feeds/"]))
273 factory.addStep(ShellCommand(
274 name = "updatefeeds",
275 description = "Updating feeds",
276 command=["./scripts/feeds", "update"]))
279 factory.addStep(ShellCommand(
280 name = "installfeeds",
281 description = "Installing feeds",
282 command=["./scripts/feeds", "install", "-a"]))
285 factory.addStep(ShellCommand(
287 description = "Seeding .config",
288 command='''cat <<EOT > .config
290 CONFIG_TARGET_%s_%s=y
291 CONFIG_ALL_NONSHARED=y
294 # CONFIG_IB_STANDALONE is not set
297 CONFIG_SIGNED_PACKAGES=y
298 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
299 CONFIG_KERNEL_KALLSYMS=y
300 CONFIG_COLLECT_KERNEL_DEBUG=y
301 CONFIG_TARGET_ALL_PROFILES=y
302 CONFIG_TARGET_MULTI_PROFILE=y
303 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
304 EOT''' %(ts[0], ts[0], ts[1]) ))
306 factory.addStep(ShellCommand(
308 description = "Removing output directory",
309 command = ["rm", "-rf", "bin/"]
312 factory.addStep(ShellCommand(
314 description = "Populating .config",
315 command = ["make", "defconfig"]
319 factory.addStep(ShellCommand(
321 description = "Checking architecture",
322 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
330 factory.addStep(SetProperty(
333 description = "Finding libc suffix",
334 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
337 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
338 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
341 factory.addStep(ShellCommand(
343 description = "Preparing dl/",
344 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
350 factory.addStep(ShellCommand(
352 description = "Populating dl/",
353 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
355 locks = [dlLock.access('exclusive')]
358 factory.addStep(ShellCommand(
360 description = "Cleaning base-files",
361 command=["make", "package/base-files/clean", "V=s"]
364 # optional clean steps
365 for pattern, maketarget in MakeTargetMap.items():
366 factory.addStep(ShellCommand(
368 description = maketarget,
369 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
373 factory.addStep(ShellCommand(
375 description = "Building tools",
376 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
380 factory.addStep(ShellCommand(
382 description = "Building toolchain",
383 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
387 factory.addStep(ShellCommand(
389 description = "Building kmods",
390 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
391 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
395 factory.addStep(ShellCommand(
397 description = "Building packages",
398 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
399 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
403 # factory.addStep(IfBuiltinShellCommand(
404 factory.addStep(ShellCommand(
406 description = "Installing packages",
407 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
411 factory.addStep(ShellCommand(
413 description = "Indexing packages",
414 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
418 #factory.addStep(IfBuiltinShellCommand(
419 factory.addStep(ShellCommand(
421 description = "Building images",
422 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
426 factory.addStep(ShellCommand(
428 description = "Calculating checksums",
429 command=["make", "-j1", "checksum", "V=s"],
434 if gpg_keyid is not None:
435 factory.addStep(MasterShellCommand(
436 name = "signprepare",
437 description = "Preparing temporary signing directory",
438 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
442 factory.addStep(ShellCommand(
444 description = "Packing files to sign",
445 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
449 factory.addStep(FileUpload(
450 slavesrc = "sign.tar.gz",
451 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
455 factory.addStep(MasterShellCommand(
457 description = "Signing files",
458 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
462 factory.addStep(FileDownload(
463 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
464 slavedest = "sign.tar.gz",
468 factory.addStep(ShellCommand(
470 description = "Unpacking signed files",
471 command = ["tar", "-xzf", "sign.tar.gz"],
476 factory.addStep(ShellCommand(
477 name = "uploadprepare",
478 description = "Preparing target directory",
479 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
480 env={'RSYNC_PASSWORD': rsync_bin_key},
481 haltOnFailure = True,
485 factory.addStep(ShellCommand(
486 name = "targetupload",
487 description = "Uploading target files",
488 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
489 env={'RSYNC_PASSWORD': rsync_bin_key},
490 haltOnFailure = True,
494 if rsync_src_url is not None:
495 factory.addStep(ShellCommand(
496 name = "sourceupload",
497 description = "Uploading source archives",
498 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
499 env={'RSYNC_PASSWORD': rsync_src_key},
500 haltOnFailure = True,
505 factory.addStep(ShellCommand(
506 name = "packageupload",
507 description = "Uploading package files",
508 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
509 env={'RSYNC_PASSWORD': rsync_bin_key},
510 haltOnFailure = False,
516 factory.addStep(ShellCommand(
518 description = "Uploading logs",
519 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
520 env={'RSYNC_PASSWORD': rsync_bin_key},
521 haltOnFailure = False,
526 from buildbot.config import BuilderConfig
528 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
531 ####### STATUS TARGETS
533 # 'status' is a list of Status Targets. The results of each build will be
534 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
535 # including web pages, email senders, and IRC bots.
539 from buildbot.status import html
540 from buildbot.status.web import authz, auth
542 if ini.has_option("status", "bind"):
543 if ini.has_option("status", "user") and ini.has_option("status", "password"):
544 authz_cfg=authz.Authz(
545 # change any of these to True to enable; see the manual for more
547 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
548 gracefulShutdown = 'auth',
549 forceBuild = 'auth', # use this to test your slave once it is set up
550 forceAllBuilds = 'auth',
553 stopAllBuilds = 'auth',
554 cancelPendingBuild = 'auth',
556 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
558 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
561 from buildbot.status import words
563 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
564 irc_host = ini.get("irc", "host")
566 irc_chan = ini.get("irc", "channel")
567 irc_nick = ini.get("irc", "nickname")
570 if ini.has_option("irc", "port"):
571 irc_port = ini.getint("irc", "port")
573 if ini.has_option("irc", "password"):
574 irc_pass = ini.get("irc", "password")
576 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
577 channels = [{ "channel": irc_chan }],
580 'successToFailure': 1,
581 'failureToSuccess': 1
585 c['status'].append(irc)
588 ####### PROJECT IDENTITY
590 # the 'title' string will appear at the top of this buildbot
591 # installation's html.WebStatus home page (linked to the
592 # 'titleURL') and is embedded in the title of the waterfall HTML page.
594 c['title'] = ini.get("general", "title")
595 c['titleURL'] = ini.get("general", "title_url")
597 # the 'buildbotURL' string should point to the location where the buildbot's
598 # internal web server (usually the html.WebStatus page) is visible. This
599 # typically uses the port number set in the Waterfall 'status' entry, but
600 # with an externally-visible host name which the buildbot cannot figure out
603 c['buildbotURL'] = ini.get("general", "buildbot_url")
608 # This specifies what database buildbot uses to store its state. You can leave
609 # this at its default for all but the largest installations.
610 'db_url' : "sqlite:///state.sqlite",