2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
43 c['slavePortnum'] = 9989
46 c['mergeRequests'] = True
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
53 if ini.has_option("general", "expire"):
54 tree_expire = ini.getint("general", "expire")
56 repo_url = ini.get("repo", "url")
58 rsync_bin_url = ini.get("rsync", "binary_url")
59 rsync_bin_key = ini.get("rsync", "binary_password")
64 if ini.has_option("rsync", "source_url"):
65 rsync_src_url = ini.get("rsync", "source_url")
66 rsync_src_key = ini.get("rsync", "source_password")
69 gpg_comment = "Unattended build signature"
70 gpg_passfile = "/dev/null"
72 if ini.has_option("gpg", "keyid"):
73 gpg_keyid = ini.get("gpg", "keyid")
75 if ini.has_option("gpg", "comment"):
76 gpg_comment = ini.get("gpg", "comment")
78 if ini.has_option("gpg", "passfile"):
79 gpg_passfile = ini.get("gpg", "passfile")
85 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
86 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
89 line = findtargets.stdout.readline()
92 ta = line.strip().split(' ')
96 # the 'change_source' setting tells the buildmaster how it should find out
97 # about source code changes. Here we point to the buildbot clone of pyflakes.
99 from buildbot.changes.gitpoller import GitPoller
100 c['change_source'] = []
101 c['change_source'].append(GitPoller(
103 workdir=home_dir+'/source.git', branch='master',
108 # Configure the Schedulers, which decide how to react to incoming changes. In this
109 # case, just kick off a 'basebuild' build
111 from buildbot.schedulers.basic import SingleBranchScheduler
112 from buildbot.schedulers.forcesched import ForceScheduler
113 from buildbot.changes import filter
115 c['schedulers'].append(SingleBranchScheduler(
117 change_filter=filter.ChangeFilter(branch='master'),
119 builderNames=targets))
121 c['schedulers'].append(ForceScheduler(
123 builderNames=targets))
127 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
128 # what steps, and which slaves can execute them. Note that any particular build will
129 # only take place on one slave.
131 from buildbot.process.factory import BuildFactory
132 from buildbot.steps.source import Git
133 from buildbot.steps.shell import ShellCommand
134 from buildbot.steps.shell import SetProperty
135 from buildbot.steps.transfer import FileUpload
136 from buildbot.steps.transfer import FileDownload
137 from buildbot.steps.master import MasterShellCommand
138 from buildbot.process.properties import WithProperties
142 "^tools/": "tools/clean",
143 "^toolchain/": "toolchain/clean",
144 "^target/linux/": "target/linux/clean",
145 "^(config|include)/": "dirclean"
148 def IsAffected(pattern):
149 def CheckAffected(change):
150 for request in change.build.requests:
151 for source in request.sources:
152 for change in source.changes:
153 for file in change.files:
154 if re.match(pattern, file):
159 def isPathBuiltin(path):
162 conf = open(".config", "r")
165 line = conf.readline()
168 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
170 incl[m.group(1)] = True
174 deps = open("tmp/.packagedeps", "r")
177 line = deps.readline()
180 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
181 if m and incl.get(m.group(1)) == True:
182 pkgs["package/%s" % m.group(2)] = True
187 if pkgs.get(path) == True:
189 path = os.path.dirname(path)
193 def isChangeBuiltin(change):
195 # for request in change.build.requests:
196 # for source in request.sources:
197 # for change in source.changes:
198 # for file in change.files:
199 # if isPathBuiltin(file):
206 dlLock = locks.SlaveLock("slave_dl")
208 checkBuiltin = re.sub('[\t\n ]+', ' ', """
210 local symbol op path file;
211 for file in $CHANGED_FILES; do
217 while read symbol op path; do
218 case "$symbol" in package-*)
219 symbol="${symbol##*(}";
220 symbol="${symbol%)}";
221 for file in $CHANGED_FILES; do
222 case "$file" in "package/$path/"*)
223 grep -qsx "$symbol=y" .config && return 0
227 done < tmp/.packagedeps;
233 class IfBuiltinShellCommand(ShellCommand):
234 def _quote(self, str):
235 if re.search("[^a-zA-Z0-9/_.-]", str):
236 return "'%s'" %(re.sub("'", "'\"'\"'", str))
239 def setCommand(self, command):
240 if not isinstance(command, (str, unicode)):
241 command = ' '.join(map(self._quote, command))
244 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
247 def setupEnvironment(self, cmd):
248 slaveEnv = self.slaveEnvironment
252 for request in self.build.requests:
253 for source in request.sources:
254 for change in source.changes:
255 for file in change.files:
256 changedFiles[file] = True
257 fullSlaveEnv = slaveEnv.copy()
258 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
259 cmd.args['env'] = fullSlaveEnv
263 for slave in c['slaves']:
264 slaveNames.append(slave.slavename)
266 for target in targets:
267 ts = target.split('/')
269 factory = BuildFactory()
271 # find number of cores
272 factory.addStep(SetProperty(
275 description = "Finding number of CPUs",
276 command = ["nproc"]))
278 # expire tree if needed
280 factory.addStep(FileDownload(
281 mastersrc = "expire.sh",
282 slavedest = "../expire.sh",
285 factory.addStep(ShellCommand(
287 description = "Checking for build tree expiry",
288 command = ["./expire.sh", str(tree_expire)],
290 haltOnFailure = True,
293 # check out the source
294 factory.addStep(Git(repourl=repo_url, mode='update'))
296 factory.addStep(ShellCommand(
298 description = "Remove tmp folder",
299 command=["rm", "-rf", "tmp/"]))
302 # factory.addStep(ShellCommand(
303 # name = "feedsconf",
304 # description = "Copy the feeds.conf",
305 # command='''cp ~/feeds.conf ./feeds.conf''' ))
308 factory.addStep(ShellCommand(
309 name = "rmfeedlinks",
310 description = "Remove feed symlinks",
311 command=["rm", "-rf", "package/feeds/"]))
314 factory.addStep(ShellCommand(
315 name = "updatefeeds",
316 description = "Updating feeds",
317 command=["./scripts/feeds", "update"]))
320 factory.addStep(ShellCommand(
321 name = "installfeeds",
322 description = "Installing feeds",
323 command=["./scripts/feeds", "install", "-a"]))
326 factory.addStep(ShellCommand(
328 description = "Seeding .config",
329 command='''cat <<EOT > .config
331 CONFIG_TARGET_%s_%s=y
332 CONFIG_ALL_NONSHARED=y
335 # CONFIG_IB_STANDALONE is not set
338 CONFIG_SIGNED_PACKAGES=y
339 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
340 CONFIG_KERNEL_KALLSYMS=y
341 CONFIG_COLLECT_KERNEL_DEBUG=y
342 CONFIG_TARGET_ALL_PROFILES=y
343 CONFIG_TARGET_MULTI_PROFILE=y
344 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
345 EOT''' %(ts[0], ts[0], ts[1]) ))
347 factory.addStep(ShellCommand(
349 description = "Removing output directory",
350 command = ["rm", "-rf", "bin/"]
353 factory.addStep(ShellCommand(
355 description = "Populating .config",
356 command = ["make", "defconfig"]
360 factory.addStep(ShellCommand(
362 description = "Checking architecture",
363 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
371 factory.addStep(SetProperty(
374 description = "Finding libc suffix",
375 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
378 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
379 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
382 factory.addStep(ShellCommand(
384 description = "Preparing dl/",
385 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
391 factory.addStep(ShellCommand(
393 description = "Populating dl/",
394 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
396 locks = [dlLock.access('exclusive')]
399 factory.addStep(ShellCommand(
401 description = "Cleaning base-files",
402 command=["make", "package/base-files/clean", "V=s"]
405 # optional clean steps
406 for pattern, maketarget in MakeTargetMap.items():
407 factory.addStep(ShellCommand(
409 description = maketarget,
410 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
414 factory.addStep(ShellCommand(
416 description = "Building tools",
417 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
421 factory.addStep(ShellCommand(
423 description = "Building toolchain",
424 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
428 factory.addStep(ShellCommand(
430 description = "Building kmods",
431 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
432 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
436 factory.addStep(ShellCommand(
438 description = "Building packages",
439 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
440 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
444 # factory.addStep(IfBuiltinShellCommand(
445 factory.addStep(ShellCommand(
447 description = "Installing packages",
448 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
449 doStepIf = isChangeBuiltin,
453 factory.addStep(ShellCommand(
455 description = "Indexing packages",
456 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
460 #factory.addStep(IfBuiltinShellCommand(
461 factory.addStep(ShellCommand(
463 description = "Building images",
464 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
465 doStepIf = isChangeBuiltin,
469 factory.addStep(ShellCommand(
471 description = "Calculating checksums",
472 command=["make", "-j1", "checksum", "V=s"],
473 doStepIf = isChangeBuiltin,
478 if gpg_keyid is not None:
479 factory.addStep(MasterShellCommand(
480 name = "signprepare",
481 description = "Preparing temporary signing directory",
482 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
486 factory.addStep(ShellCommand(
488 description = "Packing files to sign",
489 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
493 factory.addStep(FileUpload(
494 slavesrc = "sign.tar.gz",
495 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
499 factory.addStep(MasterShellCommand(
501 description = "Signing files",
502 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
506 factory.addStep(FileDownload(
507 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
508 slavedest = "sign.tar.gz",
512 factory.addStep(ShellCommand(
514 description = "Unpacking signed files",
515 command = ["tar", "-xzf", "sign.tar.gz"],
520 factory.addStep(ShellCommand(
521 name = "uploadprepare",
522 description = "Preparing target directory",
523 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
524 env={'RSYNC_PASSWORD': rsync_bin_key},
525 haltOnFailure = True,
529 factory.addStep(ShellCommand(
530 name = "targetupload",
531 description = "Uploading target files",
532 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
533 env={'RSYNC_PASSWORD': rsync_bin_key},
534 haltOnFailure = True,
538 if rsync_src_url is not None:
539 factory.addStep(ShellCommand(
540 name = "sourceupload",
541 description = "Uploading source archives",
542 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
543 env={'RSYNC_PASSWORD': rsync_src_key},
544 haltOnFailure = True,
549 factory.addStep(ShellCommand(
550 name = "packageupload",
551 description = "Uploading package files",
552 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
553 env={'RSYNC_PASSWORD': rsync_bin_key},
554 haltOnFailure = False,
560 factory.addStep(ShellCommand(
562 description = "Uploading logs",
563 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
564 env={'RSYNC_PASSWORD': rsync_bin_key},
565 haltOnFailure = False,
570 from buildbot.config import BuilderConfig
572 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
575 ####### STATUS TARGETS
577 # 'status' is a list of Status Targets. The results of each build will be
578 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
579 # including web pages, email senders, and IRC bots.
583 from buildbot.status import html
584 from buildbot.status.web import authz, auth
586 if ini.has_option("status", "bind"):
587 if ini.has_option("status", "user") and ini.has_option("status", "password"):
588 authz_cfg=authz.Authz(
589 # change any of these to True to enable; see the manual for more
591 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
592 gracefulShutdown = 'auth',
593 forceBuild = 'auth', # use this to test your slave once it is set up
594 forceAllBuilds = 'auth',
597 stopAllBuilds = 'auth',
598 cancelPendingBuild = 'auth',
600 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
602 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
605 from buildbot.status import words
607 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
608 irc_host = ini.get("irc", "host")
610 irc_chan = ini.get("irc", "channel")
611 irc_nick = ini.get("irc", "nickname")
614 if ini.has_option("irc", "port"):
615 irc_port = ini.getint("irc", "port")
617 if ini.has_option("irc", "password"):
618 irc_pass = ini.get("irc", "password")
620 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
621 channels = [{ "channel": irc_chan }],
624 'successToFailure': 1,
625 'failureToSuccess': 1
629 c['status'].append(irc)
632 ####### PROJECT IDENTITY
634 # the 'title' string will appear at the top of this buildbot
635 # installation's html.WebStatus home page (linked to the
636 # 'titleURL') and is embedded in the title of the waterfall HTML page.
638 c['title'] = ini.get("general", "title")
639 c['titleURL'] = ini.get("general", "title_url")
641 # the 'buildbotURL' string should point to the location where the buildbot's
642 # internal web server (usually the html.WebStatus page) is visible. This
643 # typically uses the port number set in the Waterfall 'status' entry, but
644 # with an externally-visible host name which the buildbot cannot figure out
647 c['buildbotURL'] = ini.get("general", "buildbot_url")
652 # This specifies what database buildbot uses to store its state. You can leave
653 # this at its default for all but the largest installations.
654 'db_url' : "sqlite:///state.sqlite",