2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
43 c['slavePortnum'] = 9989
46 c['mergeRequests'] = True
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
52 repo_url = ini.get("repo", "url")
54 rsync_bin_url = ini.get("rsync", "binary_url")
55 rsync_bin_key = ini.get("rsync", "binary_password")
60 if ini.has_option("rsync", "source_url"):
61 rsync_src_url = ini.get("rsync", "source_url")
62 rsync_src_key = ini.get("rsync", "source_password")
65 gpg_comment = "Unattended build signature"
66 gpg_passfile = "/dev/null"
68 if ini.has_option("gpg", "keyid"):
69 gpg_keyid = ini.get("gpg", "keyid")
71 if ini.has_option("gpg", "comment"):
72 gpg_comment = ini.get("gpg", "comment")
74 if ini.has_option("gpg", "passfile"):
75 gpg_passfile = ini.get("gpg", "passfile")
81 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
82 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
85 line = findtargets.stdout.readline()
88 ta = line.strip().split(' ')
92 # the 'change_source' setting tells the buildmaster how it should find out
93 # about source code changes. Here we point to the buildbot clone of pyflakes.
95 from buildbot.changes.gitpoller import GitPoller
96 c['change_source'] = []
97 c['change_source'].append(GitPoller(
99 workdir=home_dir+'/source.git', branch='master',
104 # Configure the Schedulers, which decide how to react to incoming changes. In this
105 # case, just kick off a 'basebuild' build
107 from buildbot.schedulers.basic import SingleBranchScheduler
108 from buildbot.schedulers.forcesched import ForceScheduler
109 from buildbot.changes import filter
111 c['schedulers'].append(SingleBranchScheduler(
113 change_filter=filter.ChangeFilter(branch='master'),
115 builderNames=targets))
117 c['schedulers'].append(ForceScheduler(
119 builderNames=targets))
123 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
124 # what steps, and which slaves can execute them. Note that any particular build will
125 # only take place on one slave.
127 from buildbot.process.factory import BuildFactory
128 from buildbot.steps.source import Git
129 from buildbot.steps.shell import ShellCommand
130 from buildbot.steps.shell import SetProperty
131 from buildbot.steps.transfer import FileUpload
132 from buildbot.steps.transfer import FileDownload
133 from buildbot.steps.master import MasterShellCommand
134 from buildbot.process.properties import WithProperties
138 "^tools/": "tools/clean",
139 "^toolchain/": "toolchain/clean",
140 "^target/linux/": "target/linux/clean",
141 "^(config|include)/": "dirclean"
144 def IsAffected(pattern):
145 def CheckAffected(change):
146 for request in change.build.requests:
147 for source in request.sources:
148 for change in source.changes:
149 for file in change.files:
150 if re.match(pattern, file):
155 def isPathBuiltin(path):
158 conf = open(".config", "r")
161 line = conf.readline()
164 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
166 incl[m.group(1)] = True
170 deps = open("tmp/.packagedeps", "r")
173 line = deps.readline()
176 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
177 if m and incl.get(m.group(1)) == True:
178 pkgs["package/%s" % m.group(2)] = True
183 if pkgs.get(path) == True:
185 path = os.path.dirname(path)
189 def isChangeBuiltin(change):
191 # for request in change.build.requests:
192 # for source in request.sources:
193 # for change in source.changes:
194 # for file in change.files:
195 # if isPathBuiltin(file):
202 dlLock = locks.SlaveLock("slave_dl")
204 checkBuiltin = re.sub('[\t\n ]+', ' ', """
206 local symbol op path file;
207 for file in $CHANGED_FILES; do
213 while read symbol op path; do
214 case "$symbol" in package-*)
215 symbol="${symbol##*(}";
216 symbol="${symbol%)}";
217 for file in $CHANGED_FILES; do
218 case "$file" in "package/$path/"*)
219 grep -qsx "$symbol=y" .config && return 0
223 done < tmp/.packagedeps;
229 class IfBuiltinShellCommand(ShellCommand):
230 def _quote(self, str):
231 if re.search("[^a-zA-Z0-9/_.-]", str):
232 return "'%s'" %(re.sub("'", "'\"'\"'", str))
235 def setCommand(self, command):
236 if not isinstance(command, (str, unicode)):
237 command = ' '.join(map(self._quote, command))
240 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
243 def setupEnvironment(self, cmd):
244 slaveEnv = self.slaveEnvironment
248 for request in self.build.requests:
249 for source in request.sources:
250 for change in source.changes:
251 for file in change.files:
252 changedFiles[file] = True
253 fullSlaveEnv = slaveEnv.copy()
254 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
255 cmd.args['env'] = fullSlaveEnv
259 for slave in c['slaves']:
260 slaveNames.append(slave.slavename)
262 for target in targets:
263 ts = target.split('/')
265 factory = BuildFactory()
267 # find number of cores
268 factory.addStep(SetProperty(
271 description = "Finding number of CPUs",
272 command = ["nproc"]))
274 # check out the source
275 factory.addStep(Git(repourl=repo_url, mode='update'))
277 factory.addStep(ShellCommand(
279 description = "Remove tmp folder",
280 command=["rm", "-rf", "tmp/"]))
283 # factory.addStep(ShellCommand(
284 # name = "feedsconf",
285 # description = "Copy the feeds.conf",
286 # command='''cp ~/feeds.conf ./feeds.conf''' ))
289 factory.addStep(ShellCommand(
290 name = "updatefeeds",
291 description = "Updating feeds",
292 command=["./scripts/feeds", "update"]))
295 factory.addStep(ShellCommand(
296 name = "installfeeds",
297 description = "Installing feeds",
298 command=["./scripts/feeds", "install", "-a"]))
301 factory.addStep(ShellCommand(
303 description = "Seeding .config",
304 command='''cat <<EOT > .config
306 CONFIG_TARGET_%s_%s=y
307 CONFIG_ALL_NONSHARED=y
310 # CONFIG_IB_STANDALONE is not set
313 CONFIG_SIGNED_PACKAGES=y
314 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
315 CONFIG_KERNEL_KALLSYMS=y
316 CONFIG_COLLECT_KERNEL_DEBUG=y
317 EOT''' %(ts[0], ts[0], ts[1]) ))
319 factory.addStep(ShellCommand(
321 description = "Removing output directory",
322 command = ["rm", "-rf", "bin/"]
325 factory.addStep(ShellCommand(
327 description = "Populating .config",
328 command = ["make", "defconfig"]
332 factory.addStep(ShellCommand(
334 description = "Checking architecture",
335 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
343 factory.addStep(SetProperty(
346 description = "Finding libc suffix",
347 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
350 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
351 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
354 factory.addStep(ShellCommand(
356 description = "Preparing dl/",
357 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
363 factory.addStep(ShellCommand(
365 description = "Populating dl/",
366 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
368 locks = [dlLock.access('exclusive')]
371 factory.addStep(ShellCommand(
373 description = "Cleaning base-files",
374 command=["make", "package/base-files/clean", "V=s"]
377 # optional clean steps
378 for pattern, maketarget in MakeTargetMap.items():
379 factory.addStep(ShellCommand(
381 description = maketarget,
382 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
386 factory.addStep(ShellCommand(
388 description = "Building tools",
389 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
393 factory.addStep(ShellCommand(
395 description = "Building toolchain",
396 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
400 factory.addStep(ShellCommand(
402 description = "Building kmods",
403 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
404 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
408 factory.addStep(ShellCommand(
410 description = "Building packages",
411 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
412 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
416 # factory.addStep(IfBuiltinShellCommand(
417 factory.addStep(ShellCommand(
419 description = "Installing packages",
420 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
421 doStepIf = isChangeBuiltin,
425 factory.addStep(ShellCommand(
427 description = "Indexing packages",
428 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
432 #factory.addStep(IfBuiltinShellCommand(
433 factory.addStep(ShellCommand(
435 description = "Building images",
436 command=["make", "-j1", "target/install", "V=s"],
437 doStepIf = isChangeBuiltin,
442 if gpg_keyid is not None:
443 factory.addStep(MasterShellCommand(
444 name = "signprepare",
445 description = "Preparing temporary signing directory",
446 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
450 factory.addStep(ShellCommand(
452 description = "Packing files to sign",
453 command = ["sh", "-c", WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1]))],
457 factory.addStep(FileUpload(
458 slavesrc = "sign.tar.gz",
459 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
463 factory.addStep(MasterShellCommand(
465 description = "Signing files",
466 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
470 factory.addStep(FileDownload(
471 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
472 slavedest = "sign.tar.gz",
476 factory.addStep(ShellCommand(
478 description = "Unpacking signed files",
479 command = ["tar", "-xzf", "sign.tar.gz"],
484 factory.addStep(ShellCommand(
485 name = "uploadprepare",
486 description = "Preparing target directory",
487 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
488 env={'RSYNC_PASSWORD': rsync_bin_key},
489 haltOnFailure = True,
493 factory.addStep(ShellCommand(
494 name = "targetupload",
495 description = "Uploading target files",
496 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
497 env={'RSYNC_PASSWORD': rsync_bin_key},
498 haltOnFailure = True,
502 if rsync_src_url is not None:
503 factory.addStep(ShellCommand(
504 name = "sourceupload",
505 description = "Uploading source archives",
506 command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
507 env={'RSYNC_PASSWORD': rsync_src_key},
508 haltOnFailure = True,
513 factory.addStep(ShellCommand(
514 name = "packageupload",
515 description = "Uploading package files",
516 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
517 env={'RSYNC_PASSWORD': rsync_bin_key},
518 haltOnFailure = False,
524 factory.addStep(ShellCommand(
526 description = "Uploading logs",
527 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
528 env={'RSYNC_PASSWORD': rsync_bin_key},
529 haltOnFailure = False,
534 from buildbot.config import BuilderConfig
536 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
539 ####### STATUS TARGETS
541 # 'status' is a list of Status Targets. The results of each build will be
542 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
543 # including web pages, email senders, and IRC bots.
547 from buildbot.status import html
548 from buildbot.status.web import authz, auth
550 if ini.has_option("status", "bind"):
551 if ini.has_option("status", "user") and ini.has_option("status", "password"):
552 authz_cfg=authz.Authz(
553 # change any of these to True to enable; see the manual for more
555 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
556 gracefulShutdown = 'auth',
557 forceBuild = 'auth', # use this to test your slave once it is set up
558 forceAllBuilds = 'auth',
561 stopAllBuilds = 'auth',
562 cancelPendingBuild = 'auth',
564 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
566 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
569 from buildbot.status import words
571 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
572 irc_host = ini.get("irc", "host")
574 irc_chan = ini.get("irc", "channel")
575 irc_nick = ini.get("irc", "nickname")
578 if ini.has_option("irc", "port"):
579 irc_port = ini.getint("irc", "port")
581 if ini.has_option("irc", "password"):
582 irc_pass = ini.get("irc", "password")
584 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
585 channels = [{ "channel": irc_chan }],
588 'successToFailure': 1,
589 'failureToSuccess': 1
593 c['status'].append(irc)
596 ####### PROJECT IDENTITY
598 # the 'title' string will appear at the top of this buildbot
599 # installation's html.WebStatus home page (linked to the
600 # 'titleURL') and is embedded in the title of the waterfall HTML page.
602 c['title'] = ini.get("general", "title")
603 c['titleURL'] = ini.get("general", "title_url")
605 # the 'buildbotURL' string should point to the location where the buildbot's
606 # internal web server (usually the html.WebStatus page) is visible. This
607 # typically uses the port number set in the Waterfall 'status' entry, but
608 # with an externally-visible host name which the buildbot cannot figure out
611 c['buildbotURL'] = ini.get("general", "buildbot_url")
616 # This specifies what database buildbot uses to store its state. You can leave
617 # this at its default for all but the largest installations.
618 'db_url' : "sqlite:///state.sqlite",