2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
48 c['slavePortnum'] = slave_port
51 c['mergeRequests'] = True
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
78 gpg_comment = "Unattended build signature"
79 gpg_passfile = "/dev/null"
81 if ini.has_option("gpg", "keyid"):
82 gpg_keyid = ini.get("gpg", "keyid")
84 if ini.has_option("gpg", "comment"):
85 gpg_comment = ini.get("gpg", "comment")
87 if ini.has_option("gpg", "passfile"):
88 gpg_passfile = ini.get("gpg", "passfile")
94 if not os.path.isdir(home_dir+'/source.git'):
95 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
96 subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
98 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
99 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
102 line = findtargets.stdout.readline()
105 ta = line.strip().split(' ')
106 targets.append(ta[0])
109 # the 'change_source' setting tells the buildmaster how it should find out
110 # about source code changes. Here we point to the buildbot clone of pyflakes.
112 from buildbot.changes.gitpoller import GitPoller
113 c['change_source'] = []
114 c['change_source'].append(GitPoller(
116 workdir=home_dir+'/source.git', branch='master',
121 # Configure the Schedulers, which decide how to react to incoming changes. In this
122 # case, just kick off a 'basebuild' build
124 from buildbot.schedulers.basic import SingleBranchScheduler
125 from buildbot.schedulers.forcesched import ForceScheduler
126 from buildbot.changes import filter
128 c['schedulers'].append(SingleBranchScheduler(
130 change_filter=filter.ChangeFilter(branch='master'),
132 builderNames=targets))
134 c['schedulers'].append(ForceScheduler(
136 builderNames=targets))
140 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
141 # what steps, and which slaves can execute them. Note that any particular build will
142 # only take place on one slave.
144 from buildbot.process.factory import BuildFactory
145 from buildbot.steps.source import Git
146 from buildbot.steps.shell import ShellCommand
147 from buildbot.steps.shell import SetProperty
148 from buildbot.steps.transfer import FileUpload
149 from buildbot.steps.transfer import FileDownload
150 from buildbot.steps.master import MasterShellCommand
151 from buildbot.process.properties import WithProperties
155 [ "tools", "tools/clean" ],
156 [ "chain", "toolchain/clean" ],
157 [ "linux", "target/linux/clean" ],
158 [ "dir", "dirclean" ],
159 [ "dist", "distclean" ]
162 def IsCleanRequested(pattern):
163 def CheckCleanProperty(step):
164 val = step.getProperty("clean")
165 if val and re.match(pattern, val):
170 return CheckCleanProperty
175 dlLock = locks.SlaveLock("slave_dl")
177 checkBuiltin = re.sub('[\t\n ]+', ' ', """
179 local symbol op path file;
180 for file in $CHANGED_FILES; do
186 while read symbol op path; do
187 case "$symbol" in package-*)
188 symbol="${symbol##*(}";
189 symbol="${symbol%)}";
190 for file in $CHANGED_FILES; do
191 case "$file" in "package/$path/"*)
192 grep -qsx "$symbol=y" .config && return 0
196 done < tmp/.packagedeps;
202 class IfBuiltinShellCommand(ShellCommand):
203 def _quote(self, str):
204 if re.search("[^a-zA-Z0-9/_.-]", str):
205 return "'%s'" %(re.sub("'", "'\"'\"'", str))
208 def setCommand(self, command):
209 if not isinstance(command, (str, unicode)):
210 command = ' '.join(map(self._quote, command))
213 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
216 def setupEnvironment(self, cmd):
217 slaveEnv = self.slaveEnvironment
221 for request in self.build.requests:
222 for source in request.sources:
223 for change in source.changes:
224 for file in change.files:
225 changedFiles[file] = True
226 fullSlaveEnv = slaveEnv.copy()
227 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
228 cmd.args['env'] = fullSlaveEnv
232 for slave in c['slaves']:
233 slaveNames.append(slave.slavename)
235 for target in targets:
236 ts = target.split('/')
238 factory = BuildFactory()
240 # find number of cores
241 factory.addStep(SetProperty(
244 description = "Finding number of CPUs",
245 command = ["nproc"]))
247 # expire tree if needed
249 factory.addStep(FileDownload(
250 mastersrc = "expire.sh",
251 slavedest = "../expire.sh",
254 factory.addStep(ShellCommand(
256 description = "Checking for build tree expiry",
257 command = ["./expire.sh", str(tree_expire)],
259 haltOnFailure = True,
262 # user-requested clean targets
263 for tuple in CleanTargetMap:
264 factory.addStep(ShellCommand(
266 description = 'User-requested "make %s"' % tuple[1],
267 command = ["make", tuple[1], "V=s"],
268 doStepIf = IsCleanRequested(tuple[0])
271 # check out the source
272 factory.addStep(Git(repourl=repo_url, branch=repo_branch, mode='update'))
274 factory.addStep(ShellCommand(
276 description = "Remove tmp folder",
277 command=["rm", "-rf", "tmp/"]))
280 # factory.addStep(ShellCommand(
281 # name = "feedsconf",
282 # description = "Copy the feeds.conf",
283 # command='''cp ~/feeds.conf ./feeds.conf''' ))
286 factory.addStep(ShellCommand(
287 name = "rmfeedlinks",
288 description = "Remove feed symlinks",
289 command=["rm", "-rf", "package/feeds/"]))
292 factory.addStep(ShellCommand(
293 name = "updatefeeds",
294 description = "Updating feeds",
295 command=["./scripts/feeds", "update"]))
298 factory.addStep(ShellCommand(
299 name = "installfeeds",
300 description = "Installing feeds",
301 command=["./scripts/feeds", "install", "-a"]))
304 factory.addStep(ShellCommand(
306 description = "Seeding .config",
307 command='''cat <<EOT > .config
309 CONFIG_TARGET_%s_%s=y
310 CONFIG_ALL_NONSHARED=y
313 # CONFIG_IB_STANDALONE is not set
316 CONFIG_SIGNED_PACKAGES=y
317 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
318 CONFIG_KERNEL_KALLSYMS=y
319 CONFIG_COLLECT_KERNEL_DEBUG=y
320 CONFIG_TARGET_ALL_PROFILES=y
321 CONFIG_TARGET_MULTI_PROFILE=y
322 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
323 EOT''' %(ts[0], ts[0], ts[1]) ))
325 factory.addStep(ShellCommand(
327 description = "Removing output directory",
328 command = ["rm", "-rf", "bin/"]
331 factory.addStep(ShellCommand(
333 description = "Populating .config",
334 command = ["make", "defconfig"]
338 factory.addStep(ShellCommand(
340 description = "Checking architecture",
341 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
349 factory.addStep(SetProperty(
352 description = "Finding libc suffix",
353 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
356 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
357 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
360 factory.addStep(ShellCommand(
362 description = "Preparing dl/",
363 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
369 factory.addStep(ShellCommand(
371 description = "Building GNU tar",
372 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
377 factory.addStep(ShellCommand(
379 description = "Populating dl/",
380 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
382 locks = [dlLock.access('exclusive')]
385 factory.addStep(ShellCommand(
387 description = "Cleaning base-files",
388 command=["make", "package/base-files/clean", "V=s"]
392 factory.addStep(ShellCommand(
394 description = "Building tools",
395 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
399 factory.addStep(ShellCommand(
401 description = "Building toolchain",
402 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
406 factory.addStep(ShellCommand(
408 description = "Building kmods",
409 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
410 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
414 factory.addStep(ShellCommand(
416 description = "Building packages",
417 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
418 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
422 # factory.addStep(IfBuiltinShellCommand(
423 factory.addStep(ShellCommand(
425 description = "Installing packages",
426 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
430 factory.addStep(ShellCommand(
432 description = "Indexing packages",
433 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
437 #factory.addStep(IfBuiltinShellCommand(
438 factory.addStep(ShellCommand(
440 description = "Building images",
441 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
445 factory.addStep(ShellCommand(
447 description = "Calculating checksums",
448 command=["make", "-j1", "checksum", "V=s"],
453 if gpg_keyid is not None:
454 factory.addStep(MasterShellCommand(
455 name = "signprepare",
456 description = "Preparing temporary signing directory",
457 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
461 factory.addStep(ShellCommand(
463 description = "Packing files to sign",
464 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
468 factory.addStep(FileUpload(
469 slavesrc = "sign.tar.gz",
470 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
474 factory.addStep(MasterShellCommand(
476 description = "Signing files",
477 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
481 factory.addStep(FileDownload(
482 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
483 slavedest = "sign.tar.gz",
487 factory.addStep(ShellCommand(
489 description = "Unpacking signed files",
490 command = ["tar", "-xzf", "sign.tar.gz"],
495 factory.addStep(ShellCommand(
496 name = "uploadprepare",
497 description = "Preparing target directory",
498 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
499 env={'RSYNC_PASSWORD': rsync_bin_key},
500 haltOnFailure = True,
504 factory.addStep(ShellCommand(
505 name = "targetupload",
506 description = "Uploading target files",
507 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
508 env={'RSYNC_PASSWORD': rsync_bin_key},
509 haltOnFailure = True,
513 if rsync_src_url is not None:
514 factory.addStep(ShellCommand(
515 name = "sourceupload",
516 description = "Uploading source archives",
517 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
518 env={'RSYNC_PASSWORD': rsync_src_key},
519 haltOnFailure = True,
524 factory.addStep(ShellCommand(
525 name = "packageupload",
526 description = "Uploading package files",
527 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
528 env={'RSYNC_PASSWORD': rsync_bin_key},
529 haltOnFailure = False,
535 factory.addStep(ShellCommand(
537 description = "Uploading logs",
538 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
539 env={'RSYNC_PASSWORD': rsync_bin_key},
540 haltOnFailure = False,
545 from buildbot.config import BuilderConfig
547 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
550 ####### STATUS TARGETS
552 # 'status' is a list of Status Targets. The results of each build will be
553 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
554 # including web pages, email senders, and IRC bots.
558 from buildbot.status import html
559 from buildbot.status.web import authz, auth
561 if ini.has_option("status", "bind"):
562 if ini.has_option("status", "user") and ini.has_option("status", "password"):
563 authz_cfg=authz.Authz(
564 # change any of these to True to enable; see the manual for more
566 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
567 gracefulShutdown = 'auth',
568 forceBuild = 'auth', # use this to test your slave once it is set up
569 forceAllBuilds = 'auth',
572 stopAllBuilds = 'auth',
573 cancelPendingBuild = 'auth',
575 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
577 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
580 from buildbot.status import words
582 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
583 irc_host = ini.get("irc", "host")
585 irc_chan = ini.get("irc", "channel")
586 irc_nick = ini.get("irc", "nickname")
589 if ini.has_option("irc", "port"):
590 irc_port = ini.getint("irc", "port")
592 if ini.has_option("irc", "password"):
593 irc_pass = ini.get("irc", "password")
595 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
596 channels = [{ "channel": irc_chan }],
599 'successToFailure': 1,
600 'failureToSuccess': 1
604 c['status'].append(irc)
607 ####### PROJECT IDENTITY
609 # the 'title' string will appear at the top of this buildbot
610 # installation's html.WebStatus home page (linked to the
611 # 'titleURL') and is embedded in the title of the waterfall HTML page.
613 c['title'] = ini.get("general", "title")
614 c['titleURL'] = ini.get("general", "title_url")
616 # the 'buildbotURL' string should point to the location where the buildbot's
617 # internal web server (usually the html.WebStatus page) is visible. This
618 # typically uses the port number set in the Waterfall 'status' entry, but
619 # with an externally-visible host name which the buildbot cannot figure out
622 c['buildbotURL'] = ini.get("general", "buildbot_url")
627 # This specifies what database buildbot uses to store its state. You can leave
628 # this at its default for all but the largest installations.
629 'db_url' : "sqlite:///state.sqlite",