2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
43 c['slavePortnum'] = 9989
46 c['mergeRequests'] = True
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
52 repo_url = ini.get("repo", "url")
54 rsync_bin_url = ini.get("rsync", "binary_url")
55 rsync_bin_key = ini.get("rsync", "binary_password")
60 if ini.has_option("rsync", "source_url"):
61 rsync_src_url = ini.get("rsync", "source_url")
62 rsync_src_key = ini.get("rsync", "source_password")
65 gpg_comment = "Unattended build signature"
66 gpg_passfile = "/dev/null"
68 if ini.has_option("gpg", "keyid"):
69 gpg_keyid = ini.get("gpg", "keyid")
71 if ini.has_option("gpg", "comment"):
72 gpg_comment = ini.get("gpg", "comment")
74 if ini.has_option("gpg", "passfile"):
75 gpg_passfile = ini.get("gpg", "passfile")
81 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
82 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
85 line = findtargets.stdout.readline()
88 ta = line.strip().split(' ')
92 # the 'change_source' setting tells the buildmaster how it should find out
93 # about source code changes. Here we point to the buildbot clone of pyflakes.
95 from buildbot.changes.gitpoller import GitPoller
96 c['change_source'] = []
97 c['change_source'].append(GitPoller(
99 workdir=home_dir+'/source.git', branch='master',
104 # Configure the Schedulers, which decide how to react to incoming changes. In this
105 # case, just kick off a 'basebuild' build
107 from buildbot.schedulers.basic import SingleBranchScheduler
108 from buildbot.schedulers.forcesched import ForceScheduler
109 from buildbot.changes import filter
111 c['schedulers'].append(SingleBranchScheduler(
113 change_filter=filter.ChangeFilter(branch='master'),
115 builderNames=targets))
117 c['schedulers'].append(ForceScheduler(
119 builderNames=targets))
123 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
124 # what steps, and which slaves can execute them. Note that any particular build will
125 # only take place on one slave.
127 from buildbot.process.factory import BuildFactory
128 from buildbot.steps.source import Git
129 from buildbot.steps.shell import ShellCommand
130 from buildbot.steps.shell import SetProperty
131 from buildbot.steps.transfer import FileUpload
132 from buildbot.steps.transfer import FileDownload
133 from buildbot.steps.master import MasterShellCommand
134 from buildbot.process.properties import WithProperties
138 "^tools/": "tools/clean",
139 "^toolchain/": "toolchain/clean",
140 "^target/linux/": "target/linux/clean",
141 "^(config|include)/": "dirclean"
144 def IsAffected(pattern):
145 def CheckAffected(change):
146 for request in change.build.requests:
147 for source in request.sources:
148 for change in source.changes:
149 for file in change.files:
150 if re.match(pattern, file):
155 def isPathBuiltin(path):
158 conf = open(".config", "r")
161 line = conf.readline()
164 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
166 incl[m.group(1)] = True
170 deps = open("tmp/.packagedeps", "r")
173 line = deps.readline()
176 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
177 if m and incl.get(m.group(1)) == True:
178 pkgs["package/%s" % m.group(2)] = True
183 if pkgs.get(path) == True:
185 path = os.path.dirname(path)
189 def isChangeBuiltin(change):
191 # for request in change.build.requests:
192 # for source in request.sources:
193 # for change in source.changes:
194 # for file in change.files:
195 # if isPathBuiltin(file):
202 dlLock = locks.SlaveLock("slave_dl")
204 checkBuiltin = re.sub('[\t\n ]+', ' ', """
206 local symbol op path file;
207 for file in $CHANGED_FILES; do
213 while read symbol op path; do
214 case "$symbol" in package-*)
215 symbol="${symbol##*(}";
216 symbol="${symbol%)}";
217 for file in $CHANGED_FILES; do
218 case "$file" in "package/$path/"*)
219 grep -qsx "$symbol=y" .config && return 0
223 done < tmp/.packagedeps;
229 class IfBuiltinShellCommand(ShellCommand):
230 def _quote(self, str):
231 if re.search("[^a-zA-Z0-9/_.-]", str):
232 return "'%s'" %(re.sub("'", "'\"'\"'", str))
235 def setCommand(self, command):
236 if not isinstance(command, (str, unicode)):
237 command = ' '.join(map(self._quote, command))
240 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
243 def setupEnvironment(self, cmd):
244 slaveEnv = self.slaveEnvironment
248 for request in self.build.requests:
249 for source in request.sources:
250 for change in source.changes:
251 for file in change.files:
252 changedFiles[file] = True
253 fullSlaveEnv = slaveEnv.copy()
254 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
255 cmd.args['env'] = fullSlaveEnv
259 for slave in c['slaves']:
260 slaveNames.append(slave.slavename)
262 for target in targets:
263 ts = target.split('/')
265 factory = BuildFactory()
267 # find number of cores
268 factory.addStep(SetProperty(
271 description = "Finding number of CPUs",
272 command = ["nproc"]))
274 # check out the source
275 factory.addStep(Git(repourl=repo_url, mode='update'))
277 factory.addStep(ShellCommand(
279 description = "Remove tmp folder",
280 command=["rm", "-rf", "tmp/"]))
283 # factory.addStep(ShellCommand(
284 # name = "feedsconf",
285 # description = "Copy the feeds.conf",
286 # command='''cp ~/feeds.conf ./feeds.conf''' ))
289 factory.addStep(ShellCommand(
290 name = "updatefeeds",
291 description = "Updating feeds",
292 command=["./scripts/feeds", "update"]))
295 factory.addStep(ShellCommand(
296 name = "installfeeds",
297 description = "Installing feeds",
298 command=["./scripts/feeds", "install", "-a"]))
301 factory.addStep(ShellCommand(
303 description = "Seeding .config",
304 command='''cat <<EOT > .config
306 CONFIG_TARGET_%s_%s=y
307 CONFIG_ALL_NONSHARED=y
310 # CONFIG_IB_STANDALONE is not set
313 CONFIG_SIGNED_PACKAGES=y
314 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
315 CONFIG_KERNEL_KALLSYMS=y
316 CONFIG_COLLECT_KERNEL_DEBUG=y
317 CONFIG_TARGET_ALL_PROFILES=y
318 CONFIG_TARGET_MULTI_PROFILE=y
319 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
320 EOT''' %(ts[0], ts[0], ts[1]) ))
322 factory.addStep(ShellCommand(
324 description = "Removing output directory",
325 command = ["rm", "-rf", "bin/"]
328 factory.addStep(ShellCommand(
330 description = "Populating .config",
331 command = ["make", "defconfig"]
335 factory.addStep(ShellCommand(
337 description = "Checking architecture",
338 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
346 factory.addStep(SetProperty(
349 description = "Finding libc suffix",
350 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
353 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
354 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
357 factory.addStep(ShellCommand(
359 description = "Preparing dl/",
360 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
366 factory.addStep(ShellCommand(
368 description = "Populating dl/",
369 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
371 locks = [dlLock.access('exclusive')]
374 factory.addStep(ShellCommand(
376 description = "Cleaning base-files",
377 command=["make", "package/base-files/clean", "V=s"]
380 # optional clean steps
381 for pattern, maketarget in MakeTargetMap.items():
382 factory.addStep(ShellCommand(
384 description = maketarget,
385 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
389 factory.addStep(ShellCommand(
391 description = "Building tools",
392 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
396 factory.addStep(ShellCommand(
398 description = "Building toolchain",
399 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
403 factory.addStep(ShellCommand(
405 description = "Building kmods",
406 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
407 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
411 factory.addStep(ShellCommand(
413 description = "Building packages",
414 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
415 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
419 # factory.addStep(IfBuiltinShellCommand(
420 factory.addStep(ShellCommand(
422 description = "Installing packages",
423 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
424 doStepIf = isChangeBuiltin,
428 factory.addStep(ShellCommand(
430 description = "Indexing packages",
431 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
435 #factory.addStep(IfBuiltinShellCommand(
436 factory.addStep(ShellCommand(
438 description = "Building images",
439 command=["make", "-j1", "target/install", "V=s"],
440 doStepIf = isChangeBuiltin,
444 factory.addStep(ShellCommand(
446 description = "Calculating checksums",
447 command=["make", "-j1", "checksum", "V=s"],
448 doStepIf = isChangeBuiltin,
453 if gpg_keyid is not None:
454 factory.addStep(MasterShellCommand(
455 name = "signprepare",
456 description = "Preparing temporary signing directory",
457 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
461 factory.addStep(ShellCommand(
463 description = "Packing files to sign",
464 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
468 factory.addStep(FileUpload(
469 slavesrc = "sign.tar.gz",
470 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
474 factory.addStep(MasterShellCommand(
476 description = "Signing files",
477 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
481 factory.addStep(FileDownload(
482 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
483 slavedest = "sign.tar.gz",
487 factory.addStep(ShellCommand(
489 description = "Unpacking signed files",
490 command = ["tar", "-xzf", "sign.tar.gz"],
495 factory.addStep(ShellCommand(
496 name = "uploadprepare",
497 description = "Preparing target directory",
498 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
499 env={'RSYNC_PASSWORD': rsync_bin_key},
500 haltOnFailure = True,
504 factory.addStep(ShellCommand(
505 name = "targetupload",
506 description = "Uploading target files",
507 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
508 env={'RSYNC_PASSWORD': rsync_bin_key},
509 haltOnFailure = True,
513 if rsync_src_url is not None:
514 factory.addStep(ShellCommand(
515 name = "sourceupload",
516 description = "Uploading source archives",
517 command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
518 env={'RSYNC_PASSWORD': rsync_src_key},
519 haltOnFailure = True,
524 factory.addStep(ShellCommand(
525 name = "packageupload",
526 description = "Uploading package files",
527 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
528 env={'RSYNC_PASSWORD': rsync_bin_key},
529 haltOnFailure = False,
535 factory.addStep(ShellCommand(
537 description = "Uploading logs",
538 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
539 env={'RSYNC_PASSWORD': rsync_bin_key},
540 haltOnFailure = False,
545 from buildbot.config import BuilderConfig
547 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
550 ####### STATUS TARGETS
552 # 'status' is a list of Status Targets. The results of each build will be
553 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
554 # including web pages, email senders, and IRC bots.
558 from buildbot.status import html
559 from buildbot.status.web import authz, auth
561 if ini.has_option("status", "bind"):
562 if ini.has_option("status", "user") and ini.has_option("status", "password"):
563 authz_cfg=authz.Authz(
564 # change any of these to True to enable; see the manual for more
566 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
567 gracefulShutdown = 'auth',
568 forceBuild = 'auth', # use this to test your slave once it is set up
569 forceAllBuilds = 'auth',
572 stopAllBuilds = 'auth',
573 cancelPendingBuild = 'auth',
575 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
577 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
580 from buildbot.status import words
582 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
583 irc_host = ini.get("irc", "host")
585 irc_chan = ini.get("irc", "channel")
586 irc_nick = ini.get("irc", "nickname")
589 if ini.has_option("irc", "port"):
590 irc_port = ini.getint("irc", "port")
592 if ini.has_option("irc", "password"):
593 irc_pass = ini.get("irc", "password")
595 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
596 channels = [{ "channel": irc_chan }],
599 'successToFailure': 1,
600 'failureToSuccess': 1
604 c['status'].append(irc)
607 ####### PROJECT IDENTITY
609 # the 'title' string will appear at the top of this buildbot
610 # installation's html.WebStatus home page (linked to the
611 # 'titleURL') and is embedded in the title of the waterfall HTML page.
613 c['title'] = ini.get("general", "title")
614 c['titleURL'] = ini.get("general", "title_url")
616 # the 'buildbotURL' string should point to the location where the buildbot's
617 # internal web server (usually the html.WebStatus page) is visible. This
618 # typically uses the port number set in the Waterfall 'status' entry, but
619 # with an externally-visible host name which the buildbot cannot figure out
622 c['buildbotURL'] = ini.get("general", "buildbot_url")
627 # This specifies what database buildbot uses to store its state. You can leave
628 # this at its default for all but the largest installations.
629 'db_url' : "sqlite:///state.sqlite",