2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
43 c['slavePortnum'] = 9989
46 c['mergeRequests'] = True
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
52 repo_url = ini.get("repo", "url")
54 rsync_bin_url = ini.get("rsync", "binary_url")
55 rsync_bin_key = ini.get("rsync", "binary_password")
60 if ini.has_option("rsync", "source_url"):
61 rsync_src_url = ini.get("rsync", "source_url")
62 rsync_src_key = ini.get("rsync", "source_password")
67 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
68 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
71 line = findtargets.stdout.readline()
74 ta = line.strip().split(' ')
78 # the 'change_source' setting tells the buildmaster how it should find out
79 # about source code changes. Here we point to the buildbot clone of pyflakes.
81 from buildbot.changes.gitpoller import GitPoller
82 c['change_source'] = []
83 c['change_source'].append(GitPoller(
85 workdir=home_dir+'/source.git', branch='master',
90 # Configure the Schedulers, which decide how to react to incoming changes. In this
91 # case, just kick off a 'basebuild' build
93 from buildbot.schedulers.basic import SingleBranchScheduler
94 from buildbot.schedulers.forcesched import ForceScheduler
95 from buildbot.changes import filter
97 c['schedulers'].append(SingleBranchScheduler(
99 change_filter=filter.ChangeFilter(branch='master'),
101 builderNames=targets))
103 c['schedulers'].append(ForceScheduler(
105 builderNames=targets))
109 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
110 # what steps, and which slaves can execute them. Note that any particular build will
111 # only take place on one slave.
113 from buildbot.process.factory import BuildFactory
114 from buildbot.steps.source import Git
115 from buildbot.steps.shell import ShellCommand
116 from buildbot.steps.shell import SetProperty
117 from buildbot.steps.transfer import FileDownload
118 from buildbot.process.properties import WithProperties
122 "^tools/": "tools/clean",
123 "^toolchain/": "toolchain/clean",
124 "^target/linux/": "target/linux/clean",
125 "^(config|include)/": "dirclean"
128 def IsAffected(pattern):
129 def CheckAffected(change):
130 for request in change.build.requests:
131 for source in request.sources:
132 for change in source.changes:
133 for file in change.files:
134 if re.match(pattern, file):
139 def isPathBuiltin(path):
142 conf = open(".config", "r")
145 line = conf.readline()
148 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
150 incl[m.group(1)] = True
154 deps = open("tmp/.packagedeps", "r")
157 line = deps.readline()
160 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
161 if m and incl.get(m.group(1)) == True:
162 pkgs["package/%s" % m.group(2)] = True
167 if pkgs.get(path) == True:
169 path = os.path.dirname(path)
173 def isChangeBuiltin(change):
175 # for request in change.build.requests:
176 # for source in request.sources:
177 # for change in source.changes:
178 # for file in change.files:
179 # if isPathBuiltin(file):
186 dlLock = locks.SlaveLock("slave_dl")
188 checkBuiltin = re.sub('[\t\n ]+', ' ', """
190 local symbol op path file;
191 for file in $CHANGED_FILES; do
197 while read symbol op path; do
198 case "$symbol" in package-*)
199 symbol="${symbol##*(}";
200 symbol="${symbol%)}";
201 for file in $CHANGED_FILES; do
202 case "$file" in "package/$path/"*)
203 grep -qsx "$symbol=y" .config && return 0
207 done < tmp/.packagedeps;
213 class IfBuiltinShellCommand(ShellCommand):
214 def _quote(self, str):
215 if re.search("[^a-zA-Z0-9/_.-]", str):
216 return "'%s'" %(re.sub("'", "'\"'\"'", str))
219 def setCommand(self, command):
220 if not isinstance(command, (str, unicode)):
221 command = ' '.join(map(self._quote, command))
224 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
227 def setupEnvironment(self, cmd):
228 slaveEnv = self.slaveEnvironment
232 for request in self.build.requests:
233 for source in request.sources:
234 for change in source.changes:
235 for file in change.files:
236 changedFiles[file] = True
237 fullSlaveEnv = slaveEnv.copy()
238 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
239 cmd.args['env'] = fullSlaveEnv
243 for slave in c['slaves']:
244 slaveNames.append(slave.slavename)
246 for target in targets:
247 ts = target.split('/')
249 factory = BuildFactory()
251 # find number of cores
252 factory.addStep(SetProperty(
255 description = "Finding number of CPUs",
256 command = ["nproc"]))
258 # check out the source
259 factory.addStep(Git(repourl=repo_url, mode='update'))
261 factory.addStep(ShellCommand(
263 description = "Remove tmp folder",
264 command=["rm", "-rf", "tmp/"]))
267 # factory.addStep(ShellCommand(
268 # name = "feedsconf",
269 # description = "Copy the feeds.conf",
270 # command='''cp ~/feeds.conf ./feeds.conf''' ))
273 factory.addStep(ShellCommand(
274 name = "updatefeeds",
275 description = "Updating feeds",
276 command=["./scripts/feeds", "update"]))
279 factory.addStep(ShellCommand(
280 name = "installfeeds",
281 description = "Installing feeds",
282 command=["./scripts/feeds", "install", "-a"]))
285 factory.addStep(ShellCommand(
287 description = "Seeding .config",
288 command='''cat <<EOT > .config
290 CONFIG_TARGET_%s_%s=y
291 CONFIG_ALL_NONSHARED=y
294 # CONFIG_IB_STANDALONE is not set
297 CONFIG_SIGNED_PACKAGES=y
298 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
299 CONFIG_KERNEL_KALLSYMS=y
300 CONFIG_COLLECT_KERNEL_DEBUG=y
301 EOT''' %(ts[0], ts[0], ts[1]) ))
303 factory.addStep(ShellCommand(
305 description = "Removing output directory",
306 command = ["rm", "-rf", "bin/"]
309 factory.addStep(ShellCommand(
311 description = "Populating .config",
312 command = ["make", "defconfig"]
316 factory.addStep(ShellCommand(
318 description = "Checking architecture",
319 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
327 factory.addStep(SetProperty(
330 description = "Finding libc suffix",
331 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
334 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
335 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
338 factory.addStep(ShellCommand(
340 description = "Preparing dl/",
341 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
347 factory.addStep(ShellCommand(
349 description = "Populating dl/",
350 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
352 locks = [dlLock.access('exclusive')]
355 factory.addStep(ShellCommand(
357 description = "Cleaning base-files",
358 command=["make", "package/base-files/clean", "V=s"]
361 # optional clean steps
362 for pattern, maketarget in MakeTargetMap.items():
363 factory.addStep(ShellCommand(
365 description = maketarget,
366 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
370 factory.addStep(ShellCommand(
372 description = "Building tools",
373 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
377 factory.addStep(ShellCommand(
379 description = "Building toolchain",
380 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
384 factory.addStep(ShellCommand(
386 description = "Building kmods",
387 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
388 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
392 factory.addStep(ShellCommand(
394 description = "Building packages",
395 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
396 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
400 # factory.addStep(IfBuiltinShellCommand(
401 factory.addStep(ShellCommand(
403 description = "Installing packages",
404 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
405 doStepIf = isChangeBuiltin,
409 factory.addStep(ShellCommand(
411 description = "Indexing packages",
412 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
416 #factory.addStep(IfBuiltinShellCommand(
417 factory.addStep(ShellCommand(
419 description = "Building images",
420 command=["make", "-j1", "target/install", "V=s"],
421 doStepIf = isChangeBuiltin,
426 factory.addStep(ShellCommand(
427 name = "uploadprepare",
428 description = "Preparing target directory",
429 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
430 env={'RSYNC_PASSWORD': rsync_bin_key},
431 haltOnFailure = True,
435 factory.addStep(ShellCommand(
436 name = "targetupload",
437 description = "Uploading target files",
438 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/targets/%s/%s%s/" %(ts[0], ts[1], WithProperties("%(libc)s")), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
439 env={'RSYNC_PASSWORD': rsync_bin_key},
440 haltOnFailure = True,
444 if rsync_src_url is not None:
445 factory.addStep(ShellCommand(
446 name = "sourceupload",
447 description = "Uploading source archives",
448 command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
449 env={'RSYNC_PASSWORD': rsync_src_key},
450 haltOnFailure = True,
455 factory.addStep(ShellCommand(
456 name = "packageupload",
457 description = "Uploading package files",
458 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
459 env={'RSYNC_PASSWORD': rsync_bin_key},
460 haltOnFailure = False,
466 factory.addStep(ShellCommand(
468 description = "Uploading logs",
469 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
470 env={'RSYNC_PASSWORD': rsync_bin_key},
471 haltOnFailure = False,
476 from buildbot.config import BuilderConfig
478 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
481 ####### STATUS TARGETS
483 # 'status' is a list of Status Targets. The results of each build will be
484 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
485 # including web pages, email senders, and IRC bots.
489 from buildbot.status import html
490 from buildbot.status.web import authz, auth
492 if ini.has_option("status", "bind"):
493 if ini.has_option("status", "user") and ini.has_option("status", "password"):
494 authz_cfg=authz.Authz(
495 # change any of these to True to enable; see the manual for more
497 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
498 gracefulShutdown = 'auth',
499 forceBuild = 'auth', # use this to test your slave once it is set up
500 forceAllBuilds = 'auth',
503 stopAllBuilds = 'auth',
504 cancelPendingBuild = 'auth',
506 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
508 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
511 from buildbot.status import words
513 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
514 irc_host = ini.get("irc", "host")
516 irc_chan = ini.get("irc", "channel")
517 irc_nick = ini.get("irc", "nickname")
520 if ini.has_option("irc", "port"):
521 irc_port = ini.getint("irc", "port")
523 if ini.has_option("irc", "password"):
524 irc_pass = ini.get("irc", "password")
526 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
527 channels = [{ "channel": irc_chan }],
530 'successToFailure': 1,
531 'failureToSuccess': 1
535 c['status'].append(irc)
538 ####### PROJECT IDENTITY
540 # the 'title' string will appear at the top of this buildbot
541 # installation's html.WebStatus home page (linked to the
542 # 'titleURL') and is embedded in the title of the waterfall HTML page.
544 c['title'] = ini.get("general", "title")
545 c['titleURL'] = ini.get("general", "title_url")
547 # the 'buildbotURL' string should point to the location where the buildbot's
548 # internal web server (usually the html.WebStatus page) is visible. This
549 # typically uses the port number set in the Waterfall 'status' entry, but
550 # with an externally-visible host name which the buildbot cannot figure out
553 c['buildbotURL'] = ini.get("general", "buildbot_url")
558 # This specifies what database buildbot uses to store its state. You can leave
559 # this at its default for all but the largest installations.
560 'db_url' : "sqlite:///state.sqlite",