2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
36 for section in ini.sections():
37 if section.startswith("slave "):
38 if ini.has_option(section, "name") and ini.has_option(section, "password"):
39 name = ini.get(section, "name")
40 password = ini.get(section, "password")
42 if ini.has_option(section, "builds"):
43 max_builds[name] = ini.getint(section, "builds")
44 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
47 # This must match the value configured into the buildslaves (with their
49 c['slavePortnum'] = slave_port
52 c['mergeRequests'] = True
56 home_dir = os.path.abspath(ini.get("general", "homedir"))
60 if ini.has_option("general", "expire"):
61 tree_expire = ini.getint("general", "expire")
63 if ini.has_option("general", "other_builds"):
64 other_builds = ini.getint("general", "other_builds")
66 repo_url = ini.get("repo", "url")
67 repo_branch = "master"
69 if ini.has_option("repo", "branch"):
70 repo_branch = ini.get("repo", "branch")
72 rsync_bin_url = ini.get("rsync", "binary_url")
73 rsync_bin_key = ini.get("rsync", "binary_password")
78 if ini.has_option("rsync", "source_url"):
79 rsync_src_url = ini.get("rsync", "source_url")
80 rsync_src_key = ini.get("rsync", "source_password")
84 gpg_comment = "Unattended build signature"
85 gpg_passfile = "/dev/null"
87 if ini.has_option("gpg", "home"):
88 gpg_home = ini.get("gpg", "home")
90 if ini.has_option("gpg", "keyid"):
91 gpg_keyid = ini.get("gpg", "keyid")
93 if ini.has_option("gpg", "comment"):
94 gpg_comment = ini.get("gpg", "comment")
96 if ini.has_option("gpg", "passfile"):
97 gpg_passfile = ini.get("gpg", "passfile")
103 if not os.path.isdir(home_dir+'/source.git'):
104 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
106 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
108 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
109 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
112 line = findtargets.stdout.readline()
115 ta = line.strip().split(' ')
116 targets.append(ta[0])
119 # the 'change_source' setting tells the buildmaster how it should find out
120 # about source code changes. Here we point to the buildbot clone of pyflakes.
122 from buildbot.changes.gitpoller import GitPoller
123 c['change_source'] = []
124 c['change_source'].append(GitPoller(
126 workdir=home_dir+'/work.git', branch=repo_branch,
131 # Configure the Schedulers, which decide how to react to incoming changes. In this
132 # case, just kick off a 'basebuild' build
134 from buildbot.schedulers.basic import SingleBranchScheduler
135 from buildbot.schedulers.forcesched import ForceScheduler
136 from buildbot.changes import filter
138 c['schedulers'].append(SingleBranchScheduler(
140 change_filter=filter.ChangeFilter(branch=repo_branch),
142 builderNames=targets))
144 c['schedulers'].append(ForceScheduler(
146 builderNames=targets))
150 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
151 # what steps, and which slaves can execute them. Note that any particular build will
152 # only take place on one slave.
154 from buildbot.process.factory import BuildFactory
155 from buildbot.steps.source.git import Git
156 from buildbot.steps.shell import ShellCommand
157 from buildbot.steps.shell import SetProperty
158 from buildbot.steps.transfer import FileUpload
159 from buildbot.steps.transfer import FileDownload
160 from buildbot.steps.master import MasterShellCommand
161 from buildbot.process.properties import WithProperties
165 [ "tools", "tools/clean" ],
166 [ "chain", "toolchain/clean" ],
167 [ "linux", "target/linux/clean" ],
168 [ "dir", "dirclean" ],
169 [ "dist", "distclean" ]
172 def IsCleanRequested(pattern):
173 def CheckCleanProperty(step):
174 val = step.getProperty("clean")
175 if val and re.match(pattern, val):
180 return CheckCleanProperty
182 def IsTaggingRequested(step):
183 val = step.getProperty("tag")
184 if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val):
189 def IsNoTaggingRequested(step):
190 return not IsTaggingRequested(step)
192 def IsNoMasterBuild(step):
193 return repo_branch != "master"
195 def GetBaseVersion(props):
196 if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
197 return repo_branch.split('-')[1]
201 def GetVersionPrefix(props):
202 basever = GetBaseVersion(props)
203 if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", props["tag"]):
204 return "%s/" % props["tag"]
205 elif basever != "master":
206 return "%s-SNAPSHOT/" % basever
210 def GetNumJobs(props):
211 if props.hasProperty("slavename") and props.hasProperty("nproc"):
212 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
219 dlLock = locks.SlaveLock("slave_dl")
220 tagLock = locks.MasterLock("make_tag")
222 checkBuiltin = re.sub('[\t\n ]+', ' ', """
224 local symbol op path file;
225 for file in $CHANGED_FILES; do
231 while read symbol op path; do
232 case "$symbol" in package-*)
233 symbol="${symbol##*(}";
234 symbol="${symbol%)}";
235 for file in $CHANGED_FILES; do
236 case "$file" in "package/$path/"*)
237 grep -qsx "$symbol=y" .config && return 0
241 done < tmp/.packagedeps;
247 class IfBuiltinShellCommand(ShellCommand):
248 def _quote(self, str):
249 if re.search("[^a-zA-Z0-9/_.-]", str):
250 return "'%s'" %(re.sub("'", "'\"'\"'", str))
253 def setCommand(self, command):
254 if not isinstance(command, (str, unicode)):
255 command = ' '.join(map(self._quote, command))
258 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
261 def setupEnvironment(self, cmd):
262 slaveEnv = self.slaveEnvironment
266 for request in self.build.requests:
267 for source in request.sources:
268 for change in source.changes:
269 for file in change.files:
270 changedFiles[file] = True
271 fullSlaveEnv = slaveEnv.copy()
272 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
273 cmd.args['env'] = fullSlaveEnv
277 for slave in c['slaves']:
278 slaveNames.append(slave.slavename)
280 for target in targets:
281 ts = target.split('/')
283 factory = BuildFactory()
285 # find number of cores
286 factory.addStep(SetProperty(
289 description = "Finding number of CPUs",
290 command = ["nproc"]))
292 # expire tree if needed
294 factory.addStep(FileDownload(
295 mastersrc = "expire.sh",
296 slavedest = "../expire.sh",
299 factory.addStep(ShellCommand(
301 description = "Checking for build tree expiry",
302 command = ["./expire.sh", str(tree_expire)],
304 haltOnFailure = True,
307 # user-requested clean targets
308 for tuple in CleanTargetMap:
309 factory.addStep(ShellCommand(
311 description = 'User-requested "make %s"' % tuple[1],
312 command = ["make", tuple[1], "V=s"],
313 doStepIf = IsCleanRequested(tuple[0])
316 factory.addStep(MasterShellCommand(
318 description = "Tagging Git repository",
319 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
320 '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
321 path = home_dir+'/source.git',
322 env = {'GNUPGHOME': gpg_home},
323 haltOnFailure = True,
324 doStepIf = IsTaggingRequested,
325 locks = [tagLock.access('exclusive')]
329 factory.addStep(ShellCommand(
330 name = "switchbranch",
331 description = "Checking out Git branch",
332 command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch,
333 haltOnFailure = True,
334 doStepIf = IsNoTaggingRequested
337 # check out the source
340 branch = repo_branch,
341 mode = 'incremental',
345 factory.addStep(ShellCommand(
347 description = "Fetching Git remote refs",
348 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
353 factory.addStep(ShellCommand(
355 description = "Fetching Git tags",
356 command = ["git", "fetch", "--tags", "--", repo_url],
357 haltOnFailure = True,
358 doStepIf = IsTaggingRequested
362 factory.addStep(ShellCommand(
364 description = "Checking out Git tag",
365 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
366 haltOnFailure = True,
367 doStepIf = IsTaggingRequested
370 factory.addStep(ShellCommand(
372 description = "Remove tmp folder",
373 command=["rm", "-rf", "tmp/"]))
376 # factory.addStep(ShellCommand(
377 # name = "feedsconf",
378 # description = "Copy the feeds.conf",
379 # command='''cp ~/feeds.conf ./feeds.conf''' ))
382 factory.addStep(ShellCommand(
383 name = "rmfeedlinks",
384 description = "Remove feed symlinks",
385 command=["rm", "-rf", "package/feeds/"]))
388 factory.addStep(ShellCommand(
389 name = "updatefeeds",
390 description = "Updating feeds",
391 command=["./scripts/feeds", "update"]))
394 factory.addStep(ShellCommand(
395 name = "installfeeds",
396 description = "Installing feeds",
397 command=["./scripts/feeds", "install", "-a"]))
400 factory.addStep(FileDownload(
401 mastersrc = "config.seed",
402 slavedest = ".config",
407 factory.addStep(ShellCommand(
409 description = "Seeding .config",
410 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
413 factory.addStep(ShellCommand(
415 description = "Removing output directory",
416 command = ["rm", "-rf", "bin/"]
419 factory.addStep(ShellCommand(
421 description = "Populating .config",
422 command = ["make", "defconfig"]
426 factory.addStep(ShellCommand(
428 description = "Checking architecture",
429 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
437 factory.addStep(SetProperty(
440 description = "Finding libc suffix",
441 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
444 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
445 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
448 factory.addStep(ShellCommand(
450 description = "Preparing dl/",
451 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
457 factory.addStep(ShellCommand(
459 description = "Building GNU tar",
460 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/install", "V=s"],
465 factory.addStep(ShellCommand(
467 description = "Populating dl/",
468 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
470 locks = [dlLock.access('exclusive')]
473 factory.addStep(ShellCommand(
475 description = "Cleaning base-files",
476 command=["make", "package/base-files/clean", "V=s"]
480 factory.addStep(ShellCommand(
482 description = "Building tools",
483 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
487 factory.addStep(ShellCommand(
489 description = "Building toolchain",
490 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
494 factory.addStep(ShellCommand(
496 description = "Building kmods",
497 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
498 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
502 factory.addStep(ShellCommand(
504 description = "Building packages",
505 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
506 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
510 # factory.addStep(IfBuiltinShellCommand(
511 factory.addStep(ShellCommand(
513 description = "Installing packages",
514 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
518 factory.addStep(ShellCommand(
520 description = "Indexing packages",
521 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
525 #factory.addStep(IfBuiltinShellCommand(
526 factory.addStep(ShellCommand(
528 description = "Building images",
529 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
533 factory.addStep(ShellCommand(
535 description = "Generating config.seed",
536 command=["make", "-j1", "diffconfig", "V=s"],
540 factory.addStep(ShellCommand(
542 description = "Calculating checksums",
543 command=["make", "-j1", "checksum", "V=s"],
548 if gpg_keyid is not None:
549 factory.addStep(MasterShellCommand(
550 name = "signprepare",
551 description = "Preparing temporary signing directory",
552 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
556 factory.addStep(ShellCommand(
558 description = "Packing files to sign",
559 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
563 factory.addStep(FileUpload(
564 slavesrc = "sign.tar.gz",
565 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
569 factory.addStep(MasterShellCommand(
571 description = "Signing files",
572 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
573 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
577 factory.addStep(FileDownload(
578 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
579 slavedest = "sign.tar.gz",
583 factory.addStep(ShellCommand(
585 description = "Unpacking signed files",
586 command = ["tar", "-xzf", "sign.tar.gz"],
591 factory.addStep(ShellCommand(
593 description = "Preparing upload directory structure",
594 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
598 factory.addStep(ShellCommand(
599 name = "linkprepare",
600 description = "Preparing repository symlink",
601 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
602 doStepIf = IsNoMasterBuild,
606 factory.addStep(ShellCommand(
608 description = "Uploading directory structure",
609 command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
610 env={'RSYNC_PASSWORD': rsync_bin_key},
611 haltOnFailure = True,
615 factory.addStep(ShellCommand(
616 name = "targetupload",
617 description = "Uploading target files",
618 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
619 "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
620 WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
621 env={'RSYNC_PASSWORD': rsync_bin_key},
622 haltOnFailure = True,
626 if rsync_src_url is not None:
627 factory.addStep(ShellCommand(
628 name = "sourceupload",
629 description = "Uploading source archives",
630 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
631 env={'RSYNC_PASSWORD': rsync_src_key},
632 haltOnFailure = True,
637 factory.addStep(ShellCommand(
638 name = "packageupload",
639 description = "Uploading package files",
640 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
641 env={'RSYNC_PASSWORD': rsync_bin_key},
642 haltOnFailure = False,
648 factory.addStep(ShellCommand(
650 description = "Uploading logs",
651 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
652 env={'RSYNC_PASSWORD': rsync_bin_key},
653 haltOnFailure = False,
658 from buildbot.config import BuilderConfig
660 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
663 ####### STATUS TARGETS
665 # 'status' is a list of Status Targets. The results of each build will be
666 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
667 # including web pages, email senders, and IRC bots.
671 from buildbot.status import html
672 from buildbot.status.web import authz, auth
674 if ini.has_option("status", "bind"):
675 if ini.has_option("status", "user") and ini.has_option("status", "password"):
676 authz_cfg=authz.Authz(
677 # change any of these to True to enable; see the manual for more
679 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
680 gracefulShutdown = 'auth',
681 forceBuild = 'auth', # use this to test your slave once it is set up
682 forceAllBuilds = 'auth',
685 stopAllBuilds = 'auth',
686 cancelPendingBuild = 'auth',
688 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
690 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
693 from buildbot.status import words
695 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
696 irc_host = ini.get("irc", "host")
698 irc_chan = ini.get("irc", "channel")
699 irc_nick = ini.get("irc", "nickname")
702 if ini.has_option("irc", "port"):
703 irc_port = ini.getint("irc", "port")
705 if ini.has_option("irc", "password"):
706 irc_pass = ini.get("irc", "password")
708 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
709 channels = [{ "channel": irc_chan }],
712 'successToFailure': 1,
713 'failureToSuccess': 1
717 c['status'].append(irc)
720 ####### PROJECT IDENTITY
722 # the 'title' string will appear at the top of this buildbot
723 # installation's html.WebStatus home page (linked to the
724 # 'titleURL') and is embedded in the title of the waterfall HTML page.
726 c['title'] = ini.get("general", "title")
727 c['titleURL'] = ini.get("general", "title_url")
729 # the 'buildbotURL' string should point to the location where the buildbot's
730 # internal web server (usually the html.WebStatus page) is visible. This
731 # typically uses the port number set in the Waterfall 'status' entry, but
732 # with an externally-visible host name which the buildbot cannot figure out
735 c['buildbotURL'] = ini.get("general", "buildbot_url")
740 # This specifies what database buildbot uses to store its state. You can leave
741 # this at its default for all but the largest installations.
742 'db_url' : "sqlite:///state.sqlite",