2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
36 for section in ini.sections():
37 if section.startswith("slave "):
38 if ini.has_option(section, "name") and ini.has_option(section, "password"):
39 name = ini.get(section, "name")
40 password = ini.get(section, "password")
42 if ini.has_option(section, "builds"):
43 max_builds[name] = ini.getint(section, "builds")
44 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
47 # This must match the value configured into the buildslaves (with their
49 c['slavePortnum'] = slave_port
52 c['mergeRequests'] = True
54 # Reduce amount of backlog data
55 c['buildHorizon'] = 30
60 home_dir = os.path.abspath(ini.get("general", "homedir"))
68 if ini.has_option("general", "expire"):
69 tree_expire = ini.getint("general", "expire")
71 if ini.has_option("general", "other_builds"):
72 other_builds = ini.getint("general", "other_builds")
74 if ini.has_option("general", "cc_version"):
75 cc_version = ini.get("general", "cc_version").split()
76 if len(cc_version) == 1:
77 cc_version = ["eq", cc_version[0]]
79 repo_url = ini.get("repo", "url")
80 repo_branch = "master"
82 if ini.has_option("repo", "branch"):
83 repo_branch = ini.get("repo", "branch")
85 rsync_bin_url = ini.get("rsync", "binary_url")
86 rsync_bin_key = ini.get("rsync", "binary_password")
91 if ini.has_option("rsync", "source_url"):
92 rsync_src_url = ini.get("rsync", "source_url")
93 rsync_src_key = ini.get("rsync", "source_password")
97 gpg_comment = "Unattended build signature"
98 gpg_passfile = "/dev/null"
100 if ini.has_option("gpg", "home"):
101 gpg_home = ini.get("gpg", "home")
103 if ini.has_option("gpg", "keyid"):
104 gpg_keyid = ini.get("gpg", "keyid")
106 if ini.has_option("gpg", "comment"):
107 gpg_comment = ini.get("gpg", "comment")
109 if ini.has_option("gpg", "passfile"):
110 gpg_passfile = ini.get("gpg", "passfile")
116 if not os.path.isdir(home_dir+'/source.git'):
117 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
119 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
121 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
122 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
125 line = findtargets.stdout.readline()
128 ta = line.strip().split(' ')
129 targets.append(ta[0])
132 # the 'change_source' setting tells the buildmaster how it should find out
133 # about source code changes. Here we point to the buildbot clone of pyflakes.
135 from buildbot.changes.gitpoller import GitPoller
136 c['change_source'] = []
137 c['change_source'].append(GitPoller(
139 workdir=home_dir+'/work.git', branch=repo_branch,
144 # Configure the Schedulers, which decide how to react to incoming changes. In this
145 # case, just kick off a 'basebuild' build
147 from buildbot.schedulers.basic import SingleBranchScheduler
148 from buildbot.schedulers.forcesched import ForceScheduler
149 from buildbot.changes import filter
151 c['schedulers'].append(SingleBranchScheduler(
153 change_filter=filter.ChangeFilter(branch=repo_branch),
155 builderNames=targets))
157 c['schedulers'].append(ForceScheduler(
159 builderNames=targets))
163 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
164 # what steps, and which slaves can execute them. Note that any particular build will
165 # only take place on one slave.
167 from buildbot.process.factory import BuildFactory
168 from buildbot.steps.source.git import Git
169 from buildbot.steps.shell import ShellCommand
170 from buildbot.steps.shell import SetProperty
171 from buildbot.steps.transfer import FileUpload
172 from buildbot.steps.transfer import FileDownload
173 from buildbot.steps.master import MasterShellCommand
174 from buildbot.process.properties import WithProperties
178 [ "tools", "tools/clean" ],
179 [ "chain", "toolchain/clean" ],
180 [ "linux", "target/linux/clean" ],
181 [ "dir", "dirclean" ],
182 [ "dist", "distclean" ]
185 def IsCleanRequested(pattern):
186 def CheckCleanProperty(step):
187 val = step.getProperty("clean")
188 if val and re.match(pattern, val):
193 return CheckCleanProperty
195 def IsTaggingRequested(step):
196 val = step.getProperty("tag")
197 if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val):
202 def IsNoTaggingRequested(step):
203 return not IsTaggingRequested(step)
205 def IsNoMasterBuild(step):
206 return repo_branch != "master"
208 def GetBaseVersion(props):
209 if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
210 return repo_branch.split('-')[1]
214 def GetVersionPrefix(props):
215 basever = GetBaseVersion(props)
216 if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", props["tag"]):
217 return "%s/" % props["tag"]
218 elif basever != "master":
219 return "%s-SNAPSHOT/" % basever
223 def GetNumJobs(props):
224 if props.hasProperty("slavename") and props.hasProperty("nproc"):
225 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
230 if props.hasProperty("cc_command"):
231 return props["cc_command"]
236 if props.hasProperty("cxx_command"):
237 return props["cxx_command"]
241 def MakeEnv(overrides=None):
243 'CC': WithProperties("%(cc)s", cc=GetCC),
244 'CXX': WithProperties("%(cxx)s", cxx=GetCXX)
246 if overrides is not None:
247 env.update(overrides)
253 dlLock = locks.SlaveLock("slave_dl")
254 tagLock = locks.MasterLock("make_tag")
256 checkBuiltin = re.sub('[\t\n ]+', ' ', """
258 local symbol op path file;
259 for file in $CHANGED_FILES; do
265 while read symbol op path; do
266 case "$symbol" in package-*)
267 symbol="${symbol##*(}";
268 symbol="${symbol%)}";
269 for file in $CHANGED_FILES; do
270 case "$file" in "package/$path/"*)
271 grep -qsx "$symbol=y" .config && return 0
275 done < tmp/.packagedeps;
281 class IfBuiltinShellCommand(ShellCommand):
282 def _quote(self, str):
283 if re.search("[^a-zA-Z0-9/_.-]", str):
284 return "'%s'" %(re.sub("'", "'\"'\"'", str))
287 def setCommand(self, command):
288 if not isinstance(command, (str, unicode)):
289 command = ' '.join(map(self._quote, command))
292 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
295 def setupEnvironment(self, cmd):
296 slaveEnv = self.slaveEnvironment
300 for request in self.build.requests:
301 for source in request.sources:
302 for change in source.changes:
303 for file in change.files:
304 changedFiles[file] = True
305 fullSlaveEnv = slaveEnv.copy()
306 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
307 cmd.args['env'] = fullSlaveEnv
311 for slave in c['slaves']:
312 slaveNames.append(slave.slavename)
314 for target in targets:
315 ts = target.split('/')
317 factory = BuildFactory()
319 # find number of cores
320 factory.addStep(SetProperty(
323 description = "Finding number of CPUs",
324 command = ["nproc"]))
326 # find gcc and g++ compilers
327 if cc_version is not None:
328 factory.addStep(FileDownload(
329 mastersrc = "findbin.pl",
330 slavedest = "../findbin.pl",
333 factory.addStep(SetProperty(
335 property = "cc_command",
336 description = "Finding gcc command",
337 command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
338 haltOnFailure = True))
340 factory.addStep(SetProperty(
342 property = "cxx_command",
343 description = "Finding g++ command",
344 command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
345 haltOnFailure = True))
347 # expire tree if needed
349 factory.addStep(FileDownload(
350 mastersrc = "expire.sh",
351 slavedest = "../expire.sh",
354 factory.addStep(ShellCommand(
356 description = "Checking for build tree expiry",
357 command = ["./expire.sh", str(tree_expire)],
359 haltOnFailure = True,
362 # user-requested clean targets
363 for tuple in CleanTargetMap:
364 factory.addStep(ShellCommand(
366 description = 'User-requested "make %s"' % tuple[1],
367 command = ["make", tuple[1], "V=s"],
369 doStepIf = IsCleanRequested(tuple[0])
372 factory.addStep(MasterShellCommand(
374 description = "Tagging Git repository",
375 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
376 '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
377 path = home_dir+'/source.git',
378 env = {'GNUPGHOME': gpg_home},
379 haltOnFailure = True,
380 doStepIf = IsTaggingRequested,
381 locks = [tagLock.access('exclusive')]
385 factory.addStep(ShellCommand(
386 name = "switchbranch",
387 description = "Checking out Git branch",
388 command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch,
389 haltOnFailure = True,
390 doStepIf = IsNoTaggingRequested
393 # check out the source
396 branch = repo_branch,
397 mode = 'incremental',
401 factory.addStep(ShellCommand(
403 description = "Fetching Git remote refs",
404 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
409 factory.addStep(ShellCommand(
411 description = "Fetching Git tags",
412 command = ["git", "fetch", "--tags", "--", repo_url],
413 haltOnFailure = True,
414 doStepIf = IsTaggingRequested
418 factory.addStep(ShellCommand(
420 description = "Checking out Git tag",
421 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
422 haltOnFailure = True,
423 doStepIf = IsTaggingRequested
426 factory.addStep(ShellCommand(
428 description = "Remove tmp folder",
429 command=["rm", "-rf", "tmp/"]))
432 # factory.addStep(ShellCommand(
433 # name = "feedsconf",
434 # description = "Copy the feeds.conf",
435 # command='''cp ~/feeds.conf ./feeds.conf''' ))
438 factory.addStep(ShellCommand(
439 name = "rmfeedlinks",
440 description = "Remove feed symlinks",
441 command=["rm", "-rf", "package/feeds/"]))
444 factory.addStep(ShellCommand(
445 name = "updatefeeds",
446 description = "Updating feeds",
447 command=["./scripts/feeds", "update"],
451 factory.addStep(ShellCommand(
452 name = "installfeeds",
453 description = "Installing feeds",
454 command=["./scripts/feeds", "install", "-a"],
458 factory.addStep(ShellCommand(
460 description = "Preparing ccache",
461 command = ["./ccache.sh"]
465 factory.addStep(FileDownload(
466 mastersrc = "config.seed",
467 slavedest = ".config",
472 factory.addStep(ShellCommand(
474 description = "Seeding .config",
475 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
478 factory.addStep(ShellCommand(
480 description = "Removing output directory",
481 command = ["rm", "-rf", "bin/"]
484 factory.addStep(ShellCommand(
486 description = "Populating .config",
487 command = ["make", "defconfig"],
492 factory.addStep(ShellCommand(
494 description = "Checking architecture",
495 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
503 factory.addStep(SetProperty(
506 description = "Finding libc suffix",
507 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
510 factory.addStep(FileDownload(
511 mastersrc = "ccache.sh",
512 slavedest = "ccache.sh",
517 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
518 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
521 factory.addStep(ShellCommand(
523 description = "Preparing dl/",
524 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
530 factory.addStep(ShellCommand(
532 description = "Building GNU tar",
533 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/install", "V=s"],
539 factory.addStep(ShellCommand(
541 description = "Populating dl/",
542 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
545 locks = [dlLock.access('exclusive')]
548 factory.addStep(ShellCommand(
550 description = "Cleaning base-files",
551 command=["make", "package/base-files/clean", "V=s"]
555 factory.addStep(ShellCommand(
557 description = "Building tools",
558 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
563 factory.addStep(ShellCommand(
565 description = "Building toolchain",
566 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
571 factory.addStep(ShellCommand(
573 description = "Building kmods",
574 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
576 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
580 factory.addStep(ShellCommand(
582 description = "Cleaning up package build",
583 command=["make", "package/cleanup", "V=s"]
586 factory.addStep(ShellCommand(
588 description = "Building packages",
589 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
591 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
595 # factory.addStep(IfBuiltinShellCommand(
596 factory.addStep(ShellCommand(
598 description = "Installing packages",
599 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
604 factory.addStep(ShellCommand(
606 description = "Indexing packages",
607 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
612 #factory.addStep(IfBuiltinShellCommand(
613 factory.addStep(ShellCommand(
615 description = "Building images",
616 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
621 factory.addStep(ShellCommand(
623 description = "Generating config.seed",
624 command=["make", "-j1", "diffconfig", "V=s"],
629 factory.addStep(ShellCommand(
631 description = "Calculating checksums",
632 command=["make", "-j1", "checksum", "V=s"],
638 if gpg_keyid is not None:
639 factory.addStep(MasterShellCommand(
640 name = "signprepare",
641 description = "Preparing temporary signing directory",
642 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
646 factory.addStep(ShellCommand(
648 description = "Packing files to sign",
649 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
653 factory.addStep(FileUpload(
654 slavesrc = "sign.tar.gz",
655 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
659 factory.addStep(MasterShellCommand(
661 description = "Signing files",
662 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
663 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
667 factory.addStep(FileDownload(
668 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
669 slavedest = "sign.tar.gz",
673 factory.addStep(ShellCommand(
675 description = "Unpacking signed files",
676 command = ["tar", "-xzf", "sign.tar.gz"],
681 factory.addStep(ShellCommand(
683 description = "Preparing upload directory structure",
684 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
688 factory.addStep(ShellCommand(
689 name = "linkprepare",
690 description = "Preparing repository symlink",
691 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
692 doStepIf = IsNoMasterBuild,
696 factory.addStep(ShellCommand(
698 description = "Uploading directory structure",
699 command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
700 env={'RSYNC_PASSWORD': rsync_bin_key},
701 haltOnFailure = True,
705 factory.addStep(ShellCommand(
706 name = "targetupload",
707 description = "Uploading target files",
708 command=["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
709 "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
710 WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
711 env={'RSYNC_PASSWORD': rsync_bin_key},
712 haltOnFailure = True,
716 if rsync_src_url is not None:
717 factory.addStep(ShellCommand(
718 name = "sourceupload",
719 description = "Uploading source archives",
720 command=["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
721 env={'RSYNC_PASSWORD': rsync_src_key},
722 haltOnFailure = True,
727 factory.addStep(ShellCommand(
728 name = "packageupload",
729 description = "Uploading package files",
730 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
731 env={'RSYNC_PASSWORD': rsync_bin_key},
732 haltOnFailure = False,
738 factory.addStep(ShellCommand(
740 description = "Uploading logs",
741 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
742 env={'RSYNC_PASSWORD': rsync_bin_key},
743 haltOnFailure = False,
748 from buildbot.config import BuilderConfig
750 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
753 ####### STATUS TARGETS
755 # 'status' is a list of Status Targets. The results of each build will be
756 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
757 # including web pages, email senders, and IRC bots.
761 from buildbot.status import html
762 from buildbot.status.web import authz, auth
764 if ini.has_option("status", "bind"):
765 if ini.has_option("status", "user") and ini.has_option("status", "password"):
766 authz_cfg=authz.Authz(
767 # change any of these to True to enable; see the manual for more
769 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
770 gracefulShutdown = 'auth',
771 forceBuild = 'auth', # use this to test your slave once it is set up
772 forceAllBuilds = 'auth',
775 stopAllBuilds = 'auth',
776 cancelPendingBuild = 'auth',
778 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
780 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
783 from buildbot.status import words
785 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
786 irc_host = ini.get("irc", "host")
788 irc_chan = ini.get("irc", "channel")
789 irc_nick = ini.get("irc", "nickname")
792 if ini.has_option("irc", "port"):
793 irc_port = ini.getint("irc", "port")
795 if ini.has_option("irc", "password"):
796 irc_pass = ini.get("irc", "password")
798 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
799 channels = [{ "channel": irc_chan }],
802 'successToFailure': 1,
803 'failureToSuccess': 1
807 c['status'].append(irc)
810 ####### PROJECT IDENTITY
812 # the 'title' string will appear at the top of this buildbot
813 # installation's html.WebStatus home page (linked to the
814 # 'titleURL') and is embedded in the title of the waterfall HTML page.
816 c['title'] = ini.get("general", "title")
817 c['titleURL'] = ini.get("general", "title_url")
819 # the 'buildbotURL' string should point to the location where the buildbot's
820 # internal web server (usually the html.WebStatus page) is visible. This
821 # typically uses the port number set in the Waterfall 'status' entry, but
822 # with an externally-visible host name which the buildbot cannot figure out
825 c['buildbotURL'] = ini.get("general", "buildbot_url")
830 # This specifies what database buildbot uses to store its state. You can leave
831 # this at its default for all but the largest installations.
832 'db_url' : "sqlite:///state.sqlite",