2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
36 for section in ini.sections():
37 if section.startswith("slave "):
38 if ini.has_option(section, "name") and ini.has_option(section, "password"):
39 name = ini.get(section, "name")
40 password = ini.get(section, "password")
42 if ini.has_option(section, "builds"):
43 max_builds[name] = ini.getint(section, "builds")
44 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
47 # This must match the value configured into the buildslaves (with their
49 c['slavePortnum'] = slave_port
52 c['mergeRequests'] = True
54 # Reduce amount of backlog data
55 c['buildHorizon'] = 30
60 home_dir = os.path.abspath(ini.get("general", "homedir"))
68 if ini.has_option("general", "expire"):
69 tree_expire = ini.getint("general", "expire")
71 if ini.has_option("general", "other_builds"):
72 other_builds = ini.getint("general", "other_builds")
74 if ini.has_option("general", "cc_version"):
75 cc_version = ini.get("general", "cc_version").split()
76 if len(cc_version) == 1:
77 cc_version = ["eq", cc_version[0]]
79 repo_url = ini.get("repo", "url")
80 repo_branch = "master"
82 if ini.has_option("repo", "branch"):
83 repo_branch = ini.get("repo", "branch")
85 rsync_bin_url = ini.get("rsync", "binary_url")
86 rsync_bin_key = ini.get("rsync", "binary_password")
91 if ini.has_option("rsync", "source_url"):
92 rsync_src_url = ini.get("rsync", "source_url")
93 rsync_src_key = ini.get("rsync", "source_password")
97 gpg_comment = "Unattended build signature"
98 gpg_passfile = "/dev/null"
100 if ini.has_option("gpg", "home"):
101 gpg_home = ini.get("gpg", "home")
103 if ini.has_option("gpg", "keyid"):
104 gpg_keyid = ini.get("gpg", "keyid")
106 if ini.has_option("gpg", "comment"):
107 gpg_comment = ini.get("gpg", "comment")
109 if ini.has_option("gpg", "passfile"):
110 gpg_passfile = ini.get("gpg", "passfile")
116 if not os.path.isdir(home_dir+'/source.git'):
117 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
119 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
121 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
122 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
125 line = findtargets.stdout.readline()
128 ta = line.strip().split(' ')
129 targets.append(ta[0])
132 # the 'change_source' setting tells the buildmaster how it should find out
133 # about source code changes. Here we point to the buildbot clone of pyflakes.
135 from buildbot.changes.gitpoller import GitPoller
136 c['change_source'] = []
137 c['change_source'].append(GitPoller(
139 workdir=home_dir+'/work.git', branch=repo_branch,
144 # Configure the Schedulers, which decide how to react to incoming changes. In this
145 # case, just kick off a 'basebuild' build
147 from buildbot.schedulers.basic import SingleBranchScheduler
148 from buildbot.schedulers.forcesched import ForceScheduler
149 from buildbot.changes import filter
151 c['schedulers'].append(SingleBranchScheduler(
153 change_filter=filter.ChangeFilter(branch=repo_branch),
155 builderNames=targets))
157 c['schedulers'].append(ForceScheduler(
159 builderNames=targets))
163 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
164 # what steps, and which slaves can execute them. Note that any particular build will
165 # only take place on one slave.
167 from buildbot.process.factory import BuildFactory
168 from buildbot.steps.source.git import Git
169 from buildbot.steps.shell import ShellCommand
170 from buildbot.steps.shell import SetProperty
171 from buildbot.steps.transfer import FileUpload
172 from buildbot.steps.transfer import FileDownload
173 from buildbot.steps.master import MasterShellCommand
174 from buildbot.process.properties import WithProperties
178 [ "tools", "tools/clean" ],
179 [ "chain", "toolchain/clean" ],
180 [ "linux", "target/linux/clean" ],
181 [ "dir", "dirclean" ],
182 [ "dist", "distclean" ]
185 def IsCleanRequested(pattern):
186 def CheckCleanProperty(step):
187 val = step.getProperty("clean")
188 if val and re.match(pattern, val):
193 return CheckCleanProperty
195 def IsTaggingRequested(step):
196 val = step.getProperty("tag")
197 if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
202 def IsNoTaggingRequested(step):
203 return not IsTaggingRequested(step)
205 def IsNoMasterBuild(step):
206 return repo_branch != "master"
208 def GetBaseVersion(props):
209 if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
210 return repo_branch.split('-')[1]
214 def GetVersionPrefix(props):
215 basever = GetBaseVersion(props)
216 if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
217 return "%s/" % props["tag"]
218 elif basever != "master":
219 return "%s-SNAPSHOT/" % basever
223 def GetNumJobs(props):
224 if props.hasProperty("slavename") and props.hasProperty("nproc"):
225 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
230 if props.hasProperty("cc_command"):
231 return props["cc_command"]
236 if props.hasProperty("cxx_command"):
237 return props["cxx_command"]
242 if props.hasProperty("builddir"):
243 return props["builddir"]
244 elif props.hasProperty("workdir"):
245 return props["workdir"]
249 def MakeEnv(overrides=None):
251 'CC': WithProperties("%(cc)s", cc=GetCC),
252 'CXX': WithProperties("%(cxx)s", cxx=GetCXX),
253 'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)
255 if overrides is not None:
256 env.update(overrides)
262 dlLock = locks.SlaveLock("slave_dl")
263 tagLock = locks.MasterLock("make_tag")
265 checkBuiltin = re.sub('[\t\n ]+', ' ', """
267 local symbol op path file;
268 for file in $CHANGED_FILES; do
274 while read symbol op path; do
275 case "$symbol" in package-*)
276 symbol="${symbol##*(}";
277 symbol="${symbol%)}";
278 for file in $CHANGED_FILES; do
279 case "$file" in "package/$path/"*)
280 grep -qsx "$symbol=y" .config && return 0
284 done < tmp/.packagedeps;
290 class IfBuiltinShellCommand(ShellCommand):
291 def _quote(self, str):
292 if re.search("[^a-zA-Z0-9/_.-]", str):
293 return "'%s'" %(re.sub("'", "'\"'\"'", str))
296 def setCommand(self, command):
297 if not isinstance(command, (str, unicode)):
298 command = ' '.join(map(self._quote, command))
301 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
304 def setupEnvironment(self, cmd):
305 slaveEnv = self.slaveEnvironment
309 for request in self.build.requests:
310 for source in request.sources:
311 for change in source.changes:
312 for file in change.files:
313 changedFiles[file] = True
314 fullSlaveEnv = slaveEnv.copy()
315 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
316 cmd.args['env'] = fullSlaveEnv
320 for slave in c['slaves']:
321 slaveNames.append(slave.slavename)
323 for target in targets:
324 ts = target.split('/')
326 factory = BuildFactory()
328 # find number of cores
329 factory.addStep(SetProperty(
332 description = "Finding number of CPUs",
333 command = ["nproc"]))
335 # find gcc and g++ compilers
336 if cc_version is not None:
337 factory.addStep(FileDownload(
338 mastersrc = "findbin.pl",
339 slavedest = "../findbin.pl",
342 factory.addStep(SetProperty(
344 property = "cc_command",
345 description = "Finding gcc command",
346 command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
347 haltOnFailure = True))
349 factory.addStep(SetProperty(
351 property = "cxx_command",
352 description = "Finding g++ command",
353 command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
354 haltOnFailure = True))
356 # expire tree if needed
358 factory.addStep(FileDownload(
359 mastersrc = "expire.sh",
360 slavedest = "../expire.sh",
363 factory.addStep(ShellCommand(
365 description = "Checking for build tree expiry",
366 command = ["./expire.sh", str(tree_expire)],
368 haltOnFailure = True,
371 # user-requested clean targets
372 for tuple in CleanTargetMap:
373 factory.addStep(ShellCommand(
375 description = 'User-requested "make %s"' % tuple[1],
376 command = ["make", tuple[1], "V=s"],
378 doStepIf = IsCleanRequested(tuple[0])
381 factory.addStep(MasterShellCommand(
383 description = "Tagging Git repository",
384 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
385 '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
386 path = home_dir+'/source.git',
387 env = {'GNUPGHOME': gpg_home},
388 haltOnFailure = True,
389 doStepIf = IsTaggingRequested,
390 locks = [tagLock.access('exclusive')]
394 factory.addStep(ShellCommand(
395 name = "switchbranch",
396 description = "Checking out Git branch",
397 command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch,
398 haltOnFailure = True,
399 doStepIf = IsNoTaggingRequested
402 # check out the source
405 branch = repo_branch,
406 mode = 'incremental',
410 factory.addStep(ShellCommand(
412 description = "Fetching Git remote refs",
413 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
418 factory.addStep(ShellCommand(
420 description = "Fetching Git tags",
421 command = ["git", "fetch", "--tags", "--", repo_url],
422 haltOnFailure = True,
423 doStepIf = IsTaggingRequested
427 factory.addStep(ShellCommand(
429 description = "Checking out Git tag",
430 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
431 haltOnFailure = True,
432 doStepIf = IsTaggingRequested
435 factory.addStep(ShellCommand(
437 description = "Remove tmp folder",
438 command=["rm", "-rf", "tmp/"]))
441 # factory.addStep(ShellCommand(
442 # name = "feedsconf",
443 # description = "Copy the feeds.conf",
444 # command='''cp ~/feeds.conf ./feeds.conf''' ))
447 factory.addStep(ShellCommand(
448 name = "rmfeedlinks",
449 description = "Remove feed symlinks",
450 command=["rm", "-rf", "package/feeds/"]))
453 factory.addStep(ShellCommand(
454 name = "updatefeeds",
455 description = "Updating feeds",
456 command=["./scripts/feeds", "update"],
460 factory.addStep(ShellCommand(
461 name = "installfeeds",
462 description = "Installing feeds",
463 command=["./scripts/feeds", "install", "-a"],
467 factory.addStep(FileDownload(
468 mastersrc = "config.seed",
469 slavedest = ".config",
474 factory.addStep(ShellCommand(
476 description = "Seeding .config",
477 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
480 factory.addStep(ShellCommand(
482 description = "Removing output directory",
483 command = ["rm", "-rf", "bin/"]
486 factory.addStep(ShellCommand(
488 description = "Populating .config",
489 command = ["make", "defconfig"],
494 factory.addStep(ShellCommand(
496 description = "Checking architecture",
497 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
505 factory.addStep(SetProperty(
508 description = "Finding libc suffix",
509 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
512 factory.addStep(FileDownload(
513 mastersrc = "ccache.sh",
514 slavedest = "ccache.sh",
519 factory.addStep(ShellCommand(
521 description = "Preparing ccache",
522 command = ["./ccache.sh"]
526 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
527 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
530 factory.addStep(ShellCommand(
532 description = "Preparing dl/",
533 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
539 factory.addStep(ShellCommand(
541 description = "Building GNU tar",
542 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
548 factory.addStep(ShellCommand(
550 description = "Populating dl/",
551 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
554 locks = [dlLock.access('exclusive')]
557 factory.addStep(ShellCommand(
559 description = "Cleaning base-files",
560 command=["make", "package/base-files/clean", "V=s"]
564 factory.addStep(ShellCommand(
566 description = "Building tools",
567 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
572 factory.addStep(ShellCommand(
574 description = "Building toolchain",
575 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
580 factory.addStep(ShellCommand(
582 description = "Building kmods",
583 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
585 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
589 factory.addStep(ShellCommand(
591 description = "Cleaning up package build",
592 command=["make", "package/cleanup", "V=s"]
595 factory.addStep(ShellCommand(
597 description = "Building packages",
598 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
600 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
604 # factory.addStep(IfBuiltinShellCommand(
605 factory.addStep(ShellCommand(
607 description = "Installing packages",
608 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
613 factory.addStep(ShellCommand(
615 description = "Indexing packages",
616 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
621 #factory.addStep(IfBuiltinShellCommand(
622 factory.addStep(ShellCommand(
624 description = "Building images",
625 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
630 factory.addStep(ShellCommand(
632 description = "Generating config.seed",
633 command=["make", "-j1", "diffconfig", "V=s"],
638 factory.addStep(ShellCommand(
640 description = "Calculating checksums",
641 command=["make", "-j1", "checksum", "V=s"],
647 if gpg_keyid is not None:
648 factory.addStep(MasterShellCommand(
649 name = "signprepare",
650 description = "Preparing temporary signing directory",
651 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
655 factory.addStep(ShellCommand(
657 description = "Packing files to sign",
658 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
662 factory.addStep(FileUpload(
663 slavesrc = "sign.tar.gz",
664 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
668 factory.addStep(MasterShellCommand(
670 description = "Signing files",
671 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
672 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
676 factory.addStep(FileDownload(
677 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
678 slavedest = "sign.tar.gz",
682 factory.addStep(ShellCommand(
684 description = "Unpacking signed files",
685 command = ["tar", "-xzf", "sign.tar.gz"],
690 factory.addStep(ShellCommand(
692 description = "Preparing upload directory structure",
693 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
697 factory.addStep(ShellCommand(
698 name = "linkprepare",
699 description = "Preparing repository symlink",
700 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
701 doStepIf = IsNoMasterBuild,
705 factory.addStep(ShellCommand(
707 description = "Uploading directory structure",
708 command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
709 env={'RSYNC_PASSWORD': rsync_bin_key},
710 haltOnFailure = True,
714 factory.addStep(ShellCommand(
715 name = "targetupload",
716 description = "Uploading target files",
717 command=["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
718 "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
719 WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
720 env={'RSYNC_PASSWORD': rsync_bin_key},
721 haltOnFailure = True,
725 if rsync_src_url is not None:
726 factory.addStep(ShellCommand(
727 name = "sourceupload",
728 description = "Uploading source archives",
729 command=["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
730 env={'RSYNC_PASSWORD': rsync_src_key},
731 haltOnFailure = True,
736 factory.addStep(ShellCommand(
737 name = "packageupload",
738 description = "Uploading package files",
739 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
740 env={'RSYNC_PASSWORD': rsync_bin_key},
741 haltOnFailure = False,
747 factory.addStep(ShellCommand(
749 description = "Uploading logs",
750 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
751 env={'RSYNC_PASSWORD': rsync_bin_key},
752 haltOnFailure = False,
757 from buildbot.config import BuilderConfig
759 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
762 ####### STATUS TARGETS
764 # 'status' is a list of Status Targets. The results of each build will be
765 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
766 # including web pages, email senders, and IRC bots.
770 from buildbot.status import html
771 from buildbot.status.web import authz, auth
773 if ini.has_option("status", "bind"):
774 if ini.has_option("status", "user") and ini.has_option("status", "password"):
775 authz_cfg=authz.Authz(
776 # change any of these to True to enable; see the manual for more
778 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
779 gracefulShutdown = 'auth',
780 forceBuild = 'auth', # use this to test your slave once it is set up
781 forceAllBuilds = 'auth',
784 stopAllBuilds = 'auth',
785 cancelPendingBuild = 'auth',
787 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
789 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
792 from buildbot.status import words
794 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
795 irc_host = ini.get("irc", "host")
797 irc_chan = ini.get("irc", "channel")
798 irc_nick = ini.get("irc", "nickname")
801 if ini.has_option("irc", "port"):
802 irc_port = ini.getint("irc", "port")
804 if ini.has_option("irc", "password"):
805 irc_pass = ini.get("irc", "password")
807 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
808 channels = [{ "channel": irc_chan }],
811 'successToFailure': 1,
812 'failureToSuccess': 1
816 c['status'].append(irc)
819 ####### PROJECT IDENTITY
821 # the 'title' string will appear at the top of this buildbot
822 # installation's html.WebStatus home page (linked to the
823 # 'titleURL') and is embedded in the title of the waterfall HTML page.
825 c['title'] = ini.get("general", "title")
826 c['titleURL'] = ini.get("general", "title_url")
828 # the 'buildbotURL' string should point to the location where the buildbot's
829 # internal web server (usually the html.WebStatus page) is visible. This
830 # typically uses the port number set in the Waterfall 'status' entry, but
831 # with an externally-visible host name which the buildbot cannot figure out
834 c['buildbotURL'] = ini.get("general", "buildbot_url")
839 # This specifies what database buildbot uses to store its state. You can leave
840 # this at its default for all but the largest installations.
841 'db_url' : "sqlite:///state.sqlite",