2 # ex: set syntax=python:
10 from buildbot import locks
11 from buildbot.changes import filter
12 from buildbot.changes.gitpoller import GitPoller
13 from buildbot.config import BuilderConfig
14 from buildbot.plugins import reporters
15 from buildbot.plugins import schedulers
16 from buildbot.plugins import steps
17 from buildbot.plugins import util
18 from buildbot.process import properties
19 from buildbot.process.factory import BuildFactory
20 from buildbot.process.properties import Interpolate
21 from buildbot.process.properties import Property
22 from buildbot.schedulers.basic import SingleBranchScheduler
23 from buildbot.schedulers.forcesched import BaseParameter
24 from buildbot.schedulers.forcesched import ForceScheduler
25 from buildbot.schedulers.forcesched import ValidationError
26 from buildbot.steps.master import MasterShellCommand
27 from buildbot.steps.shell import SetPropertyFromCommand
28 from buildbot.steps.shell import ShellCommand
29 from buildbot.steps.source.git import Git
30 from buildbot.steps.transfer import FileDownload
31 from buildbot.steps.transfer import FileUpload
32 from buildbot.steps.transfer import StringDownload
33 from buildbot.worker import Worker
36 # This is a sample buildmaster config file. It must be installed as
37 # 'master.cfg' in your buildmaster's base directory.
39 ini = configparser.ConfigParser()
40 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
42 # This is the dictionary that the buildmaster pays attention to. We also use
43 # a shorter alias to save typing.
44 c = BuildmasterConfig = {}
46 ####### PROJECT IDENTITY
48 # the 'title' string will appear at the top of this buildbot
49 # installation's html.WebStatus home page (linked to the
50 # 'titleURL') and is embedded in the title of the waterfall HTML page.
52 c['title'] = ini.get("general", "title")
53 c['titleURL'] = ini.get("general", "title_url")
55 # the 'buildbotURL' string should point to the location where the buildbot's
56 # internal web server (usually the html.WebStatus page) is visible. This
57 # typically uses the port number set in the Waterfall 'status' entry, but
58 # with an externally-visible host name which the buildbot cannot figure out
61 c['buildbotURL'] = ini.get("phase1", "buildbot_url")
65 # The 'workers' list defines the set of recognized buildslaves. Each element is
66 # a Worker object, specifying a unique slave name and password. The same
67 # slave name and password must be configured on the slave.
71 if ini.has_option("phase1", "port"):
72 slave_port = ini.get("phase1", "port")
77 for section in ini.sections():
78 if section.startswith("slave "):
79 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
80 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
81 sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
82 name = ini.get(section, "name")
83 password = ini.get(section, "password")
85 if ini.has_option(section, "builds"):
86 max_builds = ini.getint(section, "builds")
87 sl_props['max_builds'] = max_builds
89 sl_props['shared_wd'] = True
90 if ini.has_option(section, "cleanup"):
91 sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
92 if ini.has_option(section, "dl_lock"):
93 lockname = ini.get(section, "dl_lock")
94 sl_props['dl_lock'] = lockname
95 if lockname not in NetLocks:
96 NetLocks[lockname] = locks.MasterLock(lockname)
97 if ini.has_option(section, "ul_lock"):
98 lockname = ini.get(section, "dl_lock")
99 sl_props['ul_lock'] = lockname
100 if lockname not in NetLocks:
101 NetLocks[lockname] = locks.MasterLock(lockname)
102 if ini.has_option(section, "shared_wd"):
103 shared_wd = ini.getboolean(section, "shared_wd")
104 sl_props['shared_wd'] = shared_wd
105 if shared_wd and (max_builds != 1):
106 raise ValueError('max_builds must be 1 with shared workdir!')
107 c['workers'].append(Worker(name, password, max_builds = max_builds, properties = sl_props))
109 # 'slavePortnum' defines the TCP port to listen on for connections from workers.
110 # This must match the value configured into the buildslaves (with their
112 c['protocols'] = {'pb': {'port': slave_port}}
115 c['collapseRequests'] = True
117 # Reduce amount of backlog data
118 c['buildHorizon'] = 30
121 ####### CHANGESOURCES
123 work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
124 scripts_dir = os.path.abspath("../scripts")
137 if ini.has_option("phase1", "expire"):
138 tree_expire = ini.getint("phase1", "expire")
140 if ini.has_option("phase1", "other_builds"):
141 other_builds = ini.getint("phase1", "other_builds")
143 if ini.has_option("phase1", "cc_version"):
144 cc_version = ini.get("phase1", "cc_version").split()
145 if len(cc_version) == 1:
146 cc_version = ["eq", cc_version[0]]
148 if ini.has_option("general", "git_ssh"):
149 git_ssh = ini.getboolean("general", "git_ssh")
151 if ini.has_option("general", "git_ssh_key"):
152 git_ssh_key = ini.get("general", "git_ssh_key")
156 if ini.has_option("phase1", "config_seed"):
157 config_seed = ini.get("phase1", "config_seed")
159 repo_url = ini.get("repo", "url")
160 repo_branch = "master"
162 if ini.has_option("repo", "branch"):
163 repo_branch = ini.get("repo", "branch")
165 rsync_bin_url = ini.get("rsync", "binary_url")
166 rsync_bin_key = ini.get("rsync", "binary_password")
167 rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
169 if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
170 rsync_bin_defopts += ["--contimeout=20"]
174 rsync_src_defopts = ["-v", "-4", "--timeout=120"]
176 if ini.has_option("rsync", "source_url"):
177 rsync_src_url = ini.get("rsync", "source_url")
178 rsync_src_key = ini.get("rsync", "source_password")
180 if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
181 rsync_src_defopts += ["--contimeout=20"]
184 usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
186 if ini.has_option("usign", "key"):
187 usign_key = ini.get("usign", "key")
189 if ini.has_option("usign", "comment"):
190 usign_comment = ini.get("usign", "comment")
192 enable_kmod_archive = False
194 if ini.has_option("phase1", "kmod_archive"):
195 enable_kmod_archive = ini.getboolean("phase1", "kmod_archive")
201 if not os.path.isdir(work_dir+'/source.git'):
202 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
204 subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
206 findtargets = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'targets'],
207 stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
210 line = findtargets.stdout.readline()
213 ta = line.decode().strip().split(' ')
214 targets.append(ta[0])
217 # the 'change_source' setting tells the buildmaster how it should find out
218 # about source code changes. Here we point to the buildbot clone of pyflakes.
220 c['change_source'] = []
221 c['change_source'].append(GitPoller(
223 workdir=work_dir+'/work.git', branch=repo_branch,
228 # Configure the Schedulers, which decide how to react to incoming changes. In this
229 # case, just kick off a 'basebuild' build
231 class TagChoiceParameter(BaseParameter):
232 spec_attributes = ["strict", "choices"]
236 def __init__(self, name, label=None, **kw):
237 super().__init__(name, label, **kw)
238 self._choice_list = []
243 basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch)
246 findtags = subprocess.Popen(
247 ['git', 'ls-remote', '--tags', repo_url],
248 stdout = subprocess.PIPE)
251 line = findtags.stdout.readline()
256 tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
258 if tagver and tagver[1].find(basever[1]) == 0:
259 taglist.append(tagver[1])
261 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
262 taglist.insert(0, '')
264 self._choice_list = taglist
266 return self._choice_list
268 def parse_from_arg(self, s):
269 if self.strict and s not in self._choice_list:
270 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
274 c['schedulers'].append(SingleBranchScheduler(
276 change_filter = filter.ChangeFilter(branch=repo_branch),
277 treeStableTimer = 60,
278 builderNames = targets))
280 c['schedulers'].append(ForceScheduler(
282 buttonName = "Force builds",
283 label = "Force build details",
284 builderNames = [ "00_force_build" ],
287 util.CodebaseParameter(
289 label = "Repository",
290 branch = util.FixedParameter(name = "branch", default = ""),
291 revision = util.FixedParameter(name = "revision", default = ""),
292 repository = util.FixedParameter(name = "repository", default = ""),
293 project = util.FixedParameter(name = "project", default = "")
297 reason = util.StringParameter(
300 default = "Trigger build",
306 util.NestedParameter(
308 label="Build Options",
311 util.ChoiceStringParameter(
313 label = "Build target",
315 choices = [ "all" ] + targets
329 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
330 # what steps, and which workers can execute them. Note that any particular build will
331 # only take place on one slave.
334 [ "tools", "tools/clean" ],
335 [ "chain", "toolchain/clean" ],
336 [ "linux", "target/linux/clean" ],
337 [ "dir", "dirclean" ],
338 [ "dist", "distclean" ]
341 def IsMakeCleanRequested(pattern):
342 def CheckCleanProperty(step):
343 val = step.getProperty("clean")
344 if val and re.match(pattern, val):
349 return CheckCleanProperty
351 def IsSharedWorkdir(step):
352 return bool(step.getProperty("shared_wd"))
354 def IsCleanupRequested(step):
355 if IsSharedWorkdir(step):
357 do_cleanup = step.getProperty("do_cleanup")
363 def IsExpireRequested(step):
364 if IsSharedWorkdir(step):
367 return not IsCleanupRequested(step)
369 def IsGitFreshRequested(step):
370 do_cleanup = step.getProperty("do_cleanup")
376 def IsGitCleanRequested(step):
377 return not IsGitFreshRequested(step)
379 def IsTaggingRequested(step):
380 val = step.getProperty("tag")
381 if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
386 def IsNoTaggingRequested(step):
387 return not IsTaggingRequested(step)
389 def IsNoMasterBuild(step):
390 return repo_branch != "master"
392 def GetBaseVersion():
393 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
394 return repo_branch.split('-')[1]
399 def GetVersionPrefix(props):
400 basever = GetBaseVersion()
401 if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
402 return "%s/" % props["tag"]
403 elif basever != "master":
404 return "%s-SNAPSHOT/" % basever
409 def GetNumJobs(props):
410 if props.hasProperty("max_builds") and props.hasProperty("nproc"):
411 return str(int(int(props["nproc"]) / (props["max_builds"] + other_builds)))
417 if props.hasProperty("cc_command"):
418 return props["cc_command"]
424 if props.hasProperty("cxx_command"):
425 return props["cxx_command"]
431 if props.hasProperty("builddir"):
432 return props["builddir"]
433 elif props.hasProperty("workdir"):
434 return props["workdir"]
439 def GetCCache(props):
440 if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
441 return props["ccache_command"]
445 def GetNextBuild(builder, requests):
447 if r.properties and r.properties.hasProperty("tag"):
451 def MakeEnv(overrides=None, tryccache=False):
453 'CCC': Interpolate("%(kw:cc)s", cc=GetCC),
454 'CCXX': Interpolate("%(kw:cxx)s", cxx=GetCXX),
457 env['CC'] = Interpolate("%(kw:cwd)s/ccache_cc.sh", cwd=GetCwd)
458 env['CXX'] = Interpolate("%(kw:cwd)s/ccache_cxx.sh", cwd=GetCwd)
459 env['CCACHE'] = Interpolate("%(kw:ccache)s", ccache=GetCCache)
461 env['CC'] = env['CCC']
462 env['CXX'] = env['CCXX']
464 if overrides is not None:
465 env.update(overrides)
469 def NetLockDl(props):
471 if props.hasProperty("dl_lock"):
472 lock = NetLocks[props["dl_lock"]]
474 return [lock.access('exclusive')]
479 def NetLockUl(props):
481 if props.hasProperty("ul_lock"):
482 lock = NetLocks[props["ul_lock"]]
484 return [lock.access('exclusive')]
489 def TagPropertyValue(props):
490 if props.hasProperty("options"):
491 options = props.getProperty("options")
492 if type(options) is dict:
493 return options.get("tag")
496 def IsTargetSelected(target):
497 def CheckTargetProperty(step):
499 options = step.getProperty("options")
500 if type(options) is dict:
501 selected_target = options.get("target", "all")
502 if selected_target != "all" and selected_target != target:
509 return CheckTargetProperty
511 def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
513 seckey = base64.b64decode(seckey)
517 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
518 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
523 dlLock = locks.WorkerLock("slave_dl")
525 checkBuiltin = re.sub('[\t\n ]+', ' ', """
527 local symbol op path file;
528 for file in $CHANGED_FILES; do
534 while read symbol op path; do
535 case "$symbol" in package-*)
536 symbol="${symbol##*(}";
537 symbol="${symbol%)}";
538 for file in $CHANGED_FILES; do
539 case "$file" in "package/$path/"*)
540 grep -qsx "$symbol=y" .config && return 0
544 done < tmp/.packagedeps;
550 class IfBuiltinShellCommand(ShellCommand):
551 def _quote(self, str):
552 if re.search("[^a-zA-Z0-9/_.-]", str):
553 return "'%s'" %(re.sub("'", "'\"'\"'", str))
556 def setCommand(self, command):
557 if not isinstance(command, (str, unicode)):
558 command = ' '.join(map(self._quote, command))
561 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
564 def setupEnvironment(self, cmd):
565 slaveEnv = self.slaveEnvironment
569 for request in self.build.requests:
570 for source in request.sources:
571 for change in source.changes:
572 for file in change.files:
573 changedFiles[file] = True
574 fullSlaveEnv = slaveEnv.copy()
575 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
576 cmd.args['env'] = fullSlaveEnv
580 for slave in c['workers']:
581 slaveNames.append(slave.workername)
583 force_factory = BuildFactory()
585 c['builders'].append(BuilderConfig(
586 name = "00_force_build",
587 workernames = slaveNames,
588 factory = force_factory))
590 for target in targets:
591 ts = target.split('/')
593 factory = BuildFactory()
595 # setup shared work directory if required
596 factory.addStep(ShellCommand(
598 description = "Setting up shared work directory",
599 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
601 haltOnFailure = True,
602 doStepIf = IsSharedWorkdir))
604 # find number of cores
605 factory.addStep(SetPropertyFromCommand(
608 description = "Finding number of CPUs",
609 command = ["nproc"]))
611 # find gcc and g++ compilers
612 factory.addStep(FileDownload(
613 name = "dlfindbinpl",
614 mastersrc = scripts_dir + '/findbin.pl',
615 workerdest = "../findbin.pl",
618 factory.addStep(SetPropertyFromCommand(
620 property = "cc_command",
621 description = "Finding gcc command",
623 "../findbin.pl", "gcc",
624 cc_version[0] if cc_version is not None else '',
625 cc_version[1] if cc_version is not None else ''
627 haltOnFailure = True))
629 factory.addStep(SetPropertyFromCommand(
631 property = "cxx_command",
632 description = "Finding g++ command",
634 "../findbin.pl", "g++",
635 cc_version[0] if cc_version is not None else '',
636 cc_version[1] if cc_version is not None else ''
638 haltOnFailure = True))
640 # see if ccache is available
641 factory.addStep(SetPropertyFromCommand(
642 property = "ccache_command",
643 command = ["which", "ccache"],
644 description = "Testing for ccache command",
645 haltOnFailure = False,
646 flunkOnFailure = False,
647 warnOnFailure = False,
650 # expire tree if needed
652 factory.addStep(FileDownload(
654 doStepIf = IsExpireRequested,
655 mastersrc = scripts_dir + '/expire.sh',
656 workerdest = "../expire.sh",
659 factory.addStep(ShellCommand(
661 description = "Checking for build tree expiry",
662 command = ["./expire.sh", str(tree_expire)],
664 haltOnFailure = True,
665 doStepIf = IsExpireRequested,
668 # cleanup.sh if needed
669 factory.addStep(FileDownload(
670 name = "dlcleanupsh",
671 mastersrc = scripts_dir + '/cleanup.sh',
672 workerdest = "../cleanup.sh",
674 doStepIf = IsCleanupRequested))
676 factory.addStep(ShellCommand(
678 description = "Cleaning previous builds",
679 command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "full"],
681 haltOnFailure = True,
682 doStepIf = IsCleanupRequested,
685 factory.addStep(ShellCommand(
687 description = "Cleaning work area",
688 command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "single"],
690 haltOnFailure = True,
691 doStepIf = IsCleanupRequested,
694 # user-requested clean targets
695 for tuple in CleanTargetMap:
696 factory.addStep(ShellCommand(
698 description = 'User-requested "make %s"' % tuple[1],
699 command = ["make", tuple[1], "V=s"],
701 doStepIf = IsMakeCleanRequested(tuple[0])
704 # Workaround bug when switching from a checked out tag back to a branch
705 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
706 factory.addStep(ShellCommand(
707 name = "gitcheckout",
708 description = "Ensure that Git HEAD is sane",
709 command = "if [ -d .git ]; then git checkout -f %s; git branch --set-upstream-to origin/%s; else exit 0; fi" %(repo_branch, repo_branch),
710 haltOnFailure = True))
712 # check out the source
714 # if repo doesn't exist: 'git clone repourl'
715 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
716 # 'git fetch -t repourl branch; git reset --hard revision'
717 # Git() parameters can't take a renderer until buildbot 0.8.10, so we have to split the fresh and clean cases
718 # if buildbot is updated, one can use: method = Interpolate('%(prop:do_cleanup:#?|fresh|clean)s')
722 branch = repo_branch,
725 haltOnFailure = True,
726 doStepIf = IsGitCleanRequested,
732 branch = repo_branch,
735 haltOnFailure = True,
736 doStepIf = IsGitFreshRequested,
740 factory.addStep(ShellCommand(
742 description = "Fetching Git remote refs",
743 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
748 factory.addStep(ShellCommand(
750 description = "Checking out Git tag",
751 command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
752 haltOnFailure = True,
753 doStepIf = IsTaggingRequested
756 # Verify that Git HEAD points to a tag or branch
757 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
758 factory.addStep(ShellCommand(
760 description = "Ensure that Git HEAD is pointing to a branch or tag",
761 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
762 haltOnFailure = True))
764 factory.addStep(ShellCommand(
766 description = "Remove tmp folder",
767 command=["rm", "-rf", "tmp/"]))
770 # factory.addStep(ShellCommand(
771 # name = "feedsconf",
772 # description = "Copy the feeds.conf",
773 # command='''cp ~/feeds.conf ./feeds.conf''' ))
776 factory.addStep(ShellCommand(
777 name = "rmfeedlinks",
778 description = "Remove feed symlinks",
779 command=["rm", "-rf", "package/feeds/"]))
781 factory.addStep(StringDownload(
783 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
784 workerdest = "../ccache_cc.sh",
788 factory.addStep(StringDownload(
790 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
791 workerdest = "../ccache_cxx.sh",
797 factory.addStep(StringDownload(
798 name = "dlgitclonekey",
800 workerdest = "../git-clone.key",
804 factory.addStep(ShellCommand(
805 name = "patchfeedsconf",
806 description = "Patching feeds.conf",
807 command="sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf",
812 factory.addStep(ShellCommand(
813 name = "updatefeeds",
814 description = "Updating feeds",
815 command=["./scripts/feeds", "update"],
816 env = MakeEnv(tryccache=True, overrides={'GIT_SSH_COMMAND': Interpolate("ssh -o IdentitiesOnly=yes -o IdentityFile=%(kw:cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {}),
822 factory.addStep(ShellCommand(
823 name = "rmfeedsconf",
824 description = "Removing feeds.conf",
825 command=["rm", "feeds.conf"],
830 factory.addStep(ShellCommand(
831 name = "installfeeds",
832 description = "Installing feeds",
833 command=["./scripts/feeds", "install", "-a"],
834 env = MakeEnv(tryccache=True),
839 if config_seed is not None:
840 factory.addStep(StringDownload(
841 name = "dlconfigseed",
842 s = config_seed + '\n',
843 workerdest = ".config",
848 factory.addStep(ShellCommand(
850 description = "Seeding .config",
851 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\nCONFIG_SIGNED_PACKAGES=%s\\n' >> .config" %(ts[0], ts[0], ts[1], 'y' if usign_key is not None else 'n')
854 factory.addStep(ShellCommand(
856 description = "Removing output directory",
857 command = ["rm", "-rf", "bin/"]
860 factory.addStep(ShellCommand(
862 description = "Populating .config",
863 command = ["make", "defconfig"],
868 factory.addStep(ShellCommand(
870 description = "Checking architecture",
871 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
879 factory.addStep(SetPropertyFromCommand(
882 description = "Finding libc suffix",
883 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
886 if usign_key is not None:
887 factory.addStep(StringDownload(
888 name = "dlkeybuildpub",
889 s = UsignSec2Pub(usign_key, usign_comment),
890 workerdest = "key-build.pub",
894 factory.addStep(StringDownload(
896 s = "# fake private key",
897 workerdest = "key-build",
901 factory.addStep(StringDownload(
902 name = "dlkeybuilducert",
903 s = "# fake certificate",
904 workerdest = "key-build.ucert",
909 factory.addStep(ShellCommand(
911 description = "Preparing dl/",
912 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
918 factory.addStep(ShellCommand(
920 description = "Building and installing GNU tar",
921 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
922 env = MakeEnv(tryccache=True),
927 factory.addStep(ShellCommand(
929 description = "Populating dl/",
930 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
933 locks = [dlLock.access('exclusive')],
936 factory.addStep(ShellCommand(
938 description = "Cleaning base-files",
939 command=["make", "package/base-files/clean", "V=s"]
943 factory.addStep(ShellCommand(
945 description = "Building and installing tools",
946 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
947 env = MakeEnv(tryccache=True),
951 factory.addStep(ShellCommand(
953 description = "Building and installing toolchain",
954 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
959 factory.addStep(ShellCommand(
961 description = "Building kmods",
962 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
964 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
968 # find kernel version
969 factory.addStep(SetPropertyFromCommand(
970 name = "kernelversion",
971 property = "kernelversion",
972 description = "Finding the effective Kernel version",
973 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
974 env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
977 factory.addStep(ShellCommand(
979 description = "Cleaning up package build",
980 command=["make", "package/cleanup", "V=s"]
983 factory.addStep(ShellCommand(
985 description = "Building packages",
986 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
988 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
992 # factory.addStep(IfBuiltinShellCommand(
993 factory.addStep(ShellCommand(
995 description = "Installing packages",
996 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
1001 factory.addStep(ShellCommand(
1003 description = "Indexing packages",
1004 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
1006 haltOnFailure = True
1009 if enable_kmod_archive:
1010 # embed kmod repository. Must happen before 'images'
1012 # find rootfs staging directory
1013 factory.addStep(SetPropertyFromCommand(
1015 property = "stageroot",
1016 description = "Finding the rootfs staging directory",
1017 command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
1018 env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
1021 factory.addStep(ShellCommand(
1023 description = "Creating file overlay directory",
1024 command=["mkdir", "-p", "files/etc/opkg"],
1025 haltOnFailure = True
1028 factory.addStep(ShellCommand(
1029 name = "kmodconfig",
1030 description = "Embedding kmod repository configuration",
1031 command=Interpolate("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(prop:kernelversion)s#' " +
1032 "%(prop:stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
1033 haltOnFailure = True
1036 #factory.addStep(IfBuiltinShellCommand(
1037 factory.addStep(ShellCommand(
1039 description = "Building and installing images",
1040 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
1042 haltOnFailure = True
1045 factory.addStep(ShellCommand(
1047 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
1048 command = "make -j1 buildinfo V=s || true",
1050 haltOnFailure = True
1053 factory.addStep(ShellCommand(
1054 name = "json_overview_image_info",
1055 description = "Generate profiles.json in target folder",
1056 command = "make -j1 json_overview_image_info V=s || true",
1058 haltOnFailure = True
1061 factory.addStep(ShellCommand(
1063 description = "Calculating checksums",
1064 command=["make", "-j1", "checksum", "V=s"],
1066 haltOnFailure = True
1069 if enable_kmod_archive:
1070 factory.addStep(ShellCommand(
1072 description = "Creating kmod directory",
1073 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
1074 haltOnFailure = True
1077 factory.addStep(ShellCommand(
1078 name = "kmodprepare",
1079 description = "Preparing kmod archive",
1080 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
1081 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
1082 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1083 haltOnFailure = True
1086 factory.addStep(ShellCommand(
1088 description = "Indexing kmod archive",
1089 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
1090 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1092 haltOnFailure = True
1096 if ini.has_option("gpg", "key") or usign_key is not None:
1097 factory.addStep(MasterShellCommand(
1098 name = "signprepare",
1099 description = "Preparing temporary signing directory",
1100 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1101 haltOnFailure = True
1104 factory.addStep(ShellCommand(
1106 description = "Packing files to sign",
1107 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1108 haltOnFailure = True
1111 factory.addStep(FileUpload(
1112 workersrc = "sign.tar.gz",
1113 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1114 haltOnFailure = True
1117 factory.addStep(MasterShellCommand(
1119 description = "Signing files",
1120 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
1121 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1122 haltOnFailure = True
1125 factory.addStep(FileDownload(
1126 name = "dlsigntargz",
1127 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1128 workerdest = "sign.tar.gz",
1129 haltOnFailure = True
1132 factory.addStep(ShellCommand(
1133 name = "signunpack",
1134 description = "Unpacking signed files",
1135 command = ["tar", "-xzf", "sign.tar.gz"],
1136 haltOnFailure = True
1140 factory.addStep(ShellCommand(
1141 name = "dirprepare",
1142 description = "Preparing upload directory structure",
1143 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1144 haltOnFailure = True
1147 factory.addStep(ShellCommand(
1148 name = "linkprepare",
1149 description = "Preparing repository symlink",
1150 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=GetBaseVersion()), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1151 doStepIf = IsNoMasterBuild,
1152 haltOnFailure = True
1155 if enable_kmod_archive:
1156 factory.addStep(ShellCommand(
1157 name = "kmoddirprepare",
1158 description = "Preparing kmod archive upload directory",
1159 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1160 haltOnFailure = True
1163 factory.addStep(ShellCommand(
1165 description = "Uploading directory structure",
1166 command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
1167 env={'RSYNC_PASSWORD': rsync_bin_key},
1168 haltOnFailure = True,
1172 # download remote sha256sums to 'target-sha256sums'
1173 factory.addStep(ShellCommand(
1174 name = "target-sha256sums",
1175 description = "Fetching remote sha256sums for target",
1176 command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1177 env={'RSYNC_PASSWORD': rsync_bin_key},
1179 haltOnFailure = False,
1180 flunkOnFailure = False,
1181 warnOnFailure = False,
1184 # build list of files to upload
1185 factory.addStep(FileDownload(
1186 name = "dlsha2rsyncpl",
1187 mastersrc = scripts_dir + '/sha2rsync.pl',
1188 workerdest = "../sha2rsync.pl",
1192 factory.addStep(ShellCommand(
1194 description = "Building list of files to upload",
1195 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1196 haltOnFailure = True,
1199 factory.addStep(FileDownload(
1200 name = "dlrsync.sh",
1201 mastersrc = scripts_dir + '/rsync.sh',
1202 workerdest = "../rsync.sh",
1206 # upload new files and update existing ones
1207 factory.addStep(ShellCommand(
1208 name = "targetupload",
1209 description = "Uploading target files",
1210 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1211 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1212 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1213 env={'RSYNC_PASSWORD': rsync_bin_key},
1214 haltOnFailure = True,
1218 # delete files which don't exist locally
1219 factory.addStep(ShellCommand(
1220 name = "targetprune",
1221 description = "Pruning target files",
1222 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1223 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1224 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1225 env={'RSYNC_PASSWORD': rsync_bin_key},
1226 haltOnFailure = True,
1230 if enable_kmod_archive:
1231 factory.addStep(ShellCommand(
1232 name = "kmodupload",
1233 description = "Uploading kmod archive",
1234 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1235 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1236 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1237 env={'RSYNC_PASSWORD': rsync_bin_key},
1238 haltOnFailure = True,
1242 if rsync_src_url is not None:
1243 factory.addStep(ShellCommand(
1244 name = "sourcelist",
1245 description = "Finding source archives to upload",
1246 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -newer .config -printf '%f\\n' > sourcelist",
1247 haltOnFailure = True
1250 factory.addStep(ShellCommand(
1251 name = "sourceupload",
1252 description = "Uploading source archives",
1253 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
1254 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
1255 env={'RSYNC_PASSWORD': rsync_src_key},
1256 haltOnFailure = True,
1261 factory.addStep(ShellCommand(
1262 name = "packageupload",
1263 description = "Uploading package files",
1264 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a"] + rsync_bin_defopts + ["bin/packages/", "%s/packages/" %(rsync_bin_url)],
1265 env={'RSYNC_PASSWORD': rsync_bin_key},
1266 haltOnFailure = False,
1272 factory.addStep(ShellCommand(
1274 description = "Uploading logs",
1275 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az"] + rsync_bin_defopts + ["logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
1276 env={'RSYNC_PASSWORD': rsync_bin_key},
1277 haltOnFailure = False,
1282 factory.addStep(ShellCommand(
1284 description = "Reporting disk usage",
1285 command=["df", "-h", "."],
1286 env={'LC_ALL': 'C'},
1287 haltOnFailure = False,
1291 factory.addStep(ShellCommand(
1292 name = "ccachestat",
1293 description = "Reporting ccache stats",
1294 command=["ccache", "-s"],
1295 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1296 want_stderr = False,
1297 haltOnFailure = False,
1298 flunkOnFailure = False,
1299 warnOnFailure = False,
1303 c['builders'].append(BuilderConfig(name=target, workernames=slaveNames, factory=factory, nextBuild=GetNextBuild))
1305 c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1306 force_factory.addStep(steps.Trigger(
1307 name = "trigger_%s" % target,
1308 description = "Triggering %s build" % target,
1309 schedulerNames = [ "trigger_%s" % target ],
1310 set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
1311 doStepIf = IsTargetSelected(target)
1315 ####### STATUS TARGETS
1317 # 'status' is a list of Status Targets. The results of each build will be
1318 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1319 # including web pages, email senders, and IRC bots.
1321 if ini.has_option("phase1", "status_bind"):
1323 'port': ini.get("phase1", "status_bind"),
1325 'waterfall_view': True,
1326 'console_view': True,
1331 if ini.has_option("phase1", "status_user") and ini.has_option("phase1", "status_password"):
1332 c['www']['auth'] = util.UserPasswordAuth([
1333 (ini.get("phase1", "status_user"), ini.get("phase1", "status_password"))
1335 c['www']['authz'] = util.Authz(
1336 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1337 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[ini.get("phase1", "status_user")]) ]
1341 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
1342 irc_host = ini.get("irc", "host")
1344 irc_chan = ini.get("irc", "channel")
1345 irc_nick = ini.get("irc", "nickname")
1348 if ini.has_option("irc", "port"):
1349 irc_port = ini.getint("irc", "port")
1351 if ini.has_option("irc", "password"):
1352 irc_pass = ini.get("irc", "password")
1354 irc = reporters.IRC(irc_host, irc_nick,
1356 password = irc_pass,
1357 channels = [ irc_chan ],
1358 notify_events = [ 'exception', 'problem', 'recovery' ]
1361 c['services'].append(irc)
1366 # This specifies what database buildbot uses to store its state. You can leave
1367 # this at its default for all but the largest installations.
1368 'db_url' : "sqlite:///state.sqlite",
1371 c['buildbotNetUsageData'] = None