2 # ex: set syntax=python:
10 from buildbot import locks
11 from buildbot.changes import filter
12 from buildbot.changes.gitpoller import GitPoller
13 from buildbot.config import BuilderConfig
14 from buildbot.plugins import reporters
15 from buildbot.plugins import schedulers
16 from buildbot.plugins import steps
17 from buildbot.plugins import util
18 from buildbot.process import properties
19 from buildbot.process.factory import BuildFactory
20 from buildbot.process.properties import Interpolate
21 from buildbot.process.properties import Property
22 from buildbot.schedulers.basic import SingleBranchScheduler
23 from buildbot.schedulers.forcesched import BaseParameter
24 from buildbot.schedulers.forcesched import ForceScheduler
25 from buildbot.schedulers.forcesched import ValidationError
26 from buildbot.steps.master import MasterShellCommand
27 from buildbot.steps.shell import SetPropertyFromCommand
28 from buildbot.steps.shell import ShellCommand
29 from buildbot.steps.source.git import Git
30 from buildbot.steps.transfer import FileDownload
31 from buildbot.steps.transfer import FileUpload
32 from buildbot.steps.transfer import StringDownload
33 from buildbot.worker import Worker
36 # This is a sample buildmaster config file. It must be installed as
37 # 'master.cfg' in your buildmaster's base directory.
39 ini = configparser.ConfigParser()
40 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
42 # This is the dictionary that the buildmaster pays attention to. We also use
43 # a shorter alias to save typing.
44 c = BuildmasterConfig = {}
46 ####### PROJECT IDENTITY
48 # the 'title' string will appear at the top of this buildbot
49 # installation's html.WebStatus home page (linked to the
50 # 'titleURL') and is embedded in the title of the waterfall HTML page.
52 c['title'] = ini.get("general", "title")
53 c['titleURL'] = ini.get("general", "title_url")
55 # the 'buildbotURL' string should point to the location where the buildbot's
56 # internal web server (usually the html.WebStatus page) is visible. This
57 # typically uses the port number set in the Waterfall 'status' entry, but
58 # with an externally-visible host name which the buildbot cannot figure out
61 c['buildbotURL'] = ini.get("phase1", "buildbot_url")
65 # The 'workers' list defines the set of recognized buildslaves. Each element is
66 # a Worker object, specifying a unique slave name and password. The same
67 # slave name and password must be configured on the slave.
71 if ini.has_option("phase1", "port"):
72 slave_port = ini.get("phase1", "port")
77 for section in ini.sections():
78 if section.startswith("slave "):
79 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
80 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
81 sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
82 name = ini.get(section, "name")
83 password = ini.get(section, "password")
85 if ini.has_option(section, "builds"):
86 max_builds = ini.getint(section, "builds")
87 sl_props['max_builds'] = max_builds
89 sl_props['shared_wd'] = True
90 if ini.has_option(section, "cleanup"):
91 sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
92 if ini.has_option(section, "dl_lock"):
93 lockname = ini.get(section, "dl_lock")
94 sl_props['dl_lock'] = lockname
95 if lockname not in NetLocks:
96 NetLocks[lockname] = locks.MasterLock(lockname)
97 if ini.has_option(section, "ul_lock"):
98 lockname = ini.get(section, "dl_lock")
99 sl_props['ul_lock'] = lockname
100 if lockname not in NetLocks:
101 NetLocks[lockname] = locks.MasterLock(lockname)
102 if ini.has_option(section, "shared_wd"):
103 shared_wd = ini.getboolean(section, "shared_wd")
104 sl_props['shared_wd'] = shared_wd
105 if shared_wd and (max_builds != 1):
106 raise ValueError('max_builds must be 1 with shared workdir!')
107 c['workers'].append(Worker(name, password, max_builds = max_builds, properties = sl_props))
109 # 'slavePortnum' defines the TCP port to listen on for connections from workers.
110 # This must match the value configured into the buildslaves (with their
112 c['protocols'] = {'pb': {'port': slave_port}}
115 c['collapseRequests'] = True
117 # Reduce amount of backlog data
118 c['buildHorizon'] = 30
121 ####### CHANGESOURCES
123 work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
124 scripts_dir = os.path.abspath("../scripts")
137 if ini.has_option("phase1", "expire"):
138 tree_expire = ini.getint("phase1", "expire")
140 if ini.has_option("phase1", "other_builds"):
141 other_builds = ini.getint("phase1", "other_builds")
143 if ini.has_option("phase1", "cc_version"):
144 cc_version = ini.get("phase1", "cc_version").split()
145 if len(cc_version) == 1:
146 cc_version = ["eq", cc_version[0]]
148 if ini.has_option("general", "git_ssh"):
149 git_ssh = ini.getboolean("general", "git_ssh")
151 if ini.has_option("general", "git_ssh_key"):
152 git_ssh_key = ini.get("general", "git_ssh_key")
156 if ini.has_option("phase1", "config_seed"):
157 config_seed = ini.get("phase1", "config_seed")
159 repo_url = ini.get("repo", "url")
160 repo_branch = "master"
162 if ini.has_option("repo", "branch"):
163 repo_branch = ini.get("repo", "branch")
165 rsync_bin_url = ini.get("rsync", "binary_url")
166 rsync_bin_key = ini.get("rsync", "binary_password")
167 rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
169 if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
170 rsync_bin_defopts += ["--contimeout=20"]
174 rsync_src_defopts = ["-v", "-4", "--timeout=120"]
176 if ini.has_option("rsync", "source_url"):
177 rsync_src_url = ini.get("rsync", "source_url")
178 rsync_src_key = ini.get("rsync", "source_password")
180 if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
181 rsync_src_defopts += ["--contimeout=20"]
184 usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
186 if ini.has_option("usign", "key"):
187 usign_key = ini.get("usign", "key")
189 if ini.has_option("usign", "comment"):
190 usign_comment = ini.get("usign", "comment")
192 enable_kmod_archive = False
193 embed_kmod_repository = False
195 if ini.has_option("phase1", "kmod_archive"):
196 enable_kmod_archive = ini.getboolean("phase1", "kmod_archive")
198 if ini.has_option("phase1", "kmod_repository"):
199 embed_kmod_repository = ini.getboolean("phase1", "kmod_repository")
205 if not os.path.isdir(work_dir+'/source.git'):
206 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
208 subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
210 findtargets = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'targets'],
211 stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
214 line = findtargets.stdout.readline()
217 ta = line.decode().strip().split(' ')
218 targets.append(ta[0])
221 # the 'change_source' setting tells the buildmaster how it should find out
222 # about source code changes. Here we point to the buildbot clone of pyflakes.
224 c['change_source'] = []
225 c['change_source'].append(GitPoller(
227 workdir=work_dir+'/work.git', branch=repo_branch,
232 # Configure the Schedulers, which decide how to react to incoming changes. In this
233 # case, just kick off a 'basebuild' build
235 class TagChoiceParameter(BaseParameter):
236 spec_attributes = ["strict", "choices"]
240 def __init__(self, name, label=None, **kw):
241 super().__init__(name, label, **kw)
242 self._choice_list = []
247 basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch)
250 findtags = subprocess.Popen(
251 ['git', 'ls-remote', '--tags', repo_url],
252 stdout = subprocess.PIPE)
255 line = findtags.stdout.readline()
260 tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
262 if tagver and tagver[1].find(basever[1]) == 0:
263 taglist.append(tagver[1])
265 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
266 taglist.insert(0, '')
268 self._choice_list = taglist
270 return self._choice_list
272 def parse_from_arg(self, s):
273 if self.strict and s not in self._choice_list:
274 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
278 c['schedulers'].append(SingleBranchScheduler(
280 change_filter = filter.ChangeFilter(branch=repo_branch),
281 treeStableTimer = 60,
282 builderNames = targets))
284 c['schedulers'].append(ForceScheduler(
286 buttonName = "Force builds",
287 label = "Force build details",
288 builderNames = [ "00_force_build" ],
291 util.CodebaseParameter(
293 label = "Repository",
294 branch = util.FixedParameter(name = "branch", default = ""),
295 revision = util.FixedParameter(name = "revision", default = ""),
296 repository = util.FixedParameter(name = "repository", default = ""),
297 project = util.FixedParameter(name = "project", default = "")
301 reason = util.StringParameter(
304 default = "Trigger build",
310 util.NestedParameter(
312 label="Build Options",
315 util.ChoiceStringParameter(
317 label = "Build target",
319 choices = [ "all" ] + targets
333 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
334 # what steps, and which workers can execute them. Note that any particular build will
335 # only take place on one slave.
338 [ "tools", "tools/clean" ],
339 [ "chain", "toolchain/clean" ],
340 [ "linux", "target/linux/clean" ],
341 [ "dir", "dirclean" ],
342 [ "dist", "distclean" ]
345 def IsMakeCleanRequested(pattern):
346 def CheckCleanProperty(step):
347 val = step.getProperty("clean")
348 if val and re.match(pattern, val):
353 return CheckCleanProperty
355 def IsSharedWorkdir(step):
356 return bool(step.getProperty("shared_wd"))
358 def IsCleanupRequested(step):
359 if IsSharedWorkdir(step):
361 do_cleanup = step.getProperty("do_cleanup")
367 def IsExpireRequested(step):
368 if IsSharedWorkdir(step):
371 return not IsCleanupRequested(step)
373 def IsGitFreshRequested(step):
374 do_cleanup = step.getProperty("do_cleanup")
380 def IsGitCleanRequested(step):
381 return not IsGitFreshRequested(step)
383 def IsTaggingRequested(step):
384 val = step.getProperty("tag")
385 if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
390 def IsNoTaggingRequested(step):
391 return not IsTaggingRequested(step)
393 def IsNoMasterBuild(step):
394 return repo_branch != "master"
396 def GetBaseVersion():
397 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
398 return repo_branch.split('-')[1]
403 def GetVersionPrefix(props):
404 basever = GetBaseVersion()
405 if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
406 return "%s/" % props["tag"]
407 elif basever != "master":
408 return "%s-SNAPSHOT/" % basever
413 def GetNumJobs(props):
414 if props.hasProperty("max_builds") and props.hasProperty("nproc"):
415 return str(int(int(props["nproc"]) / (props["max_builds"] + other_builds)))
421 if props.hasProperty("cc_command"):
422 return props["cc_command"]
428 if props.hasProperty("cxx_command"):
429 return props["cxx_command"]
435 if props.hasProperty("builddir"):
436 return props["builddir"]
437 elif props.hasProperty("workdir"):
438 return props["workdir"]
443 def GetCCache(props):
444 if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
445 return props["ccache_command"]
449 def GetNextBuild(builder, requests):
451 if r.properties and r.properties.hasProperty("tag"):
455 def MakeEnv(overrides=None, tryccache=False):
457 'CCC': Interpolate("%(kw:cc)s", cc=GetCC),
458 'CCXX': Interpolate("%(kw:cxx)s", cxx=GetCXX),
461 env['CC'] = Interpolate("%(kw:cwd)s/ccache_cc.sh", cwd=GetCwd)
462 env['CXX'] = Interpolate("%(kw:cwd)s/ccache_cxx.sh", cwd=GetCwd)
463 env['CCACHE'] = Interpolate("%(kw:ccache)s", ccache=GetCCache)
465 env['CC'] = env['CCC']
466 env['CXX'] = env['CCXX']
468 if overrides is not None:
469 env.update(overrides)
473 def NetLockDl(props):
475 if props.hasProperty("dl_lock"):
476 lock = NetLocks[props["dl_lock"]]
478 return [lock.access('exclusive')]
483 def NetLockUl(props):
485 if props.hasProperty("ul_lock"):
486 lock = NetLocks[props["ul_lock"]]
488 return [lock.access('exclusive')]
493 def TagPropertyValue(props):
494 if props.hasProperty("options"):
495 options = props.getProperty("options")
496 if type(options) is dict:
497 return options.get("tag")
500 def IsTargetSelected(target):
501 def CheckTargetProperty(step):
503 options = step.getProperty("options")
504 if type(options) is dict:
505 selected_target = options.get("target", "all")
506 if selected_target != "all" and selected_target != target:
513 return CheckTargetProperty
515 def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
517 seckey = base64.b64decode(seckey)
521 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
522 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
527 dlLock = locks.WorkerLock("slave_dl")
529 checkBuiltin = re.sub('[\t\n ]+', ' ', """
531 local symbol op path file;
532 for file in $CHANGED_FILES; do
538 while read symbol op path; do
539 case "$symbol" in package-*)
540 symbol="${symbol##*(}";
541 symbol="${symbol%)}";
542 for file in $CHANGED_FILES; do
543 case "$file" in "package/$path/"*)
544 grep -qsx "$symbol=y" .config && return 0
548 done < tmp/.packagedeps;
554 class IfBuiltinShellCommand(ShellCommand):
555 def _quote(self, str):
556 if re.search("[^a-zA-Z0-9/_.-]", str):
557 return "'%s'" %(re.sub("'", "'\"'\"'", str))
560 def setCommand(self, command):
561 if not isinstance(command, (str, unicode)):
562 command = ' '.join(map(self._quote, command))
565 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
568 def setupEnvironment(self, cmd):
569 slaveEnv = self.slaveEnvironment
573 for request in self.build.requests:
574 for source in request.sources:
575 for change in source.changes:
576 for file in change.files:
577 changedFiles[file] = True
578 fullSlaveEnv = slaveEnv.copy()
579 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
580 cmd.args['env'] = fullSlaveEnv
584 for slave in c['workers']:
585 slaveNames.append(slave.workername)
587 force_factory = BuildFactory()
589 c['builders'].append(BuilderConfig(
590 name = "00_force_build",
591 workernames = slaveNames,
592 factory = force_factory))
594 for target in targets:
595 ts = target.split('/')
597 factory = BuildFactory()
599 # setup shared work directory if required
600 factory.addStep(ShellCommand(
602 description = "Setting up shared work directory",
603 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
605 haltOnFailure = True,
606 doStepIf = IsSharedWorkdir))
608 # find number of cores
609 factory.addStep(SetPropertyFromCommand(
612 description = "Finding number of CPUs",
613 command = ["nproc"]))
615 # find gcc and g++ compilers
616 factory.addStep(FileDownload(
617 name = "dlfindbinpl",
618 mastersrc = scripts_dir + '/findbin.pl',
619 workerdest = "../findbin.pl",
622 factory.addStep(SetPropertyFromCommand(
624 property = "cc_command",
625 description = "Finding gcc command",
627 "../findbin.pl", "gcc",
628 cc_version[0] if cc_version is not None else '',
629 cc_version[1] if cc_version is not None else ''
631 haltOnFailure = True))
633 factory.addStep(SetPropertyFromCommand(
635 property = "cxx_command",
636 description = "Finding g++ command",
638 "../findbin.pl", "g++",
639 cc_version[0] if cc_version is not None else '',
640 cc_version[1] if cc_version is not None else ''
642 haltOnFailure = True))
644 # see if ccache is available
645 factory.addStep(SetPropertyFromCommand(
646 property = "ccache_command",
647 command = ["which", "ccache"],
648 description = "Testing for ccache command",
649 haltOnFailure = False,
650 flunkOnFailure = False,
651 warnOnFailure = False,
654 # expire tree if needed
656 factory.addStep(FileDownload(
658 doStepIf = IsExpireRequested,
659 mastersrc = scripts_dir + '/expire.sh',
660 workerdest = "../expire.sh",
663 factory.addStep(ShellCommand(
665 description = "Checking for build tree expiry",
666 command = ["./expire.sh", str(tree_expire)],
668 haltOnFailure = True,
669 doStepIf = IsExpireRequested,
672 # cleanup.sh if needed
673 factory.addStep(FileDownload(
674 name = "dlcleanupsh",
675 mastersrc = scripts_dir + '/cleanup.sh',
676 workerdest = "../cleanup.sh",
678 doStepIf = IsCleanupRequested))
680 factory.addStep(ShellCommand(
682 description = "Cleaning previous builds",
683 command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "full"],
685 haltOnFailure = True,
686 doStepIf = IsCleanupRequested,
689 factory.addStep(ShellCommand(
691 description = "Cleaning work area",
692 command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "single"],
694 haltOnFailure = True,
695 doStepIf = IsCleanupRequested,
698 # user-requested clean targets
699 for tuple in CleanTargetMap:
700 factory.addStep(ShellCommand(
702 description = 'User-requested "make %s"' % tuple[1],
703 command = ["make", tuple[1], "V=s"],
705 doStepIf = IsMakeCleanRequested(tuple[0])
708 # Workaround bug when switching from a checked out tag back to a branch
709 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
710 factory.addStep(ShellCommand(
711 name = "gitcheckout",
712 description = "Ensure that Git HEAD is sane",
713 command = "if [ -d .git ]; then git checkout -f %s && git branch --set-upstream-to origin/%s || rm -fr .git; else exit 0; fi" %(repo_branch, repo_branch),
714 haltOnFailure = True))
716 # check out the source
718 # if repo doesn't exist: 'git clone repourl'
719 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
720 # 'git fetch -t repourl branch; git reset --hard revision'
721 # Git() parameters can't take a renderer until buildbot 0.8.10, so we have to split the fresh and clean cases
722 # if buildbot is updated, one can use: method = Interpolate('%(prop:do_cleanup:#?|fresh|clean)s')
726 branch = repo_branch,
729 haltOnFailure = True,
730 doStepIf = IsGitCleanRequested,
736 branch = repo_branch,
739 haltOnFailure = True,
740 doStepIf = IsGitFreshRequested,
744 factory.addStep(ShellCommand(
746 description = "Fetching Git remote refs",
747 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
752 factory.addStep(ShellCommand(
754 description = "Checking out Git tag",
755 command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
756 haltOnFailure = True,
757 doStepIf = IsTaggingRequested
760 # Verify that Git HEAD points to a tag or branch
761 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
762 factory.addStep(ShellCommand(
764 description = "Ensure that Git HEAD is pointing to a branch or tag",
765 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
766 haltOnFailure = True))
768 factory.addStep(ShellCommand(
770 description = "Remove tmp folder",
771 command=["rm", "-rf", "tmp/"]))
774 # factory.addStep(ShellCommand(
775 # name = "feedsconf",
776 # description = "Copy the feeds.conf",
777 # command='''cp ~/feeds.conf ./feeds.conf''' ))
780 factory.addStep(ShellCommand(
781 name = "rmfeedlinks",
782 description = "Remove feed symlinks",
783 command=["rm", "-rf", "package/feeds/"]))
785 factory.addStep(StringDownload(
787 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
788 workerdest = "../ccache_cc.sh",
792 factory.addStep(StringDownload(
794 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
795 workerdest = "../ccache_cxx.sh",
801 factory.addStep(StringDownload(
802 name = "dlgitclonekey",
804 workerdest = "../git-clone.key",
808 factory.addStep(ShellCommand(
809 name = "patchfeedsconf",
810 description = "Patching feeds.conf",
811 command="sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf",
816 factory.addStep(ShellCommand(
817 name = "updatefeeds",
818 description = "Updating feeds",
819 command=["./scripts/feeds", "update"],
820 env = MakeEnv(tryccache=True, overrides={'GIT_SSH_COMMAND': Interpolate("ssh -o IdentitiesOnly=yes -o IdentityFile=%(kw:cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {}),
826 factory.addStep(ShellCommand(
827 name = "rmfeedsconf",
828 description = "Removing feeds.conf",
829 command=["rm", "feeds.conf"],
834 factory.addStep(ShellCommand(
835 name = "installfeeds",
836 description = "Installing feeds",
837 command=["./scripts/feeds", "install", "-a"],
838 env = MakeEnv(tryccache=True),
843 if config_seed is not None:
844 factory.addStep(StringDownload(
845 name = "dlconfigseed",
846 s = config_seed + '\n',
847 workerdest = ".config",
852 factory.addStep(ShellCommand(
854 description = "Seeding .config",
855 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\nCONFIG_SIGNED_PACKAGES=%s\\n' >> .config" %(ts[0], ts[0], ts[1], 'y' if usign_key is not None else 'n')
858 factory.addStep(ShellCommand(
860 description = "Removing output directory",
861 command = ["rm", "-rf", "bin/"]
864 factory.addStep(ShellCommand(
866 description = "Populating .config",
867 command = ["make", "defconfig"],
872 factory.addStep(ShellCommand(
874 description = "Checking architecture",
875 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
883 factory.addStep(SetPropertyFromCommand(
886 description = "Finding libc suffix",
887 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
890 if usign_key is not None:
891 factory.addStep(StringDownload(
892 name = "dlkeybuildpub",
893 s = UsignSec2Pub(usign_key, usign_comment),
894 workerdest = "key-build.pub",
898 factory.addStep(StringDownload(
900 s = "# fake private key",
901 workerdest = "key-build",
905 factory.addStep(StringDownload(
906 name = "dlkeybuilducert",
907 s = "# fake certificate",
908 workerdest = "key-build.ucert",
913 factory.addStep(ShellCommand(
915 description = "Preparing dl/",
916 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
922 factory.addStep(ShellCommand(
924 description = "Building and installing GNU tar",
925 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
926 env = MakeEnv(tryccache=True),
931 factory.addStep(ShellCommand(
933 description = "Populating dl/",
934 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
937 locks = [dlLock.access('exclusive')],
940 factory.addStep(ShellCommand(
942 description = "Cleaning base-files",
943 command=["make", "package/base-files/clean", "V=s"]
947 factory.addStep(ShellCommand(
949 description = "Building and installing tools",
950 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
951 env = MakeEnv(tryccache=True),
955 factory.addStep(ShellCommand(
957 description = "Building and installing toolchain",
958 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
963 factory.addStep(ShellCommand(
965 description = "Building kmods",
966 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
968 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
972 # find kernel version
973 factory.addStep(SetPropertyFromCommand(
974 name = "kernelversion",
975 property = "kernelversion",
976 description = "Finding the effective Kernel version",
977 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
978 env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
981 factory.addStep(ShellCommand(
983 description = "Cleaning up package build",
984 command=["make", "package/cleanup", "V=s"]
987 factory.addStep(ShellCommand(
989 description = "Building packages",
990 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
992 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
996 # factory.addStep(IfBuiltinShellCommand(
997 factory.addStep(ShellCommand(
999 description = "Installing packages",
1000 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
1002 haltOnFailure = True
1005 factory.addStep(ShellCommand(
1007 description = "Indexing packages",
1008 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
1010 haltOnFailure = True
1013 if enable_kmod_archive and embed_kmod_repository:
1014 # embed kmod repository. Must happen before 'images'
1016 # find rootfs staging directory
1017 factory.addStep(SetPropertyFromCommand(
1019 property = "stageroot",
1020 description = "Finding the rootfs staging directory",
1021 command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
1022 env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) },
1026 factory.addStep(ShellCommand(
1028 description = "Creating file overlay directory",
1029 command=["mkdir", "-p", "files/etc/opkg"],
1030 haltOnFailure = True
1033 factory.addStep(ShellCommand(
1034 name = "kmodconfig",
1035 description = "Embedding kmod repository configuration",
1036 command=Interpolate("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(prop:kernelversion)s#' " +
1037 "%(prop:stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
1038 haltOnFailure = True
1041 #factory.addStep(IfBuiltinShellCommand(
1042 factory.addStep(ShellCommand(
1044 description = "Building and installing images",
1045 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
1047 haltOnFailure = True
1050 factory.addStep(ShellCommand(
1052 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
1053 command = "make -j1 buildinfo V=s || true",
1055 haltOnFailure = True
1058 factory.addStep(ShellCommand(
1059 name = "json_overview_image_info",
1060 description = "Generate profiles.json in target folder",
1061 command = "make -j1 json_overview_image_info V=s || true",
1063 haltOnFailure = True
1066 factory.addStep(ShellCommand(
1068 description = "Calculating checksums",
1069 command=["make", "-j1", "checksum", "V=s"],
1071 haltOnFailure = True
1074 if enable_kmod_archive:
1075 factory.addStep(ShellCommand(
1077 description = "Creating kmod directory",
1078 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
1079 haltOnFailure = True
1082 factory.addStep(ShellCommand(
1083 name = "kmodprepare",
1084 description = "Preparing kmod archive",
1085 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
1086 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
1087 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1088 haltOnFailure = True
1091 factory.addStep(ShellCommand(
1093 description = "Indexing kmod archive",
1094 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
1095 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1097 haltOnFailure = True
1101 if ini.has_option("gpg", "key") or usign_key is not None:
1102 factory.addStep(MasterShellCommand(
1103 name = "signprepare",
1104 description = "Preparing temporary signing directory",
1105 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1106 haltOnFailure = True
1109 factory.addStep(ShellCommand(
1111 description = "Packing files to sign",
1112 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1113 haltOnFailure = True
1116 factory.addStep(FileUpload(
1117 workersrc = "sign.tar.gz",
1118 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1119 haltOnFailure = True
1122 factory.addStep(MasterShellCommand(
1124 description = "Signing files",
1125 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
1126 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1127 haltOnFailure = True
1130 factory.addStep(FileDownload(
1131 name = "dlsigntargz",
1132 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1133 workerdest = "sign.tar.gz",
1134 haltOnFailure = True
1137 factory.addStep(ShellCommand(
1138 name = "signunpack",
1139 description = "Unpacking signed files",
1140 command = ["tar", "-xzf", "sign.tar.gz"],
1141 haltOnFailure = True
1145 factory.addStep(ShellCommand(
1146 name = "dirprepare",
1147 description = "Preparing upload directory structure",
1148 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1149 haltOnFailure = True
1152 factory.addStep(ShellCommand(
1153 name = "linkprepare",
1154 description = "Preparing repository symlink",
1155 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=GetBaseVersion()), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1156 doStepIf = IsNoMasterBuild,
1157 haltOnFailure = True
1160 if enable_kmod_archive:
1161 factory.addStep(ShellCommand(
1162 name = "kmoddirprepare",
1163 description = "Preparing kmod archive upload directory",
1164 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1165 haltOnFailure = True
1168 factory.addStep(ShellCommand(
1170 description = "Uploading directory structure",
1171 command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
1172 env={'RSYNC_PASSWORD': rsync_bin_key},
1173 haltOnFailure = True,
1177 # download remote sha256sums to 'target-sha256sums'
1178 factory.addStep(ShellCommand(
1179 name = "target-sha256sums",
1180 description = "Fetching remote sha256sums for target",
1181 command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1182 env={'RSYNC_PASSWORD': rsync_bin_key},
1184 haltOnFailure = False,
1185 flunkOnFailure = False,
1186 warnOnFailure = False,
1189 # build list of files to upload
1190 factory.addStep(FileDownload(
1191 name = "dlsha2rsyncpl",
1192 mastersrc = scripts_dir + '/sha2rsync.pl',
1193 workerdest = "../sha2rsync.pl",
1197 factory.addStep(ShellCommand(
1199 description = "Building list of files to upload",
1200 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1201 haltOnFailure = True,
1204 factory.addStep(FileDownload(
1205 name = "dlrsync.sh",
1206 mastersrc = scripts_dir + '/rsync.sh',
1207 workerdest = "../rsync.sh",
1211 # upload new files and update existing ones
1212 factory.addStep(ShellCommand(
1213 name = "targetupload",
1214 description = "Uploading target files",
1215 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1216 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1217 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1218 env={'RSYNC_PASSWORD': rsync_bin_key},
1219 haltOnFailure = True,
1223 # delete files which don't exist locally
1224 factory.addStep(ShellCommand(
1225 name = "targetprune",
1226 description = "Pruning target files",
1227 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1228 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1229 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1230 env={'RSYNC_PASSWORD': rsync_bin_key},
1231 haltOnFailure = True,
1235 if enable_kmod_archive:
1236 factory.addStep(ShellCommand(
1237 name = "kmodupload",
1238 description = "Uploading kmod archive",
1239 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1240 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1241 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1242 env={'RSYNC_PASSWORD': rsync_bin_key},
1243 haltOnFailure = True,
1247 if rsync_src_url is not None:
1248 factory.addStep(ShellCommand(
1249 name = "sourcelist",
1250 description = "Finding source archives to upload",
1251 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -newer .config -printf '%f\\n' > sourcelist",
1252 haltOnFailure = True
1255 factory.addStep(ShellCommand(
1256 name = "sourceupload",
1257 description = "Uploading source archives",
1258 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
1259 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
1260 env={'RSYNC_PASSWORD': rsync_src_key},
1261 haltOnFailure = True,
1266 factory.addStep(ShellCommand(
1267 name = "packageupload",
1268 description = "Uploading package files",
1269 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a"] + rsync_bin_defopts + ["bin/packages/", "%s/packages/" %(rsync_bin_url)],
1270 env={'RSYNC_PASSWORD': rsync_bin_key},
1271 haltOnFailure = False,
1277 factory.addStep(ShellCommand(
1279 description = "Uploading logs",
1280 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az"] + rsync_bin_defopts + ["logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
1281 env={'RSYNC_PASSWORD': rsync_bin_key},
1282 haltOnFailure = False,
1287 factory.addStep(ShellCommand(
1289 description = "Reporting disk usage",
1290 command=["df", "-h", "."],
1291 env={'LC_ALL': 'C'},
1292 haltOnFailure = False,
1296 factory.addStep(ShellCommand(
1297 name = "ccachestat",
1298 description = "Reporting ccache stats",
1299 command=["ccache", "-s"],
1300 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1301 want_stderr = False,
1302 haltOnFailure = False,
1303 flunkOnFailure = False,
1304 warnOnFailure = False,
1308 c['builders'].append(BuilderConfig(name=target, workernames=slaveNames, factory=factory, nextBuild=GetNextBuild))
1310 c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1311 force_factory.addStep(steps.Trigger(
1312 name = "trigger_%s" % target,
1313 description = "Triggering %s build" % target,
1314 schedulerNames = [ "trigger_%s" % target ],
1315 set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
1316 doStepIf = IsTargetSelected(target)
1320 ####### STATUS TARGETS
1322 # 'status' is a list of Status Targets. The results of each build will be
1323 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1324 # including web pages, email senders, and IRC bots.
1326 if ini.has_option("phase1", "status_bind"):
1328 'port': ini.get("phase1", "status_bind"),
1330 'waterfall_view': True,
1331 'console_view': True,
1336 if ini.has_option("phase1", "status_user") and ini.has_option("phase1", "status_password"):
1337 c['www']['auth'] = util.UserPasswordAuth([
1338 (ini.get("phase1", "status_user"), ini.get("phase1", "status_password"))
1340 c['www']['authz'] = util.Authz(
1341 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1342 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[ini.get("phase1", "status_user")]) ]
1346 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
1347 irc_host = ini.get("irc", "host")
1349 irc_chan = ini.get("irc", "channel")
1350 irc_nick = ini.get("irc", "nickname")
1353 if ini.has_option("irc", "port"):
1354 irc_port = ini.getint("irc", "port")
1356 if ini.has_option("irc", "password"):
1357 irc_pass = ini.get("irc", "password")
1359 irc = reporters.IRC(irc_host, irc_nick,
1361 password = irc_pass,
1362 channels = [ irc_chan ],
1363 notify_events = [ 'exception', 'problem', 'recovery' ]
1366 c['services'].append(irc)
1371 # This specifies what database buildbot uses to store its state. You can leave
1372 # this at its default for all but the largest installations.
1373 'db_url' : "sqlite:///state.sqlite",
1376 c['buildbotNetUsageData'] = None