2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes.gitpoller import GitPoller
19 from buildbot.config import BuilderConfig
20 from buildbot.plugins import reporters
21 from buildbot.plugins import schedulers
22 from buildbot.plugins import steps
23 from buildbot.plugins import util
24 from buildbot.process import properties
25 from buildbot.process import results
26 from buildbot.process.factory import BuildFactory
27 from buildbot.process.properties import Interpolate
28 from buildbot.process.properties import Property
29 from buildbot.schedulers.basic import AnyBranchScheduler
30 from buildbot.schedulers.forcesched import BaseParameter
31 from buildbot.schedulers.forcesched import ForceScheduler
32 from buildbot.schedulers.forcesched import ValidationError
33 from buildbot.steps.master import MasterShellCommand
34 from buildbot.steps.shell import SetPropertyFromCommand
35 from buildbot.steps.shell import ShellCommand
36 from buildbot.steps.source.git import Git
37 from buildbot.steps.transfer import FileDownload
38 from buildbot.steps.transfer import FileUpload
39 from buildbot.steps.transfer import StringDownload
40 from buildbot.worker import Worker
41 from buildbot.worker.local import LocalWorker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini['repo'].get("url")
65 rsync_defopts = ["-v", "-4", "--timeout=120"]
67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
68 # rsync_bin_defopts += ["--contimeout=20"]
72 def ini_parse_branch(section):
74 name = section.get("name")
77 raise ValueError("missing 'name' in " + repr(section))
79 raise ValueError("duplicate branch name in " + repr(section))
82 b["bin_url"] = section.get("binary_url")
83 b["bin_key"] = section.get("binary_password")
85 b["src_url"] = section.get("source_url")
86 b["src_key"] = section.get("source_password")
88 b["gpg_key"] = section.get("gpg_key")
90 b["usign_key"] = section.get("usign_key")
91 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
92 b["usign_comment"] = section.get("usign_comment", usign_comment)
94 b["config_seed"] = section.get("config_seed")
96 b["kmod_archive"] = section.getboolean("kmod_archive", False)
99 log.msg("Configured branch: {}".format(name))
101 # PB port can be either a numeric port or a connection string
102 pb_port = inip1.get("port") or 9989
104 # This is the dictionary that the buildmaster pays attention to. We also use
105 # a shorter alias to save typing.
106 c = BuildmasterConfig = {}
108 ####### PROJECT IDENTITY
110 # the 'title' string will appear at the top of this buildbot
111 # installation's html.WebStatus home page (linked to the
112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
114 c['title'] = ini['general'].get("title")
115 c['titleURL'] = ini['general'].get("title_url")
117 # the 'buildbotURL' string should point to the location where the buildbot's
118 # internal web server (usually the html.WebStatus page) is visible. This
119 # typically uses the port number set in the Waterfall 'status' entry, but
120 # with an externally-visible host name which the buildbot cannot figure out
123 c['buildbotURL'] = inip1.get("buildbot_url")
127 # The 'workers' list defines the set of recognized buildworkers. Each element is
128 # a Worker object, specifying a unique worker name and password. The same
129 # worker name and password must be configured on the worker.
134 for section in ini.sections():
135 if section.startswith("branch "):
136 ini_parse_branch(ini[section])
138 if section.startswith("worker "):
139 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
140 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
141 sl_props = { 'dl_lock':None, 'ul_lock':None }
142 name = ini.get(section, "name")
143 password = ini.get(section, "password")
144 if ini.has_option(section, "dl_lock"):
145 lockname = ini.get(section, "dl_lock")
146 sl_props['dl_lock'] = lockname
147 if lockname not in NetLocks:
148 NetLocks[lockname] = locks.MasterLock(lockname)
149 if ini.has_option(section, "ul_lock"):
150 lockname = ini.get(section, "ul_lock")
151 sl_props['ul_lock'] = lockname
152 if lockname not in NetLocks:
153 NetLocks[lockname] = locks.MasterLock(lockname)
154 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
156 c['protocols'] = {'pb': {'port': pb_port}}
159 c['collapseRequests'] = True
161 # Reduce amount of backlog data
162 c['configurators'] = [util.JanitorConfigurator(
163 logHorizon=timedelta(days=3),
167 @defer.inlineCallbacks
168 def getNewestCompleteTime(bldr):
169 """Returns the complete_at of the latest completed and not SKIPPED
170 build request for this builder, or None if there are no such build
171 requests. We need to filter out SKIPPED requests because we're
172 using collapseRequests=True which is unfortunately marking all
173 previous requests as complete when new buildset is created.
175 @returns: datetime instance or None, via Deferred
178 bldrid = yield bldr.getBuilderId()
179 completed = yield bldr.master.data.get(
180 ('builders', bldrid, 'buildrequests'),
182 resultspec.Filter('complete', 'eq', [True]),
183 resultspec.Filter('results', 'ne', [results.SKIPPED]),
185 order=['-complete_at'], limit=1)
189 complete_at = completed[0]['complete_at']
191 last_build = yield bldr.master.data.get(
194 resultspec.Filter('builderid', 'eq', [bldrid]),
196 order=['-started_at'], limit=1)
198 if last_build and last_build[0]:
199 last_complete_at = last_build[0]['complete_at']
200 if last_complete_at and (last_complete_at > complete_at):
201 return last_complete_at
205 @defer.inlineCallbacks
206 def prioritizeBuilders(master, builders):
207 """Returns sorted list of builders by their last timestamp of completed and
210 @returns: list of sorted builders
213 def is_building(bldr):
214 return bool(bldr.building) or bool(bldr.old_building)
217 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
218 d.addCallback(lambda complete_at: (complete_at, bldr))
222 (complete_at, bldr) = item
224 if bldr.name == "00_force_build":
226 complete_at = date.replace(tzinfo=tzutc())
227 return (complete_at, bldr.name)
231 complete_at = date.replace(tzinfo=tzutc())
233 if is_building(bldr):
235 complete_at = date.replace(tzinfo=tzutc())
237 return (complete_at, bldr.name)
239 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
240 results.sort(key=bldr_sort)
243 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
245 return [r[1] for r in results]
247 c['prioritizeBuilders'] = prioritizeBuilders
249 ####### CHANGESOURCES
251 branchNames = [branches[b]["name"] for b in branches]
256 def populateTargets():
257 log.msg("Populating targets, this will take time")
258 sourcegit = work_dir + '/source.git'
259 for branch in branchNames:
260 if os.path.isdir(sourcegit):
261 subprocess.call(["rm", "-rf", sourcegit])
263 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
265 os.makedirs(sourcegit + '/tmp', exist_ok=True)
266 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
267 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
270 line = findtargets.stdout.readline()
273 ta = line.decode().strip().split(' ')
276 subprocess.call(["rm", "-rf", sourcegit])
280 # the 'change_source' setting tells the buildmaster how it should find out
281 # about source code changes. Here we point to the buildbot clone of pyflakes.
283 c['change_source'] = []
284 c['change_source'].append(GitPoller(
286 workdir=work_dir+'/work.git', branches=branchNames,
287 pollAtLaunch=True, pollinterval=300))
291 # Configure the Schedulers, which decide how to react to incoming changes. In this
292 # case, just kick off a 'basebuild' build
294 class TagChoiceParameter(BaseParameter):
295 spec_attributes = ["strict", "choices"]
299 def __init__(self, name, label=None, **kw):
300 super().__init__(name, label, **kw)
301 self._choice_list = []
308 for b in branchNames:
309 basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
311 branchvers.append(basever[1])
313 alltags = subprocess.Popen(
314 ['git', 'ls-remote', '--tags', repo_url],
315 stdout = subprocess.PIPE)
318 line = alltags.stdout.readline()
323 (ref, tag) = line.split()
325 tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
327 # only list tags matching configured branches
328 if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
329 taglist.append(tagver[1])
331 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
332 taglist.insert(0, '')
334 self._choice_list = taglist
336 return self._choice_list
338 def updateFromKwargs(self, properties, kwargs, **unused):
339 tag = self.getFromKwargs(kwargs)
340 properties[self.name] = tag
342 # find the commit matching the tag
343 findrev = subprocess.Popen(['git', 'rev-parse', 'tags/'+tag], stdout=subprocess.PIPE, cwd=work_dir+'/work.git')
344 findrev.wait(timeout=10)
345 line = findrev.stdout.readline()
347 if findrev.returncode!=0 or not line:
348 raise ValidationError("Couldn't find tag")
350 properties['force_revision'] = line.decode().strip()
352 # find the branch matching the tag
354 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
355 for b in branchNames:
356 if b.endswith(branchver[1]):
360 raise ValidationError("Couldn't find branch")
362 properties['force_branch'] = branch
364 def parse_from_arg(self, s):
365 if self.strict and s not in self._choice_list:
366 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
370 c['schedulers'].append(AnyBranchScheduler(
372 change_filter = util.ChangeFilter(branch=branchNames),
373 treeStableTimer = 15*60,
374 builderNames = list(targets)))
376 c['schedulers'].append(ForceScheduler(
378 buttonName = "Force builds",
379 label = "Force build details",
380 builderNames = [ "00_force_build" ],
383 util.CodebaseParameter(
385 label = "Repository",
386 branch = util.FixedParameter(name = "branch", default = ""),
387 revision = util.FixedParameter(name = "revision", default = ""),
388 repository = util.FixedParameter(name = "repository", default = ""),
389 project = util.FixedParameter(name = "project", default = "")
393 reason = util.StringParameter(
396 default = "Trigger build",
402 util.ChoiceStringParameter(
404 label = "Build target",
406 choices = [ "all" ] + list(targets)
418 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
419 # what steps, and which workers can execute them. Note that any particular build will
420 # only take place on one worker.
422 def IsNoMasterBuild(step):
423 return step.getProperty("branch") != "master"
425 def IsUsignEnabled(step):
426 branch = step.getProperty("branch")
427 return branch and branches[branch].get("usign_key")
429 def IsSignEnabled(step):
430 branch = step.getProperty("branch")
431 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
433 def IsKmodArchiveEnabled(step):
434 branch = step.getProperty("branch")
435 return branch and branches[branch].get("kmod_archive")
437 def GetBaseVersion(branch):
438 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
439 return branch.split('-')[1]
444 def GetVersionPrefix(props):
445 branch = props.getProperty("branch")
446 basever = GetBaseVersion(branch)
447 if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
448 return "%s/" % props["tag"][1:]
449 elif basever != "master":
450 return "%s-SNAPSHOT/" % basever
455 def GetConfigSeed(props):
456 branch = props.getProperty("branch")
457 return branch and branches[branch].get("config_seed") or ""
460 def GetRsyncParams(props, srcorbin, urlorkey):
461 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
462 branch = props.getProperty("branch")
463 opt = srcorbin + "_" + urlorkey
464 return branch and branches[branch].get(opt)
467 def GetUsignKey(props):
468 branch = props.getProperty("branch")
469 return branch and branches[branch].get("usign_key")
471 def GetNextBuild(builder, requests):
474 # order tagged build first
475 if r.properties.hasProperty("tag"):
477 # then order by branch order
478 pbranch = r.properties.getProperty("branch")
479 for name in branchNames:
484 log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
487 def MakeEnv(overrides=None, tryccache=False):
489 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
490 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
493 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
494 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
495 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
497 env['CC'] = env['CCC']
498 env['CXX'] = env['CCXX']
500 if overrides is not None:
501 env.update(overrides)
505 def NetLockDl(props, extralock=None):
507 if props.hasProperty("dl_lock"):
508 lock = NetLocks[props["dl_lock"]]
510 return [lock.access('exclusive')]
515 def NetLockUl(props):
517 if props.hasProperty("ul_lock"):
518 lock = NetLocks[props["ul_lock"]]
520 return [lock.access('exclusive')]
524 def IsTargetSelected(target):
525 def CheckTargetProperty(step):
526 selected_target = step.getProperty("target", "all")
527 if selected_target != "all" and selected_target != target:
531 return CheckTargetProperty
534 def UsignSec2Pub(props):
535 branch = props.getProperty("branch")
537 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
538 seckey = branches[branch].get("usign_key")
539 seckey = base64.b64decode(seckey)
543 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
544 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
551 for worker in c['workers']:
552 workerNames.append(worker.workername)
554 # add a single LocalWorker to handle the forcebuild builder
555 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
557 force_factory = BuildFactory()
559 c['builders'].append(BuilderConfig(
560 name = "00_force_build",
561 workername = "__local_force_build",
562 factory = force_factory))
564 for target in targets:
565 ts = target.split('/')
567 factory = BuildFactory()
569 # setup shared work directory if required
570 factory.addStep(ShellCommand(
572 descriptionDone = "Shared work directory set up",
573 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
575 haltOnFailure = True,
578 # find number of cores
579 factory.addStep(SetPropertyFromCommand(
582 description = "Finding number of CPUs",
586 # find gcc and g++ compilers
587 factory.addStep(FileDownload(
588 name = "dlfindbinpl",
589 mastersrc = scripts_dir + '/findbin.pl',
590 workerdest = "../findbin.pl",
594 factory.addStep(SetPropertyFromCommand(
596 property = "cc_command",
597 description = "Finding gcc command",
598 command = ["../findbin.pl", "gcc", "", ""],
599 haltOnFailure = True,
602 factory.addStep(SetPropertyFromCommand(
604 property = "cxx_command",
605 description = "Finding g++ command",
606 command = ["../findbin.pl", "g++", "", ""],
607 haltOnFailure = True,
610 # see if ccache is available
611 factory.addStep(SetPropertyFromCommand(
613 property = "ccache_command",
614 description = "Testing for ccache command",
615 command = ["which", "ccache"],
616 haltOnFailure = False,
617 flunkOnFailure = False,
618 warnOnFailure = False,
619 hideStepIf = lambda r, s: r==results.FAILURE,
622 # check out the source
624 # if repo doesn't exist: 'git clone repourl'
625 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
626 # git cat-file -e <commit>
627 # git checkout -f <commit>
628 # git checkout -B <branch>
636 haltOnFailure = True,
640 factory.addStep(ShellCommand(
642 description = "Fetching Git remote refs",
643 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
644 haltOnFailure = True,
647 # Verify that Git HEAD points to a tag or branch
648 # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
649 factory.addStep(ShellCommand(
651 description = "Ensure that Git HEAD is pointing to a branch or tag",
652 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
653 haltOnFailure = True,
656 factory.addStep(ShellCommand(
658 description = "Remove tmp folder",
659 command=["rm", "-rf", "tmp/"],
663 factory.addStep(ShellCommand(
664 name = "rmfeedlinks",
665 description = "Remove feed symlinks",
666 command=["rm", "-rf", "package/feeds/"],
669 factory.addStep(StringDownload(
671 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
672 workerdest = "../ccache_cc.sh",
676 factory.addStep(StringDownload(
678 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
679 workerdest = "../ccache_cxx.sh",
684 factory.addStep(ShellCommand(
685 name = "updatefeeds",
686 description = "Updating feeds",
687 command=["./scripts/feeds", "update"],
688 env = MakeEnv(tryccache=True),
689 haltOnFailure = True,
694 factory.addStep(ShellCommand(
695 name = "installfeeds",
696 description = "Installing feeds",
697 command=["./scripts/feeds", "install", "-a"],
698 env = MakeEnv(tryccache=True),
699 haltOnFailure = True,
703 factory.addStep(StringDownload(
704 name = "dlconfigseed",
705 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
706 workerdest = ".config",
711 factory.addStep(ShellCommand(
713 descriptionDone = ".config seeded",
714 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
717 factory.addStep(ShellCommand(
719 description = "Removing output directory",
720 command = ["rm", "-rf", "bin/"],
723 factory.addStep(ShellCommand(
725 description = "Populating .config",
726 command = ["make", "defconfig"],
730 # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
731 factory.addStep(ShellCommand(
733 description = "Checking architecture",
734 descriptionDone = "Architecture validated",
735 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
739 haltOnFailure = True,
740 flunkOnFailure = False, # this is not a build FAILURE
744 factory.addStep(SetPropertyFromCommand(
747 description = "Finding libc suffix",
748 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
752 factory.addStep(StringDownload(
753 name = "dlkeybuildpub",
754 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
755 workerdest = "key-build.pub",
757 doStepIf = IsUsignEnabled,
760 factory.addStep(StringDownload(
762 s = "# fake private key",
763 workerdest = "key-build",
765 doStepIf = IsUsignEnabled,
768 factory.addStep(StringDownload(
769 name = "dlkeybuilducert",
770 s = "# fake certificate",
771 workerdest = "key-build.ucert",
773 doStepIf = IsUsignEnabled,
777 factory.addStep(ShellCommand(
779 description = "Preparing dl/",
780 descriptionDone = "dl/ prepared",
781 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
782 workdir = Property("builddir"),
788 factory.addStep(ShellCommand(
790 description = "Pruning dl/",
791 descriptionDone = "dl/ pruned",
792 command = 'find dl/ -atime +15 -delete -print',
797 factory.addStep(ShellCommand(
799 description = "Building and installing GNU tar",
800 descriptionDone = "GNU tar built and installed",
801 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
802 env = MakeEnv(tryccache=True),
803 haltOnFailure = True,
807 factory.addStep(ShellCommand(
809 description = "Populating dl/",
810 descriptionDone = "dl/ populated",
811 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
817 factory.addStep(ShellCommand(
819 description = "Cleaning base-files",
820 command=["make", "package/base-files/clean", "V=s"],
824 factory.addStep(ShellCommand(
826 description = "Building and installing tools",
827 descriptionDone = "Tools built and installed",
828 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
829 env = MakeEnv(tryccache=True),
830 haltOnFailure = True,
833 factory.addStep(ShellCommand(
835 description = "Building and installing toolchain",
836 descriptionDone = "Toolchain built and installed",
837 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
839 haltOnFailure = True,
842 factory.addStep(ShellCommand(
844 description = "Building kmods",
845 descriptionDone = "Kmods built",
846 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
848 haltOnFailure = True,
851 # find kernel version
852 factory.addStep(SetPropertyFromCommand(
853 name = "kernelversion",
854 property = "kernelversion",
855 description = "Finding the effective Kernel version",
856 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
857 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
860 factory.addStep(ShellCommand(
862 description = "Cleaning up package build",
863 descriptionDone = "Package build cleaned up",
864 command=["make", "package/cleanup", "V=s"],
867 factory.addStep(ShellCommand(
869 description = "Building packages",
870 descriptionDone = "Packages built",
871 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
873 haltOnFailure = True,
876 factory.addStep(ShellCommand(
878 description = "Installing packages",
879 descriptionDone = "Packages installed",
880 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
882 haltOnFailure = True,
885 factory.addStep(ShellCommand(
887 description = "Indexing packages",
888 descriptionDone = "Packages indexed",
889 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
891 haltOnFailure = True,
894 factory.addStep(ShellCommand(
896 description = "Building and installing images",
897 descriptionDone = "Images built and installed",
898 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
900 haltOnFailure = True,
903 factory.addStep(ShellCommand(
905 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
906 command = "make -j1 buildinfo V=s || true",
908 haltOnFailure = True,
911 factory.addStep(ShellCommand(
912 name = "json_overview_image_info",
913 description = "Generating profiles.json in target folder",
914 command = "make -j1 json_overview_image_info V=s || true",
916 haltOnFailure = True,
919 factory.addStep(ShellCommand(
921 description = "Calculating checksums",
922 descriptionDone = "Checksums calculated",
923 command=["make", "-j1", "checksum", "V=s"],
925 haltOnFailure = True,
928 factory.addStep(ShellCommand(
930 descriptionDone = "Kmod directory created",
931 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
932 haltOnFailure = True,
933 doStepIf = IsKmodArchiveEnabled,
936 factory.addStep(ShellCommand(
937 name = "kmodprepare",
938 description = "Preparing kmod archive",
939 descriptionDone = "Kmod archive prepared",
940 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
941 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
942 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
943 haltOnFailure = True,
944 doStepIf = IsKmodArchiveEnabled,
947 factory.addStep(ShellCommand(
949 description = "Indexing kmod archive",
950 descriptionDone = "Kmod archive indexed",
951 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
952 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
954 haltOnFailure = True,
955 doStepIf = IsKmodArchiveEnabled,
959 factory.addStep(MasterShellCommand(
960 name = "signprepare",
961 descriptionDone = "Temporary signing directory prepared",
962 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
963 haltOnFailure = True,
964 doStepIf = IsSignEnabled,
968 factory.addStep(ShellCommand(
970 description = "Packing files to sign",
971 descriptionDone = "Files to sign packed",
972 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
973 haltOnFailure = True,
974 doStepIf = IsSignEnabled,
977 factory.addStep(FileUpload(
978 workersrc = "sign.tar.gz",
979 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
980 haltOnFailure = True,
981 doStepIf = IsSignEnabled,
984 factory.addStep(MasterShellCommand(
986 description = "Signing files",
987 descriptionDone = "Files signed",
988 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
989 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
990 haltOnFailure = True,
991 doStepIf = IsSignEnabled,
994 factory.addStep(FileDownload(
995 name = "dlsigntargz",
996 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
997 workerdest = "sign.tar.gz",
998 haltOnFailure = True,
999 doStepIf = IsSignEnabled,
1002 factory.addStep(ShellCommand(
1003 name = "signunpack",
1004 description = "Unpacking signed files",
1005 descriptionDone = "Signed files unpacked",
1006 command = ["tar", "-xzf", "sign.tar.gz"],
1007 haltOnFailure = True,
1008 doStepIf = IsSignEnabled,
1012 factory.addStep(ShellCommand(
1013 name = "dirprepare",
1014 descriptionDone = "Upload directory structure prepared",
1015 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1016 haltOnFailure = True,
1019 factory.addStep(ShellCommand(
1020 name = "linkprepare",
1021 descriptionDone = "Repository symlink prepared",
1022 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1023 doStepIf = IsNoMasterBuild,
1024 haltOnFailure = True,
1027 factory.addStep(ShellCommand(
1028 name = "kmoddirprepare",
1029 descriptionDone = "Kmod archive upload directory prepared",
1030 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1031 haltOnFailure = True,
1032 doStepIf = IsKmodArchiveEnabled,
1035 factory.addStep(ShellCommand(
1037 description = "Uploading directory structure",
1038 descriptionDone = "Directory structure uploaded",
1039 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1040 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1041 haltOnFailure = True,
1044 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1047 # download remote sha256sums to 'target-sha256sums'
1048 factory.addStep(ShellCommand(
1049 name = "target-sha256sums",
1050 description = "Fetching remote sha256sums for target",
1051 descriptionDone = "Remote sha256sums for target fetched",
1052 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1053 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1055 haltOnFailure = False,
1056 flunkOnFailure = False,
1057 warnOnFailure = False,
1058 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1061 # build list of files to upload
1062 factory.addStep(FileDownload(
1063 name = "dlsha2rsyncpl",
1064 mastersrc = scripts_dir + '/sha2rsync.pl',
1065 workerdest = "../sha2rsync.pl",
1069 factory.addStep(ShellCommand(
1071 description = "Building list of files to upload",
1072 descriptionDone = "List of files to upload built",
1073 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1074 haltOnFailure = True,
1077 factory.addStep(FileDownload(
1078 name = "dlrsync.sh",
1079 mastersrc = scripts_dir + '/rsync.sh',
1080 workerdest = "../rsync.sh",
1084 # upload new files and update existing ones
1085 factory.addStep(ShellCommand(
1086 name = "targetupload",
1087 description = "Uploading target files",
1088 descriptionDone = "Target files uploaded",
1089 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1090 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1091 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1092 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1093 haltOnFailure = True,
1095 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1098 # delete files which don't exist locally
1099 factory.addStep(ShellCommand(
1100 name = "targetprune",
1101 description = "Pruning target files",
1102 descriptionDone = "Target files pruned",
1103 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1104 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1105 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1106 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1107 haltOnFailure = True,
1110 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1113 factory.addStep(ShellCommand(
1114 name = "kmodupload",
1115 description = "Uploading kmod archive",
1116 descriptionDone = "Kmod archive uploaded",
1117 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1118 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1119 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1120 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1121 haltOnFailure = True,
1124 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1127 factory.addStep(ShellCommand(
1128 name = "sourcelist",
1129 description = "Finding source archives to upload",
1130 descriptionDone = "Source archives to upload found",
1131 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1132 haltOnFailure = True,
1135 factory.addStep(ShellCommand(
1136 name = "sourceupload",
1137 description = "Uploading source archives",
1138 descriptionDone = "Source archives uploaded",
1139 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1140 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1141 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1142 haltOnFailure = True,
1145 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1148 factory.addStep(ShellCommand(
1150 description = "Reporting disk usage",
1151 command=["df", "-h", "."],
1152 env={'LC_ALL': 'C'},
1154 haltOnFailure = False,
1155 flunkOnFailure = False,
1156 warnOnFailure = False,
1160 factory.addStep(ShellCommand(
1162 description = "Reporting estimated file space usage",
1163 command=["du", "-sh", "."],
1164 env={'LC_ALL': 'C'},
1166 haltOnFailure = False,
1167 flunkOnFailure = False,
1168 warnOnFailure = False,
1172 factory.addStep(ShellCommand(
1173 name = "ccachestat",
1174 description = "Reporting ccache stats",
1175 command=["ccache", "-s"],
1176 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1178 want_stderr = False,
1179 haltOnFailure = False,
1180 flunkOnFailure = False,
1181 warnOnFailure = False,
1182 hideStepIf = lambda r, s: r==results.FAILURE,
1185 c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1187 c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1188 force_factory.addStep(steps.Trigger(
1189 name = "trigger_%s" % target,
1190 description = "Triggering %s build" % target,
1191 schedulerNames = [ "trigger_%s" % target ],
1192 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
1193 set_properties = { "reason": Property("reason"), "tag": Property("tag"), },
1194 doStepIf = IsTargetSelected(target),
1198 ####### STATUS TARGETS
1200 # 'status' is a list of Status Targets. The results of each build will be
1201 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1202 # including web pages, email senders, and IRC bots.
1204 if "status_bind" in inip1:
1206 'port': inip1.get("status_bind"),
1208 'waterfall_view': True,
1209 'console_view': True,
1214 if "status_user" in inip1 and "status_password" in inip1:
1215 c['www']['auth'] = util.UserPasswordAuth([
1216 (inip1.get("status_user"), inip1.get("status_password"))
1218 c['www']['authz'] = util.Authz(
1219 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1220 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1224 if ini.has_section("irc"):
1226 irc_host = iniirc.get("host", None)
1227 irc_port = iniirc.getint("port", 6667)
1228 irc_chan = iniirc.get("channel", None)
1229 irc_nick = iniirc.get("nickname", None)
1230 irc_pass = iniirc.get("password", None)
1232 if irc_host and irc_nick and irc_chan:
1233 irc = reporters.IRC(irc_host, irc_nick,
1235 password = irc_pass,
1236 channels = [ irc_chan ],
1237 notify_events = [ 'exception', 'problem', 'recovery' ]
1240 c['services'].append(irc)
1242 c['revlink'] = util.RevlinkMatch([
1243 r'https://git.openwrt.org/openwrt/(.*).git'
1245 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1250 # This specifies what database buildbot uses to store its state. You can leave
1251 # this at its default for all but the largest installations.
1252 'db_url' : "sqlite:///state.sqlite",
1255 c['buildbotNetUsageData'] = None