2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes.gitpoller import GitPoller
19 from buildbot.config import BuilderConfig
20 from buildbot.plugins import reporters
21 from buildbot.plugins import schedulers
22 from buildbot.plugins import steps
23 from buildbot.plugins import util
24 from buildbot.process import properties
25 from buildbot.process import results
26 from buildbot.process.factory import BuildFactory
27 from buildbot.process.properties import Interpolate
28 from buildbot.process.properties import Property
29 from buildbot.schedulers.basic import AnyBranchScheduler
30 from buildbot.schedulers.forcesched import BaseParameter
31 from buildbot.schedulers.forcesched import ForceScheduler
32 from buildbot.schedulers.forcesched import ValidationError
33 from buildbot.steps.master import MasterShellCommand
34 from buildbot.steps.shell import SetPropertyFromCommand
35 from buildbot.steps.shell import ShellCommand
36 from buildbot.steps.source.git import Git
37 from buildbot.steps.transfer import FileDownload
38 from buildbot.steps.transfer import FileUpload
39 from buildbot.steps.transfer import StringDownload
40 from buildbot.worker import Worker
41 from buildbot.worker.local import LocalWorker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini['repo'].get("url")
65 rsync_defopts = ["-v", "-4", "--timeout=120"]
67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
68 # rsync_bin_defopts += ["--contimeout=20"]
72 def ini_parse_branch(section):
74 name = section.get("name")
77 raise ValueError("missing 'name' in " + repr(section))
79 raise ValueError("duplicate branch name in " + repr(section))
82 b["bin_url"] = section.get("binary_url")
83 b["bin_key"] = section.get("binary_password")
85 b["src_url"] = section.get("source_url")
86 b["src_key"] = section.get("source_password")
88 b["gpg_key"] = section.get("gpg_key")
90 b["usign_key"] = section.get("usign_key")
91 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
92 b["usign_comment"] = section.get("usign_comment", usign_comment)
94 b["config_seed"] = section.get("config_seed")
96 b["kmod_archive"] = section.getboolean("kmod_archive", False)
99 log.msg("Configured branch: {}".format(name))
101 # PB port can be either a numeric port or a connection string
102 pb_port = inip1.get("port") or 9989
104 # This is the dictionary that the buildmaster pays attention to. We also use
105 # a shorter alias to save typing.
106 c = BuildmasterConfig = {}
108 ####### PROJECT IDENTITY
110 # the 'title' string will appear at the top of this buildbot
111 # installation's html.WebStatus home page (linked to the
112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
114 c['title'] = ini['general'].get("title")
115 c['titleURL'] = ini['general'].get("title_url")
117 # the 'buildbotURL' string should point to the location where the buildbot's
118 # internal web server (usually the html.WebStatus page) is visible. This
119 # typically uses the port number set in the Waterfall 'status' entry, but
120 # with an externally-visible host name which the buildbot cannot figure out
123 c['buildbotURL'] = inip1.get("buildbot_url")
127 # The 'workers' list defines the set of recognized buildworkers. Each element is
128 # a Worker object, specifying a unique worker name and password. The same
129 # worker name and password must be configured on the worker.
134 for section in ini.sections():
135 if section.startswith("branch "):
136 ini_parse_branch(ini[section])
138 if section.startswith("worker "):
139 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
140 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
141 sl_props = { 'dl_lock':None, 'ul_lock':None }
142 name = ini.get(section, "name")
143 password = ini.get(section, "password")
144 if ini.has_option(section, "dl_lock"):
145 lockname = ini.get(section, "dl_lock")
146 sl_props['dl_lock'] = lockname
147 if lockname not in NetLocks:
148 NetLocks[lockname] = locks.MasterLock(lockname)
149 if ini.has_option(section, "ul_lock"):
150 lockname = ini.get(section, "ul_lock")
151 sl_props['ul_lock'] = lockname
152 if lockname not in NetLocks:
153 NetLocks[lockname] = locks.MasterLock(lockname)
154 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
156 c['protocols'] = {'pb': {'port': pb_port}}
159 c['collapseRequests'] = True
161 # Reduce amount of backlog data
162 c['configurators'] = [util.JanitorConfigurator(
163 logHorizon=timedelta(days=3),
167 @defer.inlineCallbacks
168 def getNewestCompleteTime(bldr):
169 """Returns the complete_at of the latest completed and not SKIPPED
170 build request for this builder, or None if there are no such build
171 requests. We need to filter out SKIPPED requests because we're
172 using collapseRequests=True which is unfortunately marking all
173 previous requests as complete when new buildset is created.
175 @returns: datetime instance or None, via Deferred
178 bldrid = yield bldr.getBuilderId()
179 completed = yield bldr.master.data.get(
180 ('builders', bldrid, 'buildrequests'),
182 resultspec.Filter('complete', 'eq', [True]),
183 resultspec.Filter('results', 'ne', [results.SKIPPED]),
185 order=['-complete_at'], limit=1)
189 complete_at = completed[0]['complete_at']
191 last_build = yield bldr.master.data.get(
194 resultspec.Filter('builderid', 'eq', [bldrid]),
196 order=['-started_at'], limit=1)
198 if last_build and last_build[0]:
199 last_complete_at = last_build[0]['complete_at']
200 if last_complete_at and (last_complete_at > complete_at):
201 return last_complete_at
205 @defer.inlineCallbacks
206 def prioritizeBuilders(master, builders):
207 """Returns sorted list of builders by their last timestamp of completed and
208 not skipped build, ordered first by branch name.
210 @returns: list of sorted builders
213 bldrNamePrio = { "__Janitor": 0, "00_force_build": 0 }
215 for bname in branchNames:
216 bldrNamePrio[bname] = i
219 def is_building(bldr):
220 return bool(bldr.building) or bool(bldr.old_building)
223 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
224 d.addCallback(lambda complete_at: (complete_at, bldr))
228 (complete_at, bldr) = item
231 for (name, prio) in bldrNamePrio.items():
232 if bldr.name.startswith(name):
238 complete_at = date.replace(tzinfo=tzutc())
240 if is_building(bldr):
242 complete_at = date.replace(tzinfo=tzutc())
244 return (pos, complete_at, bldr.name)
246 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
247 results.sort(key=bldr_sort)
250 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
252 return [r[1] for r in results]
254 c['prioritizeBuilders'] = prioritizeBuilders
256 ####### CHANGESOURCES
258 branchNames = [branches[b]["name"] for b in branches]
263 def populateTargets():
264 log.msg("Populating targets, this will take time")
265 sourcegit = work_dir + '/source.git'
266 for branch in branchNames:
267 if os.path.isdir(sourcegit):
268 subprocess.call(["rm", "-rf", sourcegit])
270 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
272 os.makedirs(sourcegit + '/tmp', exist_ok=True)
273 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
274 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
277 line = findtargets.stdout.readline()
280 ta = line.decode().strip().split(' ')
283 subprocess.call(["rm", "-rf", sourcegit])
287 # the 'change_source' setting tells the buildmaster how it should find out
288 # about source code changes. Here we point to the buildbot clone of pyflakes.
290 c['change_source'] = []
291 c['change_source'].append(GitPoller(
293 workdir=work_dir+'/work.git', branches=branchNames,
294 pollAtLaunch=True, pollinterval=300))
298 # Configure the Schedulers, which decide how to react to incoming changes. In this
299 # case, just kick off a 'basebuild' build
301 class TagChoiceParameter(BaseParameter):
302 spec_attributes = ["strict", "choices"]
306 def __init__(self, name, label=None, **kw):
307 super().__init__(name, label, **kw)
308 self._choice_list = []
310 def getRevTags(self, findtag=None):
314 for b in branchNames:
315 basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
317 branchvers.append(basever[1])
319 alltags = subprocess.Popen(
320 ['git', 'ls-remote', '--tags', repo_url],
321 stdout = subprocess.PIPE)
324 line = alltags.stdout.readline()
329 (rev, tag) = line.split()
331 tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
333 # only list tags matching configured branches
334 if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
335 if findtag and findtag != tagver[1]:
337 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
343 taglist = [rt['tag'] for rt in self.getRevTags()]
344 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
345 taglist.insert(0, '')
347 self._choice_list = taglist
349 return self._choice_list
351 def updateFromKwargs(self, properties, kwargs, **unused):
352 tag = self.getFromKwargs(kwargs)
353 properties[self.name] = tag
355 # find the commit matching the tag
356 findtag = self.getRevTags(tag)
359 raise ValidationError("Couldn't find tag")
361 properties['force_revision'] = findtag[0]['rev']
363 # find the branch matching the tag
365 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
366 for b in branchNames:
367 if b.endswith(branchver[1]):
371 raise ValidationError("Couldn't find branch")
373 properties['force_branch'] = branch
375 def parse_from_arg(self, s):
376 if self.strict and s not in self._choice_list:
377 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
381 @defer.inlineCallbacks
382 def builderNames(props):
383 """ since we have per branch and per target builders,
384 address the relevant builder for each new buildrequest
385 based on the request's desired branch and target.
387 branch = props.getProperty("branch")
388 target = props.getProperty("target", "")
393 # if that didn't work, try sourcestamp to find a branch
395 # match builders with target branch
396 ss = props.sourcestamps[0]
398 branch = ss['branch']
400 log.msg("couldn't find builder")
401 return [] # nothing works
403 bname = branch + "_" + target
406 for b in (yield props.master.data.get(('builders',))):
407 if not b['name'].startswith(bname):
409 builders.append(b['name'])
414 c['schedulers'].append(AnyBranchScheduler(
416 change_filter = util.ChangeFilter(branch=branchNames),
417 treeStableTimer = 15*60,
418 builderNames = builderNames))
420 c['schedulers'].append(ForceScheduler(
422 buttonName = "Force builds",
423 label = "Force build details",
424 builderNames = [ "00_force_build" ],
427 util.CodebaseParameter(
429 label = "Repository",
430 branch = util.FixedParameter(name = "branch", default = ""),
431 revision = util.FixedParameter(name = "revision", default = ""),
432 repository = util.FixedParameter(name = "repository", default = ""),
433 project = util.FixedParameter(name = "project", default = "")
437 reason = util.StringParameter(
440 default = "Trigger build",
446 util.ChoiceStringParameter(
448 label = "Build target",
450 choices = [ "all" ] + list(targets)
460 c['schedulers'].append(schedulers.Triggerable(name="trigger", builderNames=builderNames))
464 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
465 # what steps, and which workers can execute them. Note that any particular build will
466 # only take place on one worker.
468 def IsNoMasterBuild(step):
469 return step.getProperty("branch") != "master"
471 def IsUsignEnabled(step):
472 branch = step.getProperty("branch")
473 return branch and branches[branch].get("usign_key")
475 def IsSignEnabled(step):
476 branch = step.getProperty("branch")
477 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
479 def IsKmodArchiveEnabled(step):
480 branch = step.getProperty("branch")
481 return branch and branches[branch].get("kmod_archive")
483 def GetBaseVersion(branch):
484 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
485 return branch.split('-')[1]
490 def GetVersionPrefix(props):
491 branch = props.getProperty("branch")
492 basever = GetBaseVersion(branch)
493 if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
494 return "%s/" % props["tag"][1:]
495 elif basever != "master":
496 return "%s-SNAPSHOT/" % basever
501 def GetConfigSeed(props):
502 branch = props.getProperty("branch")
503 return branch and branches[branch].get("config_seed") or ""
506 def GetRsyncParams(props, srcorbin, urlorkey):
507 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
508 branch = props.getProperty("branch")
509 opt = srcorbin + "_" + urlorkey
510 return branch and branches[branch].get(opt)
513 def GetUsignKey(props):
514 branch = props.getProperty("branch")
515 return branch and branches[branch].get("usign_key")
517 def GetNextBuild(builder, requests):
520 # order tagged build first
521 if r.properties.hasProperty("tag"):
525 log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
528 def MakeEnv(overrides=None, tryccache=False):
530 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
531 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
534 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
535 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
536 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
538 env['CC'] = env['CCC']
539 env['CXX'] = env['CCXX']
541 if overrides is not None:
542 env.update(overrides)
546 def NetLockDl(props, extralock=None):
548 if props.hasProperty("dl_lock"):
549 lock = NetLocks[props["dl_lock"]]
551 return [lock.access('exclusive')]
556 def NetLockUl(props):
558 if props.hasProperty("ul_lock"):
559 lock = NetLocks[props["ul_lock"]]
561 return [lock.access('exclusive')]
565 def IsTargetSelected(target):
566 def CheckTargetProperty(step):
567 selected_target = step.getProperty("target", "all")
568 if selected_target != "all" and selected_target != target:
572 return CheckTargetProperty
575 def UsignSec2Pub(props):
576 branch = props.getProperty("branch")
578 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
579 seckey = branches[branch].get("usign_key")
580 seckey = base64.b64decode(seckey)
584 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
585 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
592 for worker in c['workers']:
593 workerNames.append(worker.workername)
595 # add a single LocalWorker to handle the forcebuild builder
596 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
598 force_factory = BuildFactory()
599 force_factory.addStep(steps.Trigger(
600 name = "trigger_build",
601 schedulerNames = [ "trigger" ],
602 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
603 set_properties = { "reason": Property("reason"), "tag": Property("tag"), "target": Property("target") },
606 c['builders'].append(BuilderConfig(
607 name = "00_force_build",
608 workername = "__local_force_build",
609 factory = force_factory))
611 for target in targets:
612 ts = target.split('/')
614 factory = BuildFactory()
616 # setup shared work directory if required
617 factory.addStep(ShellCommand(
619 descriptionDone = "Shared work directory set up",
620 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
622 haltOnFailure = True,
625 # find number of cores
626 factory.addStep(SetPropertyFromCommand(
629 description = "Finding number of CPUs",
633 # find gcc and g++ compilers
634 factory.addStep(FileDownload(
635 name = "dlfindbinpl",
636 mastersrc = scripts_dir + '/findbin.pl',
637 workerdest = "../findbin.pl",
641 factory.addStep(SetPropertyFromCommand(
643 property = "cc_command",
644 description = "Finding gcc command",
645 command = ["../findbin.pl", "gcc", "", ""],
646 haltOnFailure = True,
649 factory.addStep(SetPropertyFromCommand(
651 property = "cxx_command",
652 description = "Finding g++ command",
653 command = ["../findbin.pl", "g++", "", ""],
654 haltOnFailure = True,
657 # see if ccache is available
658 factory.addStep(SetPropertyFromCommand(
660 property = "ccache_command",
661 description = "Testing for ccache command",
662 command = ["which", "ccache"],
663 haltOnFailure = False,
664 flunkOnFailure = False,
665 warnOnFailure = False,
666 hideStepIf = lambda r, s: r==results.FAILURE,
669 # check out the source
671 # if repo doesn't exist: 'git clone repourl'
672 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
673 # git cat-file -e <commit>
674 # git checkout -f <commit>
675 # git checkout -B <branch>
683 haltOnFailure = True,
687 factory.addStep(ShellCommand(
689 description = "Fetching Git remote refs",
690 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
691 haltOnFailure = True,
694 # Verify that Git HEAD points to a tag or branch
695 # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
696 factory.addStep(ShellCommand(
698 description = "Ensure that Git HEAD is pointing to a branch or tag",
699 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
700 haltOnFailure = True,
703 factory.addStep(ShellCommand(
705 description = "Remove tmp folder",
706 command=["rm", "-rf", "tmp/"],
710 factory.addStep(ShellCommand(
711 name = "rmfeedlinks",
712 description = "Remove feed symlinks",
713 command=["rm", "-rf", "package/feeds/"],
716 factory.addStep(StringDownload(
718 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
719 workerdest = "../ccache_cc.sh",
723 factory.addStep(StringDownload(
725 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
726 workerdest = "../ccache_cxx.sh",
731 factory.addStep(ShellCommand(
732 name = "updatefeeds",
733 description = "Updating feeds",
734 command=["./scripts/feeds", "update"],
735 env = MakeEnv(tryccache=True),
736 haltOnFailure = True,
741 factory.addStep(ShellCommand(
742 name = "installfeeds",
743 description = "Installing feeds",
744 command=["./scripts/feeds", "install", "-a"],
745 env = MakeEnv(tryccache=True),
746 haltOnFailure = True,
750 factory.addStep(StringDownload(
751 name = "dlconfigseed",
752 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
753 workerdest = ".config",
758 factory.addStep(ShellCommand(
760 descriptionDone = ".config seeded",
761 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
764 factory.addStep(ShellCommand(
766 description = "Removing output directory",
767 command = ["rm", "-rf", "bin/"],
770 factory.addStep(ShellCommand(
772 description = "Populating .config",
773 command = ["make", "defconfig"],
777 # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
778 factory.addStep(ShellCommand(
780 description = "Checking architecture",
781 descriptionDone = "Architecture validated",
782 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
786 haltOnFailure = True,
787 flunkOnFailure = False, # this is not a build FAILURE
791 factory.addStep(SetPropertyFromCommand(
794 description = "Finding libc suffix",
795 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
799 factory.addStep(StringDownload(
800 name = "dlkeybuildpub",
801 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
802 workerdest = "key-build.pub",
804 doStepIf = IsUsignEnabled,
807 factory.addStep(StringDownload(
809 s = "# fake private key",
810 workerdest = "key-build",
812 doStepIf = IsUsignEnabled,
815 factory.addStep(StringDownload(
816 name = "dlkeybuilducert",
817 s = "# fake certificate",
818 workerdest = "key-build.ucert",
820 doStepIf = IsUsignEnabled,
824 factory.addStep(ShellCommand(
826 description = "Preparing dl/",
827 descriptionDone = "dl/ prepared",
828 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
829 workdir = Property("builddir"),
835 factory.addStep(ShellCommand(
837 description = "Pruning dl/",
838 descriptionDone = "dl/ pruned",
839 command = 'find dl/ -atime +15 -delete -print',
844 factory.addStep(ShellCommand(
846 description = "Building and installing GNU tar",
847 descriptionDone = "GNU tar built and installed",
848 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
849 env = MakeEnv(tryccache=True),
850 haltOnFailure = True,
854 factory.addStep(ShellCommand(
856 description = "Populating dl/",
857 descriptionDone = "dl/ populated",
858 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
864 factory.addStep(ShellCommand(
866 description = "Cleaning base-files",
867 command=["make", "package/base-files/clean", "V=s"],
871 factory.addStep(ShellCommand(
873 description = "Building and installing tools",
874 descriptionDone = "Tools built and installed",
875 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
876 env = MakeEnv(tryccache=True),
877 haltOnFailure = True,
880 factory.addStep(ShellCommand(
882 description = "Building and installing toolchain",
883 descriptionDone = "Toolchain built and installed",
884 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
886 haltOnFailure = True,
889 factory.addStep(ShellCommand(
891 description = "Building kmods",
892 descriptionDone = "Kmods built",
893 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
895 haltOnFailure = True,
898 # find kernel version
899 factory.addStep(SetPropertyFromCommand(
900 name = "kernelversion",
901 property = "kernelversion",
902 description = "Finding the effective Kernel version",
903 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
904 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
907 factory.addStep(ShellCommand(
909 description = "Cleaning up package build",
910 descriptionDone = "Package build cleaned up",
911 command=["make", "package/cleanup", "V=s"],
914 factory.addStep(ShellCommand(
916 description = "Building packages",
917 descriptionDone = "Packages built",
918 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
920 haltOnFailure = True,
923 factory.addStep(ShellCommand(
925 description = "Installing packages",
926 descriptionDone = "Packages installed",
927 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
929 haltOnFailure = True,
932 factory.addStep(ShellCommand(
934 description = "Indexing packages",
935 descriptionDone = "Packages indexed",
936 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
938 haltOnFailure = True,
941 factory.addStep(ShellCommand(
943 description = "Building and installing images",
944 descriptionDone = "Images built and installed",
945 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
947 haltOnFailure = True,
950 factory.addStep(ShellCommand(
952 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
953 command = "make -j1 buildinfo V=s || true",
955 haltOnFailure = True,
958 factory.addStep(ShellCommand(
959 name = "json_overview_image_info",
960 description = "Generating profiles.json in target folder",
961 command = "make -j1 json_overview_image_info V=s || true",
963 haltOnFailure = True,
966 factory.addStep(ShellCommand(
968 description = "Calculating checksums",
969 descriptionDone = "Checksums calculated",
970 command=["make", "-j1", "checksum", "V=s"],
972 haltOnFailure = True,
975 factory.addStep(ShellCommand(
977 descriptionDone = "Kmod directory created",
978 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
979 haltOnFailure = True,
980 doStepIf = IsKmodArchiveEnabled,
983 factory.addStep(ShellCommand(
984 name = "kmodprepare",
985 description = "Preparing kmod archive",
986 descriptionDone = "Kmod archive prepared",
987 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
988 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
989 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
990 haltOnFailure = True,
991 doStepIf = IsKmodArchiveEnabled,
994 factory.addStep(ShellCommand(
996 description = "Indexing kmod archive",
997 descriptionDone = "Kmod archive indexed",
998 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
999 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1001 haltOnFailure = True,
1002 doStepIf = IsKmodArchiveEnabled,
1006 factory.addStep(MasterShellCommand(
1007 name = "signprepare",
1008 descriptionDone = "Temporary signing directory prepared",
1009 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1010 haltOnFailure = True,
1011 doStepIf = IsSignEnabled,
1015 factory.addStep(ShellCommand(
1017 description = "Packing files to sign",
1018 descriptionDone = "Files to sign packed",
1019 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1020 haltOnFailure = True,
1021 doStepIf = IsSignEnabled,
1024 factory.addStep(FileUpload(
1025 workersrc = "sign.tar.gz",
1026 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1027 haltOnFailure = True,
1028 doStepIf = IsSignEnabled,
1031 factory.addStep(MasterShellCommand(
1033 description = "Signing files",
1034 descriptionDone = "Files signed",
1035 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
1036 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1037 haltOnFailure = True,
1038 doStepIf = IsSignEnabled,
1041 factory.addStep(FileDownload(
1042 name = "dlsigntargz",
1043 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1044 workerdest = "sign.tar.gz",
1045 haltOnFailure = True,
1046 doStepIf = IsSignEnabled,
1049 factory.addStep(ShellCommand(
1050 name = "signunpack",
1051 description = "Unpacking signed files",
1052 descriptionDone = "Signed files unpacked",
1053 command = ["tar", "-xzf", "sign.tar.gz"],
1054 haltOnFailure = True,
1055 doStepIf = IsSignEnabled,
1059 factory.addStep(ShellCommand(
1060 name = "dirprepare",
1061 descriptionDone = "Upload directory structure prepared",
1062 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1063 haltOnFailure = True,
1066 factory.addStep(ShellCommand(
1067 name = "linkprepare",
1068 descriptionDone = "Repository symlink prepared",
1069 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1070 doStepIf = IsNoMasterBuild,
1071 haltOnFailure = True,
1074 factory.addStep(ShellCommand(
1075 name = "kmoddirprepare",
1076 descriptionDone = "Kmod archive upload directory prepared",
1077 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1078 haltOnFailure = True,
1079 doStepIf = IsKmodArchiveEnabled,
1082 factory.addStep(ShellCommand(
1084 description = "Uploading directory structure",
1085 descriptionDone = "Directory structure uploaded",
1086 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1087 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1088 haltOnFailure = True,
1091 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1094 # download remote sha256sums to 'target-sha256sums'
1095 factory.addStep(ShellCommand(
1096 name = "target-sha256sums",
1097 description = "Fetching remote sha256sums for target",
1098 descriptionDone = "Remote sha256sums for target fetched",
1099 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1100 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1102 haltOnFailure = False,
1103 flunkOnFailure = False,
1104 warnOnFailure = False,
1105 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1108 # build list of files to upload
1109 factory.addStep(FileDownload(
1110 name = "dlsha2rsyncpl",
1111 mastersrc = scripts_dir + '/sha2rsync.pl',
1112 workerdest = "../sha2rsync.pl",
1116 factory.addStep(ShellCommand(
1118 description = "Building list of files to upload",
1119 descriptionDone = "List of files to upload built",
1120 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1121 haltOnFailure = True,
1124 factory.addStep(FileDownload(
1125 name = "dlrsync.sh",
1126 mastersrc = scripts_dir + '/rsync.sh',
1127 workerdest = "../rsync.sh",
1131 # upload new files and update existing ones
1132 factory.addStep(ShellCommand(
1133 name = "targetupload",
1134 description = "Uploading target files",
1135 descriptionDone = "Target files uploaded",
1136 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1137 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1138 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1139 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1140 haltOnFailure = True,
1142 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1145 # delete files which don't exist locally
1146 factory.addStep(ShellCommand(
1147 name = "targetprune",
1148 description = "Pruning target files",
1149 descriptionDone = "Target files pruned",
1150 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1151 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1152 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1153 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1154 haltOnFailure = True,
1157 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1160 factory.addStep(ShellCommand(
1161 name = "kmodupload",
1162 description = "Uploading kmod archive",
1163 descriptionDone = "Kmod archive uploaded",
1164 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1165 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1166 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1167 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1168 haltOnFailure = True,
1171 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1174 factory.addStep(ShellCommand(
1175 name = "sourcelist",
1176 description = "Finding source archives to upload",
1177 descriptionDone = "Source archives to upload found",
1178 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1179 haltOnFailure = True,
1182 factory.addStep(ShellCommand(
1183 name = "sourceupload",
1184 description = "Uploading source archives",
1185 descriptionDone = "Source archives uploaded",
1186 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1187 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1188 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1189 haltOnFailure = True,
1192 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1195 factory.addStep(ShellCommand(
1197 description = "Reporting disk usage",
1198 command=["df", "-h", "."],
1199 env={'LC_ALL': 'C'},
1201 haltOnFailure = False,
1202 flunkOnFailure = False,
1203 warnOnFailure = False,
1207 factory.addStep(ShellCommand(
1209 description = "Reporting estimated file space usage",
1210 command=["du", "-sh", "."],
1211 env={'LC_ALL': 'C'},
1213 haltOnFailure = False,
1214 flunkOnFailure = False,
1215 warnOnFailure = False,
1219 factory.addStep(ShellCommand(
1220 name = "ccachestat",
1221 description = "Reporting ccache stats",
1222 command=["ccache", "-s"],
1223 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1225 want_stderr = False,
1226 haltOnFailure = False,
1227 flunkOnFailure = False,
1228 warnOnFailure = False,
1229 hideStepIf = lambda r, s: r==results.FAILURE,
1232 for brname in branchNames:
1233 bldrname = brname + "_" + target
1234 c['builders'].append(BuilderConfig(name=bldrname, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1237 ####### STATUS TARGETS
1239 # 'status' is a list of Status Targets. The results of each build will be
1240 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1241 # including web pages, email senders, and IRC bots.
1243 if "status_bind" in inip1:
1245 'port': inip1.get("status_bind"),
1247 'waterfall_view': True,
1248 'console_view': True,
1253 if "status_user" in inip1 and "status_password" in inip1:
1254 c['www']['auth'] = util.UserPasswordAuth([
1255 (inip1.get("status_user"), inip1.get("status_password"))
1257 c['www']['authz'] = util.Authz(
1258 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1259 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1263 if ini.has_section("irc"):
1265 irc_host = iniirc.get("host", None)
1266 irc_port = iniirc.getint("port", 6667)
1267 irc_chan = iniirc.get("channel", None)
1268 irc_nick = iniirc.get("nickname", None)
1269 irc_pass = iniirc.get("password", None)
1271 if irc_host and irc_nick and irc_chan:
1272 irc = reporters.IRC(irc_host, irc_nick,
1274 password = irc_pass,
1275 channels = [ irc_chan ],
1276 notify_events = [ 'exception', 'problem', 'recovery' ]
1279 c['services'].append(irc)
1281 c['revlink'] = util.RevlinkMatch([
1282 r'https://git.openwrt.org/openwrt/(.*).git'
1284 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1289 # This specifies what database buildbot uses to store its state. You can leave
1290 # this at its default for all but the largest installations.
1291 'db_url' : "sqlite:///state.sqlite",
1294 c['buildbotNetUsageData'] = None