2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes.gitpoller import GitPoller
19 from buildbot.config import BuilderConfig
20 from buildbot.plugins import reporters
21 from buildbot.plugins import schedulers
22 from buildbot.plugins import steps
23 from buildbot.plugins import util
24 from buildbot.process import properties
25 from buildbot.process import results
26 from buildbot.process.factory import BuildFactory
27 from buildbot.process.properties import Interpolate
28 from buildbot.process.properties import Property
29 from buildbot.schedulers.basic import AnyBranchScheduler
30 from buildbot.schedulers.forcesched import BaseParameter
31 from buildbot.schedulers.forcesched import ForceScheduler
32 from buildbot.schedulers.forcesched import ValidationError
33 from buildbot.steps.master import MasterShellCommand
34 from buildbot.steps.shell import SetPropertyFromCommand
35 from buildbot.steps.shell import ShellCommand
36 from buildbot.steps.source.git import Git
37 from buildbot.steps.transfer import FileDownload
38 from buildbot.steps.transfer import FileUpload
39 from buildbot.steps.transfer import StringDownload
40 from buildbot.worker import Worker
41 from buildbot.worker.local import LocalWorker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini['repo'].get("url")
65 rsync_defopts = ["-v", "-4", "--timeout=120"]
67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
68 # rsync_bin_defopts += ["--contimeout=20"]
72 def ini_parse_branch(section):
74 name = section.get("name")
77 raise ValueError("missing 'name' in " + repr(section))
79 raise ValueError("duplicate branch name in " + repr(section))
82 b["bin_url"] = section.get("binary_url")
83 b["bin_key"] = section.get("binary_password")
85 b["src_url"] = section.get("source_url")
86 b["src_key"] = section.get("source_password")
88 b["gpg_key"] = section.get("gpg_key")
90 b["usign_key"] = section.get("usign_key")
91 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
92 b["usign_comment"] = section.get("usign_comment", usign_comment)
94 b["config_seed"] = section.get("config_seed")
96 b["kmod_archive"] = section.getboolean("kmod_archive", False)
99 log.msg("Configured branch: {}".format(name))
101 # PB port can be either a numeric port or a connection string
102 pb_port = inip1.get("port") or 9989
104 # This is the dictionary that the buildmaster pays attention to. We also use
105 # a shorter alias to save typing.
106 c = BuildmasterConfig = {}
108 ####### PROJECT IDENTITY
110 # the 'title' string will appear at the top of this buildbot
111 # installation's html.WebStatus home page (linked to the
112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
114 c['title'] = ini['general'].get("title")
115 c['titleURL'] = ini['general'].get("title_url")
117 # the 'buildbotURL' string should point to the location where the buildbot's
118 # internal web server (usually the html.WebStatus page) is visible. This
119 # typically uses the port number set in the Waterfall 'status' entry, but
120 # with an externally-visible host name which the buildbot cannot figure out
123 c['buildbotURL'] = inip1.get("buildbot_url")
127 # The 'workers' list defines the set of recognized buildworkers. Each element is
128 # a Worker object, specifying a unique worker name and password. The same
129 # worker name and password must be configured on the worker.
134 for section in ini.sections():
135 if section.startswith("branch "):
136 ini_parse_branch(ini[section])
138 if section.startswith("worker "):
139 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
140 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
141 sl_props = { 'dl_lock':None, 'ul_lock':None }
142 name = ini.get(section, "name")
143 password = ini.get(section, "password")
144 if ini.has_option(section, "dl_lock"):
145 lockname = ini.get(section, "dl_lock")
146 sl_props['dl_lock'] = lockname
147 if lockname not in NetLocks:
148 NetLocks[lockname] = locks.MasterLock(lockname)
149 if ini.has_option(section, "ul_lock"):
150 lockname = ini.get(section, "ul_lock")
151 sl_props['ul_lock'] = lockname
152 if lockname not in NetLocks:
153 NetLocks[lockname] = locks.MasterLock(lockname)
154 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
156 c['protocols'] = {'pb': {'port': pb_port}}
159 c['collapseRequests'] = True
161 # Reduce amount of backlog data
162 c['configurators'] = [util.JanitorConfigurator(
163 logHorizon=timedelta(days=3),
167 @defer.inlineCallbacks
168 def getNewestCompleteTime(bldr):
169 """Returns the complete_at of the latest completed and not SKIPPED
170 build request for this builder, or None if there are no such build
171 requests. We need to filter out SKIPPED requests because we're
172 using collapseRequests=True which is unfortunately marking all
173 previous requests as complete when new buildset is created.
175 @returns: datetime instance or None, via Deferred
178 bldrid = yield bldr.getBuilderId()
179 completed = yield bldr.master.data.get(
180 ('builders', bldrid, 'buildrequests'),
182 resultspec.Filter('complete', 'eq', [True]),
183 resultspec.Filter('results', 'ne', [results.SKIPPED]),
185 order=['-complete_at'], limit=1)
189 complete_at = completed[0]['complete_at']
191 last_build = yield bldr.master.data.get(
194 resultspec.Filter('builderid', 'eq', [bldrid]),
196 order=['-started_at'], limit=1)
198 if last_build and last_build[0]:
199 last_complete_at = last_build[0]['complete_at']
200 if last_complete_at and (last_complete_at > complete_at):
201 return last_complete_at
205 @defer.inlineCallbacks
206 def prioritizeBuilders(master, builders):
207 """Returns sorted list of builders by their last timestamp of completed and
210 @returns: list of sorted builders
213 def is_building(bldr):
214 return bool(bldr.building) or bool(bldr.old_building)
217 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
218 d.addCallback(lambda complete_at: (complete_at, bldr))
222 (complete_at, bldr) = item
224 if bldr.name == "00_force_build":
226 complete_at = date.replace(tzinfo=tzutc())
227 return (complete_at, bldr.name)
231 complete_at = date.replace(tzinfo=tzutc())
233 if is_building(bldr):
235 complete_at = date.replace(tzinfo=tzutc())
237 return (complete_at, bldr.name)
239 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
240 results.sort(key=bldr_sort)
243 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
245 return [r[1] for r in results]
247 c['prioritizeBuilders'] = prioritizeBuilders
249 ####### CHANGESOURCES
251 branchNames = [branches[b]["name"] for b in branches]
256 def populateTargets():
257 log.msg("Populating targets, this will take time")
258 sourcegit = work_dir + '/source.git'
259 for branch in branchNames:
260 if os.path.isdir(sourcegit):
261 subprocess.call(["rm", "-rf", sourcegit])
263 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
265 os.makedirs(sourcegit + '/tmp', exist_ok=True)
266 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
267 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
270 line = findtargets.stdout.readline()
273 ta = line.decode().strip().split(' ')
276 subprocess.call(["rm", "-rf", sourcegit])
280 # the 'change_source' setting tells the buildmaster how it should find out
281 # about source code changes. Here we point to the buildbot clone of pyflakes.
283 c['change_source'] = []
284 c['change_source'].append(GitPoller(
286 workdir=work_dir+'/work.git', branches=branchNames,
287 pollAtLaunch=True, pollinterval=300))
291 # Configure the Schedulers, which decide how to react to incoming changes. In this
292 # case, just kick off a 'basebuild' build
294 class TagChoiceParameter(BaseParameter):
295 spec_attributes = ["strict", "choices"]
299 def __init__(self, name, label=None, **kw):
300 super().__init__(name, label, **kw)
301 self._choice_list = []
303 def getRevTags(self, findtag=None):
307 for b in branchNames:
308 basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
310 branchvers.append(basever[1])
312 alltags = subprocess.Popen(
313 ['git', 'ls-remote', '--tags', repo_url],
314 stdout = subprocess.PIPE)
317 line = alltags.stdout.readline()
322 (rev, tag) = line.split()
324 tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
326 # only list tags matching configured branches
327 if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
328 if findtag and findtag != tagver[1]:
330 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
336 taglist = [rt['tag'] for rt in self.getRevTags()]
337 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
338 taglist.insert(0, '')
340 self._choice_list = taglist
342 return self._choice_list
344 def updateFromKwargs(self, properties, kwargs, **unused):
345 tag = self.getFromKwargs(kwargs)
346 properties[self.name] = tag
348 # find the commit matching the tag
349 findtag = self.getRevTags(tag)
352 raise ValidationError("Couldn't find tag")
354 properties['force_revision'] = findtag[0]['rev']
356 # find the branch matching the tag
358 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
359 for b in branchNames:
360 if b.endswith(branchver[1]):
364 raise ValidationError("Couldn't find branch")
366 properties['force_branch'] = branch
368 def parse_from_arg(self, s):
369 if self.strict and s not in self._choice_list:
370 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
374 @defer.inlineCallbacks
375 def builderNames(props):
376 """ since we have per branch and per target builders,
377 address the relevant builder for each new buildrequest
378 based on the request's desired branch and target.
380 branch = props.getProperty("branch")
381 target = props.getProperty("target", "")
386 # if that didn't work, try sourcestamp to find a branch
388 # match builders with target branch
389 ss = props.sourcestamps[0]
391 branch = ss['branch']
393 log.msg("couldn't find builder")
394 return [] # nothing works
396 bname = branch + "_" + target
399 for b in (yield props.master.data.get(('builders',))):
400 if not b['name'].startswith(bname):
402 builders.append(b['name'])
407 c['schedulers'].append(AnyBranchScheduler(
409 change_filter = util.ChangeFilter(branch=branchNames),
410 treeStableTimer = 15*60,
411 builderNames = builderNames))
413 c['schedulers'].append(ForceScheduler(
415 buttonName = "Force builds",
416 label = "Force build details",
417 builderNames = [ "00_force_build" ],
420 util.CodebaseParameter(
422 label = "Repository",
423 branch = util.FixedParameter(name = "branch", default = ""),
424 revision = util.FixedParameter(name = "revision", default = ""),
425 repository = util.FixedParameter(name = "repository", default = ""),
426 project = util.FixedParameter(name = "project", default = "")
430 reason = util.StringParameter(
433 default = "Trigger build",
439 util.ChoiceStringParameter(
441 label = "Build target",
443 choices = [ "all" ] + list(targets)
453 c['schedulers'].append(schedulers.Triggerable(name="trigger", builderNames=builderNames))
457 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
458 # what steps, and which workers can execute them. Note that any particular build will
459 # only take place on one worker.
461 def IsNoMasterBuild(step):
462 return step.getProperty("branch") != "master"
464 def IsUsignEnabled(step):
465 branch = step.getProperty("branch")
466 return branch and branches[branch].get("usign_key")
468 def IsSignEnabled(step):
469 branch = step.getProperty("branch")
470 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
472 def IsKmodArchiveEnabled(step):
473 branch = step.getProperty("branch")
474 return branch and branches[branch].get("kmod_archive")
476 def GetBaseVersion(branch):
477 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
478 return branch.split('-')[1]
483 def GetVersionPrefix(props):
484 branch = props.getProperty("branch")
485 basever = GetBaseVersion(branch)
486 if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
487 return "%s/" % props["tag"][1:]
488 elif basever != "master":
489 return "%s-SNAPSHOT/" % basever
494 def GetConfigSeed(props):
495 branch = props.getProperty("branch")
496 return branch and branches[branch].get("config_seed") or ""
499 def GetRsyncParams(props, srcorbin, urlorkey):
500 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
501 branch = props.getProperty("branch")
502 opt = srcorbin + "_" + urlorkey
503 return branch and branches[branch].get(opt)
506 def GetUsignKey(props):
507 branch = props.getProperty("branch")
508 return branch and branches[branch].get("usign_key")
510 def GetNextBuild(builder, requests):
513 # order tagged build first
514 if r.properties.hasProperty("tag"):
516 # then order by branch order
517 pbranch = r.properties.getProperty("branch")
518 for name in branchNames:
523 log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
526 def MakeEnv(overrides=None, tryccache=False):
528 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
529 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
532 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
533 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
534 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
536 env['CC'] = env['CCC']
537 env['CXX'] = env['CCXX']
539 if overrides is not None:
540 env.update(overrides)
544 def NetLockDl(props, extralock=None):
546 if props.hasProperty("dl_lock"):
547 lock = NetLocks[props["dl_lock"]]
549 return [lock.access('exclusive')]
554 def NetLockUl(props):
556 if props.hasProperty("ul_lock"):
557 lock = NetLocks[props["ul_lock"]]
559 return [lock.access('exclusive')]
563 def IsTargetSelected(target):
564 def CheckTargetProperty(step):
565 selected_target = step.getProperty("target", "all")
566 if selected_target != "all" and selected_target != target:
570 return CheckTargetProperty
573 def UsignSec2Pub(props):
574 branch = props.getProperty("branch")
576 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
577 seckey = branches[branch].get("usign_key")
578 seckey = base64.b64decode(seckey)
582 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
583 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
590 for worker in c['workers']:
591 workerNames.append(worker.workername)
593 # add a single LocalWorker to handle the forcebuild builder
594 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
596 force_factory = BuildFactory()
597 force_factory.addStep(steps.Trigger(
598 name = "trigger_build",
599 schedulerNames = [ "trigger" ],
600 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
601 set_properties = { "reason": Property("reason"), "tag": Property("tag"), "target": Property("target") },
604 c['builders'].append(BuilderConfig(
605 name = "00_force_build",
606 workername = "__local_force_build",
607 factory = force_factory))
609 for target in targets:
610 ts = target.split('/')
612 factory = BuildFactory()
614 # setup shared work directory if required
615 factory.addStep(ShellCommand(
617 descriptionDone = "Shared work directory set up",
618 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
620 haltOnFailure = True,
623 # find number of cores
624 factory.addStep(SetPropertyFromCommand(
627 description = "Finding number of CPUs",
631 # find gcc and g++ compilers
632 factory.addStep(FileDownload(
633 name = "dlfindbinpl",
634 mastersrc = scripts_dir + '/findbin.pl',
635 workerdest = "../findbin.pl",
639 factory.addStep(SetPropertyFromCommand(
641 property = "cc_command",
642 description = "Finding gcc command",
643 command = ["../findbin.pl", "gcc", "", ""],
644 haltOnFailure = True,
647 factory.addStep(SetPropertyFromCommand(
649 property = "cxx_command",
650 description = "Finding g++ command",
651 command = ["../findbin.pl", "g++", "", ""],
652 haltOnFailure = True,
655 # see if ccache is available
656 factory.addStep(SetPropertyFromCommand(
658 property = "ccache_command",
659 description = "Testing for ccache command",
660 command = ["which", "ccache"],
661 haltOnFailure = False,
662 flunkOnFailure = False,
663 warnOnFailure = False,
664 hideStepIf = lambda r, s: r==results.FAILURE,
667 # check out the source
669 # if repo doesn't exist: 'git clone repourl'
670 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
671 # git cat-file -e <commit>
672 # git checkout -f <commit>
673 # git checkout -B <branch>
681 haltOnFailure = True,
685 factory.addStep(ShellCommand(
687 description = "Fetching Git remote refs",
688 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
689 haltOnFailure = True,
692 # Verify that Git HEAD points to a tag or branch
693 # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
694 factory.addStep(ShellCommand(
696 description = "Ensure that Git HEAD is pointing to a branch or tag",
697 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
698 haltOnFailure = True,
701 factory.addStep(ShellCommand(
703 description = "Remove tmp folder",
704 command=["rm", "-rf", "tmp/"],
708 factory.addStep(ShellCommand(
709 name = "rmfeedlinks",
710 description = "Remove feed symlinks",
711 command=["rm", "-rf", "package/feeds/"],
714 factory.addStep(StringDownload(
716 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
717 workerdest = "../ccache_cc.sh",
721 factory.addStep(StringDownload(
723 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
724 workerdest = "../ccache_cxx.sh",
729 factory.addStep(ShellCommand(
730 name = "updatefeeds",
731 description = "Updating feeds",
732 command=["./scripts/feeds", "update"],
733 env = MakeEnv(tryccache=True),
734 haltOnFailure = True,
739 factory.addStep(ShellCommand(
740 name = "installfeeds",
741 description = "Installing feeds",
742 command=["./scripts/feeds", "install", "-a"],
743 env = MakeEnv(tryccache=True),
744 haltOnFailure = True,
748 factory.addStep(StringDownload(
749 name = "dlconfigseed",
750 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
751 workerdest = ".config",
756 factory.addStep(ShellCommand(
758 descriptionDone = ".config seeded",
759 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
762 factory.addStep(ShellCommand(
764 description = "Removing output directory",
765 command = ["rm", "-rf", "bin/"],
768 factory.addStep(ShellCommand(
770 description = "Populating .config",
771 command = ["make", "defconfig"],
775 # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
776 factory.addStep(ShellCommand(
778 description = "Checking architecture",
779 descriptionDone = "Architecture validated",
780 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
784 haltOnFailure = True,
785 flunkOnFailure = False, # this is not a build FAILURE
789 factory.addStep(SetPropertyFromCommand(
792 description = "Finding libc suffix",
793 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
797 factory.addStep(StringDownload(
798 name = "dlkeybuildpub",
799 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
800 workerdest = "key-build.pub",
802 doStepIf = IsUsignEnabled,
805 factory.addStep(StringDownload(
807 s = "# fake private key",
808 workerdest = "key-build",
810 doStepIf = IsUsignEnabled,
813 factory.addStep(StringDownload(
814 name = "dlkeybuilducert",
815 s = "# fake certificate",
816 workerdest = "key-build.ucert",
818 doStepIf = IsUsignEnabled,
822 factory.addStep(ShellCommand(
824 description = "Preparing dl/",
825 descriptionDone = "dl/ prepared",
826 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
827 workdir = Property("builddir"),
833 factory.addStep(ShellCommand(
835 description = "Pruning dl/",
836 descriptionDone = "dl/ pruned",
837 command = 'find dl/ -atime +15 -delete -print',
842 factory.addStep(ShellCommand(
844 description = "Building and installing GNU tar",
845 descriptionDone = "GNU tar built and installed",
846 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
847 env = MakeEnv(tryccache=True),
848 haltOnFailure = True,
852 factory.addStep(ShellCommand(
854 description = "Populating dl/",
855 descriptionDone = "dl/ populated",
856 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
862 factory.addStep(ShellCommand(
864 description = "Cleaning base-files",
865 command=["make", "package/base-files/clean", "V=s"],
869 factory.addStep(ShellCommand(
871 description = "Building and installing tools",
872 descriptionDone = "Tools built and installed",
873 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
874 env = MakeEnv(tryccache=True),
875 haltOnFailure = True,
878 factory.addStep(ShellCommand(
880 description = "Building and installing toolchain",
881 descriptionDone = "Toolchain built and installed",
882 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
884 haltOnFailure = True,
887 factory.addStep(ShellCommand(
889 description = "Building kmods",
890 descriptionDone = "Kmods built",
891 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
893 haltOnFailure = True,
896 # find kernel version
897 factory.addStep(SetPropertyFromCommand(
898 name = "kernelversion",
899 property = "kernelversion",
900 description = "Finding the effective Kernel version",
901 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
902 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
905 factory.addStep(ShellCommand(
907 description = "Cleaning up package build",
908 descriptionDone = "Package build cleaned up",
909 command=["make", "package/cleanup", "V=s"],
912 factory.addStep(ShellCommand(
914 description = "Building packages",
915 descriptionDone = "Packages built",
916 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
918 haltOnFailure = True,
921 factory.addStep(ShellCommand(
923 description = "Installing packages",
924 descriptionDone = "Packages installed",
925 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
927 haltOnFailure = True,
930 factory.addStep(ShellCommand(
932 description = "Indexing packages",
933 descriptionDone = "Packages indexed",
934 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
936 haltOnFailure = True,
939 factory.addStep(ShellCommand(
941 description = "Building and installing images",
942 descriptionDone = "Images built and installed",
943 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
945 haltOnFailure = True,
948 factory.addStep(ShellCommand(
950 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
951 command = "make -j1 buildinfo V=s || true",
953 haltOnFailure = True,
956 factory.addStep(ShellCommand(
957 name = "json_overview_image_info",
958 description = "Generating profiles.json in target folder",
959 command = "make -j1 json_overview_image_info V=s || true",
961 haltOnFailure = True,
964 factory.addStep(ShellCommand(
966 description = "Calculating checksums",
967 descriptionDone = "Checksums calculated",
968 command=["make", "-j1", "checksum", "V=s"],
970 haltOnFailure = True,
973 factory.addStep(ShellCommand(
975 descriptionDone = "Kmod directory created",
976 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
977 haltOnFailure = True,
978 doStepIf = IsKmodArchiveEnabled,
981 factory.addStep(ShellCommand(
982 name = "kmodprepare",
983 description = "Preparing kmod archive",
984 descriptionDone = "Kmod archive prepared",
985 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
986 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
987 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
988 haltOnFailure = True,
989 doStepIf = IsKmodArchiveEnabled,
992 factory.addStep(ShellCommand(
994 description = "Indexing kmod archive",
995 descriptionDone = "Kmod archive indexed",
996 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
997 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
999 haltOnFailure = True,
1000 doStepIf = IsKmodArchiveEnabled,
1004 factory.addStep(MasterShellCommand(
1005 name = "signprepare",
1006 descriptionDone = "Temporary signing directory prepared",
1007 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1008 haltOnFailure = True,
1009 doStepIf = IsSignEnabled,
1013 factory.addStep(ShellCommand(
1015 description = "Packing files to sign",
1016 descriptionDone = "Files to sign packed",
1017 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1018 haltOnFailure = True,
1019 doStepIf = IsSignEnabled,
1022 factory.addStep(FileUpload(
1023 workersrc = "sign.tar.gz",
1024 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1025 haltOnFailure = True,
1026 doStepIf = IsSignEnabled,
1029 factory.addStep(MasterShellCommand(
1031 description = "Signing files",
1032 descriptionDone = "Files signed",
1033 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
1034 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1035 haltOnFailure = True,
1036 doStepIf = IsSignEnabled,
1039 factory.addStep(FileDownload(
1040 name = "dlsigntargz",
1041 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1042 workerdest = "sign.tar.gz",
1043 haltOnFailure = True,
1044 doStepIf = IsSignEnabled,
1047 factory.addStep(ShellCommand(
1048 name = "signunpack",
1049 description = "Unpacking signed files",
1050 descriptionDone = "Signed files unpacked",
1051 command = ["tar", "-xzf", "sign.tar.gz"],
1052 haltOnFailure = True,
1053 doStepIf = IsSignEnabled,
1057 factory.addStep(ShellCommand(
1058 name = "dirprepare",
1059 descriptionDone = "Upload directory structure prepared",
1060 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1061 haltOnFailure = True,
1064 factory.addStep(ShellCommand(
1065 name = "linkprepare",
1066 descriptionDone = "Repository symlink prepared",
1067 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1068 doStepIf = IsNoMasterBuild,
1069 haltOnFailure = True,
1072 factory.addStep(ShellCommand(
1073 name = "kmoddirprepare",
1074 descriptionDone = "Kmod archive upload directory prepared",
1075 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1076 haltOnFailure = True,
1077 doStepIf = IsKmodArchiveEnabled,
1080 factory.addStep(ShellCommand(
1082 description = "Uploading directory structure",
1083 descriptionDone = "Directory structure uploaded",
1084 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1085 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1086 haltOnFailure = True,
1089 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1092 # download remote sha256sums to 'target-sha256sums'
1093 factory.addStep(ShellCommand(
1094 name = "target-sha256sums",
1095 description = "Fetching remote sha256sums for target",
1096 descriptionDone = "Remote sha256sums for target fetched",
1097 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1098 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1100 haltOnFailure = False,
1101 flunkOnFailure = False,
1102 warnOnFailure = False,
1103 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1106 # build list of files to upload
1107 factory.addStep(FileDownload(
1108 name = "dlsha2rsyncpl",
1109 mastersrc = scripts_dir + '/sha2rsync.pl',
1110 workerdest = "../sha2rsync.pl",
1114 factory.addStep(ShellCommand(
1116 description = "Building list of files to upload",
1117 descriptionDone = "List of files to upload built",
1118 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1119 haltOnFailure = True,
1122 factory.addStep(FileDownload(
1123 name = "dlrsync.sh",
1124 mastersrc = scripts_dir + '/rsync.sh',
1125 workerdest = "../rsync.sh",
1129 # upload new files and update existing ones
1130 factory.addStep(ShellCommand(
1131 name = "targetupload",
1132 description = "Uploading target files",
1133 descriptionDone = "Target files uploaded",
1134 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1135 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1136 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1137 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1138 haltOnFailure = True,
1140 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1143 # delete files which don't exist locally
1144 factory.addStep(ShellCommand(
1145 name = "targetprune",
1146 description = "Pruning target files",
1147 descriptionDone = "Target files pruned",
1148 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1149 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1150 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1151 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1152 haltOnFailure = True,
1155 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1158 factory.addStep(ShellCommand(
1159 name = "kmodupload",
1160 description = "Uploading kmod archive",
1161 descriptionDone = "Kmod archive uploaded",
1162 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1163 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1164 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1165 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1166 haltOnFailure = True,
1169 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1172 factory.addStep(ShellCommand(
1173 name = "sourcelist",
1174 description = "Finding source archives to upload",
1175 descriptionDone = "Source archives to upload found",
1176 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1177 haltOnFailure = True,
1180 factory.addStep(ShellCommand(
1181 name = "sourceupload",
1182 description = "Uploading source archives",
1183 descriptionDone = "Source archives uploaded",
1184 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1185 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1186 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1187 haltOnFailure = True,
1190 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1193 factory.addStep(ShellCommand(
1195 description = "Reporting disk usage",
1196 command=["df", "-h", "."],
1197 env={'LC_ALL': 'C'},
1199 haltOnFailure = False,
1200 flunkOnFailure = False,
1201 warnOnFailure = False,
1205 factory.addStep(ShellCommand(
1207 description = "Reporting estimated file space usage",
1208 command=["du", "-sh", "."],
1209 env={'LC_ALL': 'C'},
1211 haltOnFailure = False,
1212 flunkOnFailure = False,
1213 warnOnFailure = False,
1217 factory.addStep(ShellCommand(
1218 name = "ccachestat",
1219 description = "Reporting ccache stats",
1220 command=["ccache", "-s"],
1221 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1223 want_stderr = False,
1224 haltOnFailure = False,
1225 flunkOnFailure = False,
1226 warnOnFailure = False,
1227 hideStepIf = lambda r, s: r==results.FAILURE,
1230 for brname in branchNames:
1231 bldrname = brname + "_" + target
1232 c['builders'].append(BuilderConfig(name=bldrname, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1235 ####### STATUS TARGETS
1237 # 'status' is a list of Status Targets. The results of each build will be
1238 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1239 # including web pages, email senders, and IRC bots.
1241 if "status_bind" in inip1:
1243 'port': inip1.get("status_bind"),
1245 'waterfall_view': True,
1246 'console_view': True,
1251 if "status_user" in inip1 and "status_password" in inip1:
1252 c['www']['auth'] = util.UserPasswordAuth([
1253 (inip1.get("status_user"), inip1.get("status_password"))
1255 c['www']['authz'] = util.Authz(
1256 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1257 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1261 if ini.has_section("irc"):
1263 irc_host = iniirc.get("host", None)
1264 irc_port = iniirc.getint("port", 6667)
1265 irc_chan = iniirc.get("channel", None)
1266 irc_nick = iniirc.get("nickname", None)
1267 irc_pass = iniirc.get("password", None)
1269 if irc_host and irc_nick and irc_chan:
1270 irc = reporters.IRC(irc_host, irc_nick,
1272 password = irc_pass,
1273 channels = [ irc_chan ],
1274 notify_events = [ 'exception', 'problem', 'recovery' ]
1277 c['services'].append(irc)
1279 c['revlink'] = util.RevlinkMatch([
1280 r'https://git.openwrt.org/openwrt/(.*).git'
1282 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1287 # This specifies what database buildbot uses to store its state. You can leave
1288 # this at its default for all but the largest installations.
1289 'db_url' : "sqlite:///state.sqlite",
1292 c['buildbotNetUsageData'] = None