2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes.gitpoller import GitPoller
19 from buildbot.config import BuilderConfig
20 from buildbot.plugins import reporters
21 from buildbot.plugins import schedulers
22 from buildbot.plugins import steps
23 from buildbot.plugins import util
24 from buildbot.process import properties
25 from buildbot.process import results
26 from buildbot.process.factory import BuildFactory
27 from buildbot.process.properties import Interpolate
28 from buildbot.process.properties import Property
29 from buildbot.schedulers.basic import AnyBranchScheduler
30 from buildbot.schedulers.forcesched import BaseParameter
31 from buildbot.schedulers.forcesched import ForceScheduler
32 from buildbot.schedulers.forcesched import ValidationError
33 from buildbot.steps.master import MasterShellCommand
34 from buildbot.steps.shell import SetPropertyFromCommand
35 from buildbot.steps.shell import ShellCommand
36 from buildbot.steps.source.git import Git
37 from buildbot.steps.transfer import FileDownload
38 from buildbot.steps.transfer import FileUpload
39 from buildbot.steps.transfer import StringDownload
40 from buildbot.worker import Worker
41 from buildbot.worker.local import LocalWorker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini['repo'].get("url")
65 rsync_defopts = ["-v", "-4", "--timeout=120"]
67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
68 # rsync_bin_defopts += ["--contimeout=20"]
72 def ini_parse_branch(section):
74 name = section.get("name")
77 raise ValueError("missing 'name' in " + repr(section))
79 raise ValueError("duplicate branch name in " + repr(section))
82 b["bin_url"] = section.get("binary_url")
83 b["bin_key"] = section.get("binary_password")
85 b["src_url"] = section.get("source_url")
86 b["src_key"] = section.get("source_password")
88 b["gpg_key"] = section.get("gpg_key")
90 b["usign_key"] = section.get("usign_key")
91 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
92 b["usign_comment"] = section.get("usign_comment", usign_comment)
94 b["config_seed"] = section.get("config_seed")
96 b["kmod_archive"] = section.getboolean("kmod_archive", False)
99 log.msg("Configured branch: {}".format(name))
101 # PB port can be either a numeric port or a connection string
102 pb_port = inip1.get("port") or 9989
104 # This is the dictionary that the buildmaster pays attention to. We also use
105 # a shorter alias to save typing.
106 c = BuildmasterConfig = {}
108 ####### PROJECT IDENTITY
110 # the 'title' string will appear at the top of this buildbot
111 # installation's html.WebStatus home page (linked to the
112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
114 c['title'] = ini['general'].get("title")
115 c['titleURL'] = ini['general'].get("title_url")
117 # the 'buildbotURL' string should point to the location where the buildbot's
118 # internal web server (usually the html.WebStatus page) is visible. This
119 # typically uses the port number set in the Waterfall 'status' entry, but
120 # with an externally-visible host name which the buildbot cannot figure out
123 c['buildbotURL'] = inip1.get("buildbot_url")
127 # The 'workers' list defines the set of recognized buildworkers. Each element is
128 # a Worker object, specifying a unique worker name and password. The same
129 # worker name and password must be configured on the worker.
134 def ini_parse_workers(section):
135 name = section.get("name")
136 password = section.get("password")
137 phase = section.getint("phase")
138 tagonly = section.getboolean("tag_only")
140 if not name or not password or not phase == 1:
141 log.msg("invalid worker configuration ignored: {}".format(repr(section)))
144 sl_props = { 'dl_lock':None, 'ul_lock':None, 'tag_only':tagonly }
145 if "dl_lock" in section:
146 lockname = section.get("dl_lock")
147 sl_props['dl_lock'] = lockname
148 if lockname not in NetLocks:
149 NetLocks[lockname] = locks.MasterLock(lockname)
150 if "ul_lock" in section:
151 lockname = section.get("ul_lock")
152 sl_props['ul_lock'] = lockname
153 if lockname not in NetLocks:
154 NetLocks[lockname] = locks.MasterLock(lockname)
156 log.msg("Configured worker: {}".format(name))
157 # NB: phase1 build factory requires workers to be single-build only
158 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
161 for section in ini.sections():
162 if section.startswith("branch "):
163 ini_parse_branch(ini[section])
165 if section.startswith("worker "):
166 ini_parse_workers(ini[section])
168 # list of branches in build-priority order
169 branchNames = [branches[b]["name"] for b in branches]
171 c['protocols'] = {'pb': {'port': pb_port}}
174 c['collapseRequests'] = True
176 # Reduce amount of backlog data
177 c['configurators'] = [util.JanitorConfigurator(
178 logHorizon=timedelta(days=3),
182 @defer.inlineCallbacks
183 def getNewestCompleteTime(bldr):
184 """Returns the complete_at of the latest completed and not SKIPPED
185 build request for this builder, or None if there are no such build
186 requests. We need to filter out SKIPPED requests because we're
187 using collapseRequests=True which is unfortunately marking all
188 previous requests as complete when new buildset is created.
190 @returns: datetime instance or None, via Deferred
193 bldrid = yield bldr.getBuilderId()
194 completed = yield bldr.master.data.get(
195 ('builders', bldrid, 'buildrequests'),
197 resultspec.Filter('complete', 'eq', [True]),
198 resultspec.Filter('results', 'ne', [results.SKIPPED]),
200 order=['-complete_at'], limit=1)
204 complete_at = completed[0]['complete_at']
206 last_build = yield bldr.master.data.get(
209 resultspec.Filter('builderid', 'eq', [bldrid]),
211 order=['-started_at'], limit=1)
213 if last_build and last_build[0]:
214 last_complete_at = last_build[0]['complete_at']
215 if last_complete_at and (last_complete_at > complete_at):
216 return last_complete_at
220 @defer.inlineCallbacks
221 def prioritizeBuilders(master, builders):
222 """Returns sorted list of builders by their last timestamp of completed and
223 not skipped build, ordered first by branch name.
225 @returns: list of sorted builders
228 bldrNamePrio = { "__Janitor": 0, "00_force_build": 0 }
230 for bname in branchNames:
231 bldrNamePrio[bname] = i
234 def is_building(bldr):
235 return bool(bldr.building) or bool(bldr.old_building)
238 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
239 d.addCallback(lambda complete_at: (complete_at, bldr))
243 (complete_at, bldr) = item
246 for (name, prio) in bldrNamePrio.items():
247 if bldr.name.startswith(name):
253 complete_at = date.replace(tzinfo=tzutc())
255 if is_building(bldr):
257 complete_at = date.replace(tzinfo=tzutc())
259 return (pos, complete_at, bldr.name)
261 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
262 results.sort(key=bldr_sort)
265 # log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
267 return [r[1] for r in results]
269 c['prioritizeBuilders'] = prioritizeBuilders
271 ####### CHANGESOURCES
276 def populateTargets():
277 """ fetch a shallow clone of each configured branch in turn:
278 execute dump-target-info.pl and collate the results to ensure
279 targets that only exist in specific branches get built.
280 This takes a while during master startup but is executed only once.
282 log.msg("Populating targets, this will take time")
283 sourcegit = work_dir + '/source.git'
284 for branch in branchNames:
285 if os.path.isdir(sourcegit):
286 subprocess.call(["rm", "-rf", sourcegit])
288 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
290 os.makedirs(sourcegit + '/tmp', exist_ok=True)
291 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
292 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
295 line = findtargets.stdout.readline()
298 ta = line.decode().strip().split(' ')
301 subprocess.call(["rm", "-rf", sourcegit])
305 # the 'change_source' setting tells the buildmaster how it should find out
306 # about source code changes.
308 c['change_source'] = []
309 c['change_source'].append(GitPoller(
311 workdir=work_dir+'/work.git', branches=branchNames,
312 pollAtLaunch=True, pollinterval=300))
316 # Configure the Schedulers, which decide how to react to incoming changes.
318 # Selector for known valid tags
319 class TagChoiceParameter(BaseParameter):
320 spec_attributes = ["strict", "choices"]
324 def __init__(self, name, label=None, **kw):
325 super().__init__(name, label, **kw)
326 self._choice_list = []
328 def getRevTags(self, findtag=None):
332 # we will filter out tags that do no match the configured branches
333 for b in branchNames:
334 basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
336 branchvers.append(basever[1])
338 # grab tags from remote repository
339 alltags = subprocess.Popen(
340 ['git', 'ls-remote', '--tags', repo_url],
341 stdout = subprocess.PIPE)
344 line = alltags.stdout.readline()
349 (rev, tag) = line.split()
351 # does it match known format? ('vNN.NN.NN(-rcN)')
352 tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
354 # only list valid tags matching configured branches
355 if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
356 # if we want a specific tag, ignore all that don't match
357 if findtag and findtag != tagver[1]:
359 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
365 taglist = [rt['tag'] for rt in self.getRevTags()]
366 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
367 taglist.insert(0, '')
369 self._choice_list = taglist
371 return self._choice_list
373 def updateFromKwargs(self, properties, kwargs, **unused):
374 tag = self.getFromKwargs(kwargs)
375 properties[self.name] = tag
377 # find the commit matching the tag
378 findtag = self.getRevTags(tag)
381 raise ValidationError("Couldn't find tag")
383 properties['force_revision'] = findtag[0]['rev']
385 # find the branch matching the tag
387 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
388 for b in branchNames:
389 if b.endswith(branchver[1]):
393 raise ValidationError("Couldn't find branch")
395 properties['force_branch'] = branch
397 def parse_from_arg(self, s):
398 if self.strict and s not in self._choice_list:
399 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
403 @defer.inlineCallbacks
404 def builderNames(props):
405 """ since we have per branch and per target builders,
406 address the relevant builder for each new buildrequest
407 based on the request's desired branch and target.
409 branch = props.getProperty("branch")
410 target = props.getProperty("target", "")
415 # if that didn't work, try sourcestamp to find a branch
417 # match builders with target branch
418 ss = props.sourcestamps[0]
420 branch = ss['branch']
422 log.msg("couldn't find builder")
423 return [] # nothing works
425 bname = branch + "_" + target
428 for b in (yield props.master.data.get(('builders',))):
429 if not b['name'].startswith(bname):
431 builders.append(b['name'])
436 c['schedulers'].append(AnyBranchScheduler(
438 change_filter = util.ChangeFilter(branch=branchNames),
439 treeStableTimer = 15*60,
440 builderNames = builderNames))
442 c['schedulers'].append(ForceScheduler(
444 buttonName = "Force builds",
445 label = "Force build details",
446 builderNames = [ "00_force_build" ],
449 util.CodebaseParameter(
451 label = "Repository",
452 branch = util.FixedParameter(name = "branch", default = ""),
453 revision = util.FixedParameter(name = "revision", default = ""),
454 repository = util.FixedParameter(name = "repository", default = ""),
455 project = util.FixedParameter(name = "project", default = "")
459 reason = util.StringParameter(
462 default = "Trigger build",
468 # NB: avoid nesting to simplify processing of properties
469 util.ChoiceStringParameter(
471 label = "Build target",
473 choices = [ "all" ] + list(targets)
483 c['schedulers'].append(schedulers.Triggerable(name="trigger", builderNames=builderNames))
487 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
488 # what steps, and which workers can execute them. Note that any particular build will
489 # only take place on one worker.
491 def IsNoMasterBuild(step):
492 return step.getProperty("branch") != "master"
494 def IsUsignEnabled(step):
495 branch = step.getProperty("branch")
496 return branch and branches[branch].get("usign_key")
498 def IsSignEnabled(step):
499 branch = step.getProperty("branch")
500 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
502 def IsKmodArchiveEnabled(step):
503 branch = step.getProperty("branch")
504 return branch and branches[branch].get("kmod_archive")
506 def GetBaseVersion(branch):
507 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
508 return branch.split('-')[1]
513 def GetVersionPrefix(props):
514 branch = props.getProperty("branch")
515 basever = GetBaseVersion(branch)
516 if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
517 return "%s/" % props["tag"][1:]
518 elif basever != "master":
519 return "%s-SNAPSHOT/" % basever
524 def GetConfigSeed(props):
525 branch = props.getProperty("branch")
526 return branch and branches[branch].get("config_seed") or ""
529 def GetRsyncParams(props, srcorbin, urlorkey):
530 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
531 branch = props.getProperty("branch")
532 opt = srcorbin + "_" + urlorkey
533 return branch and branches[branch].get(opt)
536 def GetUsignKey(props):
537 branch = props.getProperty("branch")
538 return branch and branches[branch].get("usign_key")
540 def GetNextBuild(builder, requests):
543 # order tagged build first
544 if r.properties.hasProperty("tag"):
548 #log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
551 def MakeEnv(overrides=None, tryccache=False):
553 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
554 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
557 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
558 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
559 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
561 env['CC'] = env['CCC']
562 env['CXX'] = env['CCXX']
564 if overrides is not None:
565 env.update(overrides)
569 def NetLockDl(props, extralock=None):
571 if props.hasProperty("dl_lock"):
572 lock = NetLocks[props["dl_lock"]]
574 return [lock.access('exclusive')]
579 def NetLockUl(props):
581 if props.hasProperty("ul_lock"):
582 lock = NetLocks[props["ul_lock"]]
584 return [lock.access('exclusive')]
588 def IsTargetSelected(target):
589 def CheckTargetProperty(step):
590 selected_target = step.getProperty("target", "all")
591 if selected_target != "all" and selected_target != target:
595 return CheckTargetProperty
598 def UsignSec2Pub(props):
599 branch = props.getProperty("branch")
601 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
602 seckey = branches[branch].get("usign_key")
603 seckey = base64.b64decode(seckey)
607 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
608 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
611 def canStartBuild(builder, wfb, request):
612 """ filter out non tag requests for tag_only workers. """
613 wtagonly = wfb.worker.properties.getProperty('tag_only')
614 tag = request.properties.getProperty('tag')
616 if wtagonly and not tag:
625 for worker in c['workers']:
626 workerNames.append(worker.workername)
628 # add a single LocalWorker to handle the forcebuild builder
629 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
631 force_factory = BuildFactory()
632 force_factory.addStep(steps.Trigger(
633 name = "trigger_build",
634 schedulerNames = [ "trigger" ],
635 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
636 set_properties = { "reason": Property("reason"), "tag": Property("tag"), "target": Property("target") },
639 c['builders'].append(BuilderConfig(
640 name = "00_force_build",
641 workername = "__local_force_build",
642 factory = force_factory))
645 # NB the phase1 build factory assumes workers are single-build only
646 for target in targets:
647 ts = target.split('/')
649 factory = BuildFactory()
651 # setup shared work directory if required
652 factory.addStep(ShellCommand(
654 descriptionDone = "Shared work directory set up",
655 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
657 haltOnFailure = True,
660 # find number of cores
661 factory.addStep(SetPropertyFromCommand(
664 description = "Finding number of CPUs",
668 # find gcc and g++ compilers
669 factory.addStep(FileDownload(
670 name = "dlfindbinpl",
671 mastersrc = scripts_dir + '/findbin.pl',
672 workerdest = "../findbin.pl",
676 factory.addStep(SetPropertyFromCommand(
678 property = "cc_command",
679 description = "Finding gcc command",
680 command = ["../findbin.pl", "gcc", "", ""],
681 haltOnFailure = True,
684 factory.addStep(SetPropertyFromCommand(
686 property = "cxx_command",
687 description = "Finding g++ command",
688 command = ["../findbin.pl", "g++", "", ""],
689 haltOnFailure = True,
692 # see if ccache is available
693 factory.addStep(SetPropertyFromCommand(
695 property = "ccache_command",
696 description = "Testing for ccache command",
697 command = ["which", "ccache"],
698 haltOnFailure = False,
699 flunkOnFailure = False,
700 warnOnFailure = False,
701 hideStepIf = lambda r, s: r==results.FAILURE,
704 # check out the source
706 # if repo doesn't exist: 'git clone repourl'
707 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
708 # git cat-file -e <commit>
709 # git checkout -f <commit>
710 # git checkout -B <branch>
718 haltOnFailure = True,
722 factory.addStep(ShellCommand(
724 description = "Fetching Git remote refs",
725 descriptionDone = "Git remote refs fetched",
726 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
727 haltOnFailure = True,
730 # Verify that Git HEAD points to a tag or branch
731 # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
732 factory.addStep(ShellCommand(
734 description = "Ensuring that Git HEAD is pointing to a branch or tag",
735 descriptionDone = "Git HEAD is sane",
736 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
737 haltOnFailure = True,
740 factory.addStep(ShellCommand(
742 description = "Remove tmp folder",
743 command=["rm", "-rf", "tmp/"],
747 factory.addStep(ShellCommand(
748 name = "rmfeedlinks",
749 description = "Remove feed symlinks",
750 command=["rm", "-rf", "package/feeds/"],
753 factory.addStep(StringDownload(
755 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
756 workerdest = "../ccache_cc.sh",
760 factory.addStep(StringDownload(
762 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
763 workerdest = "../ccache_cxx.sh",
768 factory.addStep(ShellCommand(
769 name = "updatefeeds",
770 description = "Updating feeds",
771 command=["./scripts/feeds", "update"],
772 env = MakeEnv(tryccache=True),
773 haltOnFailure = True,
778 factory.addStep(ShellCommand(
779 name = "installfeeds",
780 description = "Installing feeds",
781 command=["./scripts/feeds", "install", "-a"],
782 env = MakeEnv(tryccache=True),
783 haltOnFailure = True,
787 factory.addStep(StringDownload(
788 name = "dlconfigseed",
789 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
790 workerdest = ".config",
795 factory.addStep(ShellCommand(
797 descriptionDone = ".config seeded",
798 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
801 factory.addStep(ShellCommand(
803 description = "Removing output directory",
804 command = ["rm", "-rf", "bin/"],
807 factory.addStep(ShellCommand(
809 description = "Populating .config",
810 command = ["make", "defconfig"],
814 # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
815 factory.addStep(ShellCommand(
817 description = "Checking architecture",
818 descriptionDone = "Architecture validated",
819 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
823 haltOnFailure = True,
824 flunkOnFailure = False, # this is not a build FAILURE - TODO mark build as SKIPPED
828 factory.addStep(SetPropertyFromCommand(
831 description = "Finding libc suffix",
832 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
836 factory.addStep(StringDownload(
837 name = "dlkeybuildpub",
838 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
839 workerdest = "key-build.pub",
841 doStepIf = IsUsignEnabled,
844 factory.addStep(StringDownload(
846 s = "# fake private key",
847 workerdest = "key-build",
849 doStepIf = IsUsignEnabled,
852 factory.addStep(StringDownload(
853 name = "dlkeybuilducert",
854 s = "# fake certificate",
855 workerdest = "key-build.ucert",
857 doStepIf = IsUsignEnabled,
861 factory.addStep(ShellCommand(
863 description = "Preparing dl/",
864 descriptionDone = "dl/ prepared",
865 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
866 workdir = Property("builddir"),
872 factory.addStep(ShellCommand(
874 description = "Pruning dl/",
875 descriptionDone = "dl/ pruned",
876 command = 'find dl/ -atime +15 -delete -print',
881 factory.addStep(ShellCommand(
883 description = "Building and installing GNU tar",
884 descriptionDone = "GNU tar built and installed",
885 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
886 env = MakeEnv(tryccache=True),
887 haltOnFailure = True,
891 factory.addStep(ShellCommand(
893 description = "Populating dl/",
894 descriptionDone = "dl/ populated",
895 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
901 factory.addStep(ShellCommand(
903 description = "Cleaning base-files",
904 command=["make", "package/base-files/clean", "V=s"],
908 factory.addStep(ShellCommand(
910 description = "Building and installing tools",
911 descriptionDone = "Tools built and installed",
912 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
913 env = MakeEnv(tryccache=True),
914 haltOnFailure = True,
917 factory.addStep(ShellCommand(
919 description = "Building and installing toolchain",
920 descriptionDone = "Toolchain built and installed",
921 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
923 haltOnFailure = True,
926 factory.addStep(ShellCommand(
928 description = "Building kmods",
929 descriptionDone = "Kmods built",
930 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
932 haltOnFailure = True,
935 # find kernel version
936 factory.addStep(SetPropertyFromCommand(
937 name = "kernelversion",
938 property = "kernelversion",
939 description = "Finding the effective Kernel version",
940 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
941 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
944 factory.addStep(ShellCommand(
946 description = "Cleaning up package build",
947 descriptionDone = "Package build cleaned up",
948 command=["make", "package/cleanup", "V=s"],
951 factory.addStep(ShellCommand(
953 description = "Building packages",
954 descriptionDone = "Packages built",
955 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
957 haltOnFailure = True,
960 factory.addStep(ShellCommand(
962 description = "Installing packages",
963 descriptionDone = "Packages installed",
964 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
966 haltOnFailure = True,
969 factory.addStep(ShellCommand(
971 description = "Indexing packages",
972 descriptionDone = "Packages indexed",
973 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
975 haltOnFailure = True,
978 factory.addStep(ShellCommand(
980 description = "Building and installing images",
981 descriptionDone = "Images built and installed",
982 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
984 haltOnFailure = True,
987 factory.addStep(ShellCommand(
989 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
990 command = "make -j1 buildinfo V=s || true",
992 haltOnFailure = True,
995 factory.addStep(ShellCommand(
996 name = "json_overview_image_info",
997 description = "Generating profiles.json in target folder",
998 command = "make -j1 json_overview_image_info V=s || true",
1000 haltOnFailure = True,
1003 factory.addStep(ShellCommand(
1005 description = "Calculating checksums",
1006 descriptionDone = "Checksums calculated",
1007 command=["make", "-j1", "checksum", "V=s"],
1009 haltOnFailure = True,
1012 factory.addStep(ShellCommand(
1014 descriptionDone = "Kmod directory created",
1015 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
1016 haltOnFailure = True,
1017 doStepIf = IsKmodArchiveEnabled,
1020 factory.addStep(ShellCommand(
1021 name = "kmodprepare",
1022 description = "Preparing kmod archive",
1023 descriptionDone = "Kmod archive prepared",
1024 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
1025 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
1026 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1027 haltOnFailure = True,
1028 doStepIf = IsKmodArchiveEnabled,
1031 factory.addStep(ShellCommand(
1033 description = "Indexing kmod archive",
1034 descriptionDone = "Kmod archive indexed",
1035 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
1036 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1038 haltOnFailure = True,
1039 doStepIf = IsKmodArchiveEnabled,
1043 factory.addStep(MasterShellCommand(
1044 name = "signprepare",
1045 descriptionDone = "Temporary signing directory prepared",
1046 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1047 haltOnFailure = True,
1048 doStepIf = IsSignEnabled,
1052 factory.addStep(ShellCommand(
1054 description = "Packing files to sign",
1055 descriptionDone = "Files to sign packed",
1056 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1057 haltOnFailure = True,
1058 doStepIf = IsSignEnabled,
1061 factory.addStep(FileUpload(
1062 workersrc = "sign.tar.gz",
1063 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1064 haltOnFailure = True,
1065 doStepIf = IsSignEnabled,
1068 factory.addStep(MasterShellCommand(
1070 description = "Signing files",
1071 descriptionDone = "Files signed",
1072 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
1073 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1074 haltOnFailure = True,
1075 doStepIf = IsSignEnabled,
1078 factory.addStep(FileDownload(
1079 name = "dlsigntargz",
1080 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1081 workerdest = "sign.tar.gz",
1082 haltOnFailure = True,
1083 doStepIf = IsSignEnabled,
1086 factory.addStep(ShellCommand(
1087 name = "signunpack",
1088 description = "Unpacking signed files",
1089 descriptionDone = "Signed files unpacked",
1090 command = ["tar", "-xzf", "sign.tar.gz"],
1091 haltOnFailure = True,
1092 doStepIf = IsSignEnabled,
1096 factory.addStep(ShellCommand(
1097 name = "dirprepare",
1098 descriptionDone = "Upload directory structure prepared",
1099 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1100 haltOnFailure = True,
1103 factory.addStep(ShellCommand(
1104 name = "linkprepare",
1105 descriptionDone = "Repository symlink prepared",
1106 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1107 doStepIf = IsNoMasterBuild,
1108 haltOnFailure = True,
1111 factory.addStep(ShellCommand(
1112 name = "kmoddirprepare",
1113 descriptionDone = "Kmod archive upload directory prepared",
1114 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1115 haltOnFailure = True,
1116 doStepIf = IsKmodArchiveEnabled,
1119 factory.addStep(ShellCommand(
1121 description = "Uploading directory structure",
1122 descriptionDone = "Directory structure uploaded",
1123 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1124 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1125 haltOnFailure = True,
1128 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1131 # download remote sha256sums to 'target-sha256sums'
1132 factory.addStep(ShellCommand(
1133 name = "target-sha256sums",
1134 description = "Fetching remote sha256sums for target",
1135 descriptionDone = "Remote sha256sums for target fetched",
1136 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1137 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1139 haltOnFailure = False,
1140 flunkOnFailure = False,
1141 warnOnFailure = False,
1142 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1145 # build list of files to upload
1146 factory.addStep(FileDownload(
1147 name = "dlsha2rsyncpl",
1148 mastersrc = scripts_dir + '/sha2rsync.pl',
1149 workerdest = "../sha2rsync.pl",
1153 factory.addStep(ShellCommand(
1155 description = "Building list of files to upload",
1156 descriptionDone = "List of files to upload built",
1157 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1158 haltOnFailure = True,
1161 factory.addStep(FileDownload(
1162 name = "dlrsync.sh",
1163 mastersrc = scripts_dir + '/rsync.sh',
1164 workerdest = "../rsync.sh",
1168 # upload new files and update existing ones
1169 factory.addStep(ShellCommand(
1170 name = "targetupload",
1171 description = "Uploading target files",
1172 descriptionDone = "Target files uploaded",
1173 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1174 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1175 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1176 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1177 haltOnFailure = True,
1179 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1182 # delete files which don't exist locally
1183 factory.addStep(ShellCommand(
1184 name = "targetprune",
1185 description = "Pruning target files",
1186 descriptionDone = "Target files pruned",
1187 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1188 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1189 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1190 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1191 haltOnFailure = True,
1194 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1197 factory.addStep(ShellCommand(
1198 name = "kmodupload",
1199 description = "Uploading kmod archive",
1200 descriptionDone = "Kmod archive uploaded",
1201 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1202 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1203 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1204 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1205 haltOnFailure = True,
1208 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1211 factory.addStep(ShellCommand(
1212 name = "sourcelist",
1213 description = "Finding source archives to upload",
1214 descriptionDone = "Source archives to upload found",
1215 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1216 haltOnFailure = True,
1219 factory.addStep(ShellCommand(
1220 name = "sourceupload",
1221 description = "Uploading source archives",
1222 descriptionDone = "Source archives uploaded",
1223 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1224 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1225 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1226 haltOnFailure = True,
1229 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1232 factory.addStep(ShellCommand(
1234 description = "Reporting disk usage",
1235 command=["df", "-h", "."],
1236 env={'LC_ALL': 'C'},
1238 haltOnFailure = False,
1239 flunkOnFailure = False,
1240 warnOnFailure = False,
1244 factory.addStep(ShellCommand(
1246 description = "Reporting estimated file space usage",
1247 command=["du", "-sh", "."],
1248 env={'LC_ALL': 'C'},
1250 haltOnFailure = False,
1251 flunkOnFailure = False,
1252 warnOnFailure = False,
1256 factory.addStep(ShellCommand(
1257 name = "ccachestat",
1258 description = "Reporting ccache stats",
1259 command=["ccache", "-s"],
1260 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1262 want_stderr = False,
1263 haltOnFailure = False,
1264 flunkOnFailure = False,
1265 warnOnFailure = False,
1266 hideStepIf = lambda r, s: r==results.FAILURE,
1269 for brname in branchNames:
1270 bldrname = brname + "_" + target
1271 c['builders'].append(BuilderConfig(name=bldrname, workernames=workerNames, factory=factory, tags=[brname,], nextBuild=GetNextBuild, canStartBuild=canStartBuild))
1274 ####### STATUS TARGETS
1276 # 'status' is a list of Status Targets. The results of each build will be
1277 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1278 # including web pages, email senders, and IRC bots.
1280 if "status_bind" in inip1:
1282 'port': inip1.get("status_bind"),
1284 'waterfall_view': True,
1285 'console_view': True,
1290 if "status_user" in inip1 and "status_password" in inip1:
1291 c['www']['auth'] = util.UserPasswordAuth([
1292 (inip1.get("status_user"), inip1.get("status_password"))
1294 c['www']['authz'] = util.Authz(
1295 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1296 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1300 if ini.has_section("irc"):
1302 irc_host = iniirc.get("host", None)
1303 irc_port = iniirc.getint("port", 6667)
1304 irc_chan = iniirc.get("channel", None)
1305 irc_nick = iniirc.get("nickname", None)
1306 irc_pass = iniirc.get("password", None)
1308 if irc_host and irc_nick and irc_chan:
1309 irc = reporters.IRC(irc_host, irc_nick,
1311 password = irc_pass,
1312 channels = [ irc_chan ],
1313 notify_events = [ 'exception', 'problem', 'recovery' ]
1316 c['services'].append(irc)
1318 c['revlink'] = util.RevlinkMatch([
1319 r'https://git.openwrt.org/openwrt/(.*).git'
1321 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1326 # This specifies what database buildbot uses to store its state. You can leave
1327 # this at its default for all but the largest installations.
1328 'db_url' : "sqlite:///state.sqlite",
1331 c['buildbotNetUsageData'] = None