2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes.gitpoller import GitPoller
19 from buildbot.config import BuilderConfig
20 from buildbot.plugins import reporters
21 from buildbot.plugins import schedulers
22 from buildbot.plugins import steps
23 from buildbot.plugins import util
24 from buildbot.process import properties
25 from buildbot.process import results
26 from buildbot.process.factory import BuildFactory
27 from buildbot.process.properties import Interpolate
28 from buildbot.process.properties import Property
29 from buildbot.schedulers.basic import AnyBranchScheduler
30 from buildbot.schedulers.forcesched import BaseParameter
31 from buildbot.schedulers.forcesched import ForceScheduler
32 from buildbot.schedulers.forcesched import ValidationError
33 from buildbot.steps.master import MasterShellCommand
34 from buildbot.steps.shell import SetPropertyFromCommand
35 from buildbot.steps.shell import ShellCommand
36 from buildbot.steps.source.git import Git
37 from buildbot.steps.transfer import FileDownload
38 from buildbot.steps.transfer import FileUpload
39 from buildbot.steps.transfer import StringDownload
40 from buildbot.worker import Worker
41 from buildbot.worker.local import LocalWorker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini['repo'].get("url")
65 rsync_defopts = ["-v", "-4", "--timeout=120"]
67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
68 # rsync_bin_defopts += ["--contimeout=20"]
72 def ini_parse_branch(section):
74 name = section.get("name")
77 raise ValueError("missing 'name' in " + repr(section))
79 raise ValueError("duplicate branch name in " + repr(section))
82 b["bin_url"] = section.get("binary_url")
83 b["bin_key"] = section.get("binary_password")
85 b["src_url"] = section.get("source_url")
86 b["src_key"] = section.get("source_password")
88 b["gpg_key"] = section.get("gpg_key")
90 b["usign_key"] = section.get("usign_key")
91 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
92 b["usign_comment"] = section.get("usign_comment", usign_comment)
94 b["config_seed"] = section.get("config_seed")
96 b["kmod_archive"] = section.getboolean("kmod_archive", False)
99 log.msg("Configured branch: {}".format(name))
101 # PB port can be either a numeric port or a connection string
102 pb_port = inip1.get("port") or 9989
104 # This is the dictionary that the buildmaster pays attention to. We also use
105 # a shorter alias to save typing.
106 c = BuildmasterConfig = {}
108 ####### PROJECT IDENTITY
110 # the 'title' string will appear at the top of this buildbot
111 # installation's html.WebStatus home page (linked to the
112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
114 c['title'] = ini['general'].get("title")
115 c['titleURL'] = ini['general'].get("title_url")
117 # the 'buildbotURL' string should point to the location where the buildbot's
118 # internal web server (usually the html.WebStatus page) is visible. This
119 # typically uses the port number set in the Waterfall 'status' entry, but
120 # with an externally-visible host name which the buildbot cannot figure out
123 c['buildbotURL'] = inip1.get("buildbot_url")
127 # The 'workers' list defines the set of recognized buildworkers. Each element is
128 # a Worker object, specifying a unique worker name and password. The same
129 # worker name and password must be configured on the worker.
134 def ini_parse_workers(section):
135 name = section.get("name")
136 password = section.get("password")
137 phase = section.getint("phase")
138 tagonly = section.getboolean("tag_only")
140 if not name or not password or not phase == 1:
141 log.msg("invalid worker configuration ignored: {}".format(repr(section)))
144 sl_props = { 'dl_lock':None, 'ul_lock':None, 'tag_only':tagonly }
145 if "dl_lock" in section:
146 lockname = section.get("dl_lock")
147 sl_props['dl_lock'] = lockname
148 if lockname not in NetLocks:
149 NetLocks[lockname] = locks.MasterLock(lockname)
150 if "ul_lock" in section:
151 lockname = section.get("ul_lock")
152 sl_props['ul_lock'] = lockname
153 if lockname not in NetLocks:
154 NetLocks[lockname] = locks.MasterLock(lockname)
156 log.msg("Configured worker: {}".format(name))
157 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
160 for section in ini.sections():
161 if section.startswith("branch "):
162 ini_parse_branch(ini[section])
164 if section.startswith("worker "):
165 ini_parse_workers(ini[section])
167 branchNames = [branches[b]["name"] for b in branches]
169 c['protocols'] = {'pb': {'port': pb_port}}
172 c['collapseRequests'] = True
174 # Reduce amount of backlog data
175 c['configurators'] = [util.JanitorConfigurator(
176 logHorizon=timedelta(days=3),
180 @defer.inlineCallbacks
181 def getNewestCompleteTime(bldr):
182 """Returns the complete_at of the latest completed and not SKIPPED
183 build request for this builder, or None if there are no such build
184 requests. We need to filter out SKIPPED requests because we're
185 using collapseRequests=True which is unfortunately marking all
186 previous requests as complete when new buildset is created.
188 @returns: datetime instance or None, via Deferred
191 bldrid = yield bldr.getBuilderId()
192 completed = yield bldr.master.data.get(
193 ('builders', bldrid, 'buildrequests'),
195 resultspec.Filter('complete', 'eq', [True]),
196 resultspec.Filter('results', 'ne', [results.SKIPPED]),
198 order=['-complete_at'], limit=1)
202 complete_at = completed[0]['complete_at']
204 last_build = yield bldr.master.data.get(
207 resultspec.Filter('builderid', 'eq', [bldrid]),
209 order=['-started_at'], limit=1)
211 if last_build and last_build[0]:
212 last_complete_at = last_build[0]['complete_at']
213 if last_complete_at and (last_complete_at > complete_at):
214 return last_complete_at
218 @defer.inlineCallbacks
219 def prioritizeBuilders(master, builders):
220 """Returns sorted list of builders by their last timestamp of completed and
221 not skipped build, ordered first by branch name.
223 @returns: list of sorted builders
226 bldrNamePrio = { "__Janitor": 0, "00_force_build": 0 }
228 for bname in branchNames:
229 bldrNamePrio[bname] = i
232 def is_building(bldr):
233 return bool(bldr.building) or bool(bldr.old_building)
236 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
237 d.addCallback(lambda complete_at: (complete_at, bldr))
241 (complete_at, bldr) = item
244 for (name, prio) in bldrNamePrio.items():
245 if bldr.name.startswith(name):
251 complete_at = date.replace(tzinfo=tzutc())
253 if is_building(bldr):
255 complete_at = date.replace(tzinfo=tzutc())
257 return (pos, complete_at, bldr.name)
259 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
260 results.sort(key=bldr_sort)
263 # log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
265 return [r[1] for r in results]
267 c['prioritizeBuilders'] = prioritizeBuilders
269 ####### CHANGESOURCES
274 def populateTargets():
275 log.msg("Populating targets, this will take time")
276 sourcegit = work_dir + '/source.git'
277 for branch in branchNames:
278 if os.path.isdir(sourcegit):
279 subprocess.call(["rm", "-rf", sourcegit])
281 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
283 os.makedirs(sourcegit + '/tmp', exist_ok=True)
284 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
285 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
288 line = findtargets.stdout.readline()
291 ta = line.decode().strip().split(' ')
294 subprocess.call(["rm", "-rf", sourcegit])
298 # the 'change_source' setting tells the buildmaster how it should find out
299 # about source code changes. Here we point to the buildbot clone of pyflakes.
301 c['change_source'] = []
302 c['change_source'].append(GitPoller(
304 workdir=work_dir+'/work.git', branches=branchNames,
305 pollAtLaunch=True, pollinterval=300))
309 # Configure the Schedulers, which decide how to react to incoming changes. In this
310 # case, just kick off a 'basebuild' build
312 class TagChoiceParameter(BaseParameter):
313 spec_attributes = ["strict", "choices"]
317 def __init__(self, name, label=None, **kw):
318 super().__init__(name, label, **kw)
319 self._choice_list = []
321 def getRevTags(self, findtag=None):
325 for b in branchNames:
326 basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
328 branchvers.append(basever[1])
330 alltags = subprocess.Popen(
331 ['git', 'ls-remote', '--tags', repo_url],
332 stdout = subprocess.PIPE)
335 line = alltags.stdout.readline()
340 (rev, tag) = line.split()
342 tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
344 # only list tags matching configured branches
345 if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
346 if findtag and findtag != tagver[1]:
348 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
354 taglist = [rt['tag'] for rt in self.getRevTags()]
355 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
356 taglist.insert(0, '')
358 self._choice_list = taglist
360 return self._choice_list
362 def updateFromKwargs(self, properties, kwargs, **unused):
363 tag = self.getFromKwargs(kwargs)
364 properties[self.name] = tag
366 # find the commit matching the tag
367 findtag = self.getRevTags(tag)
370 raise ValidationError("Couldn't find tag")
372 properties['force_revision'] = findtag[0]['rev']
374 # find the branch matching the tag
376 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
377 for b in branchNames:
378 if b.endswith(branchver[1]):
382 raise ValidationError("Couldn't find branch")
384 properties['force_branch'] = branch
386 def parse_from_arg(self, s):
387 if self.strict and s not in self._choice_list:
388 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
392 @defer.inlineCallbacks
393 def builderNames(props):
394 """ since we have per branch and per target builders,
395 address the relevant builder for each new buildrequest
396 based on the request's desired branch and target.
398 branch = props.getProperty("branch")
399 target = props.getProperty("target", "")
404 # if that didn't work, try sourcestamp to find a branch
406 # match builders with target branch
407 ss = props.sourcestamps[0]
409 branch = ss['branch']
411 log.msg("couldn't find builder")
412 return [] # nothing works
414 bname = branch + "_" + target
417 for b in (yield props.master.data.get(('builders',))):
418 if not b['name'].startswith(bname):
420 builders.append(b['name'])
425 c['schedulers'].append(AnyBranchScheduler(
427 change_filter = util.ChangeFilter(branch=branchNames),
428 treeStableTimer = 15*60,
429 builderNames = builderNames))
431 c['schedulers'].append(ForceScheduler(
433 buttonName = "Force builds",
434 label = "Force build details",
435 builderNames = [ "00_force_build" ],
438 util.CodebaseParameter(
440 label = "Repository",
441 branch = util.FixedParameter(name = "branch", default = ""),
442 revision = util.FixedParameter(name = "revision", default = ""),
443 repository = util.FixedParameter(name = "repository", default = ""),
444 project = util.FixedParameter(name = "project", default = "")
448 reason = util.StringParameter(
451 default = "Trigger build",
457 util.ChoiceStringParameter(
459 label = "Build target",
461 choices = [ "all" ] + list(targets)
471 c['schedulers'].append(schedulers.Triggerable(name="trigger", builderNames=builderNames))
475 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
476 # what steps, and which workers can execute them. Note that any particular build will
477 # only take place on one worker.
479 def IsNoMasterBuild(step):
480 return step.getProperty("branch") != "master"
482 def IsUsignEnabled(step):
483 branch = step.getProperty("branch")
484 return branch and branches[branch].get("usign_key")
486 def IsSignEnabled(step):
487 branch = step.getProperty("branch")
488 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
490 def IsKmodArchiveEnabled(step):
491 branch = step.getProperty("branch")
492 return branch and branches[branch].get("kmod_archive")
494 def GetBaseVersion(branch):
495 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
496 return branch.split('-')[1]
501 def GetVersionPrefix(props):
502 branch = props.getProperty("branch")
503 basever = GetBaseVersion(branch)
504 if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
505 return "%s/" % props["tag"][1:]
506 elif basever != "master":
507 return "%s-SNAPSHOT/" % basever
512 def GetConfigSeed(props):
513 branch = props.getProperty("branch")
514 return branch and branches[branch].get("config_seed") or ""
517 def GetRsyncParams(props, srcorbin, urlorkey):
518 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
519 branch = props.getProperty("branch")
520 opt = srcorbin + "_" + urlorkey
521 return branch and branches[branch].get(opt)
524 def GetUsignKey(props):
525 branch = props.getProperty("branch")
526 return branch and branches[branch].get("usign_key")
528 def GetNextBuild(builder, requests):
531 # order tagged build first
532 if r.properties.hasProperty("tag"):
536 #log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
539 def MakeEnv(overrides=None, tryccache=False):
541 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
542 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
545 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
546 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
547 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
549 env['CC'] = env['CCC']
550 env['CXX'] = env['CCXX']
552 if overrides is not None:
553 env.update(overrides)
557 def NetLockDl(props, extralock=None):
559 if props.hasProperty("dl_lock"):
560 lock = NetLocks[props["dl_lock"]]
562 return [lock.access('exclusive')]
567 def NetLockUl(props):
569 if props.hasProperty("ul_lock"):
570 lock = NetLocks[props["ul_lock"]]
572 return [lock.access('exclusive')]
576 def IsTargetSelected(target):
577 def CheckTargetProperty(step):
578 selected_target = step.getProperty("target", "all")
579 if selected_target != "all" and selected_target != target:
583 return CheckTargetProperty
586 def UsignSec2Pub(props):
587 branch = props.getProperty("branch")
589 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
590 seckey = branches[branch].get("usign_key")
591 seckey = base64.b64decode(seckey)
595 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
596 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
599 def canStartBuild(builder, wfb, request):
600 """ filter out non tag requests for tag_only workers. """
601 wtagonly = wfb.worker.properties.getProperty('tag_only')
602 tag = request.properties.getProperty('tag')
604 if wtagonly and not tag:
613 for worker in c['workers']:
614 workerNames.append(worker.workername)
616 # add a single LocalWorker to handle the forcebuild builder
617 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
619 force_factory = BuildFactory()
620 force_factory.addStep(steps.Trigger(
621 name = "trigger_build",
622 schedulerNames = [ "trigger" ],
623 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
624 set_properties = { "reason": Property("reason"), "tag": Property("tag"), "target": Property("target") },
627 c['builders'].append(BuilderConfig(
628 name = "00_force_build",
629 workername = "__local_force_build",
630 factory = force_factory))
632 for target in targets:
633 ts = target.split('/')
635 factory = BuildFactory()
637 # setup shared work directory if required
638 factory.addStep(ShellCommand(
640 descriptionDone = "Shared work directory set up",
641 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
643 haltOnFailure = True,
646 # find number of cores
647 factory.addStep(SetPropertyFromCommand(
650 description = "Finding number of CPUs",
654 # find gcc and g++ compilers
655 factory.addStep(FileDownload(
656 name = "dlfindbinpl",
657 mastersrc = scripts_dir + '/findbin.pl',
658 workerdest = "../findbin.pl",
662 factory.addStep(SetPropertyFromCommand(
664 property = "cc_command",
665 description = "Finding gcc command",
666 command = ["../findbin.pl", "gcc", "", ""],
667 haltOnFailure = True,
670 factory.addStep(SetPropertyFromCommand(
672 property = "cxx_command",
673 description = "Finding g++ command",
674 command = ["../findbin.pl", "g++", "", ""],
675 haltOnFailure = True,
678 # see if ccache is available
679 factory.addStep(SetPropertyFromCommand(
681 property = "ccache_command",
682 description = "Testing for ccache command",
683 command = ["which", "ccache"],
684 haltOnFailure = False,
685 flunkOnFailure = False,
686 warnOnFailure = False,
687 hideStepIf = lambda r, s: r==results.FAILURE,
690 # check out the source
692 # if repo doesn't exist: 'git clone repourl'
693 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
694 # git cat-file -e <commit>
695 # git checkout -f <commit>
696 # git checkout -B <branch>
704 haltOnFailure = True,
708 factory.addStep(ShellCommand(
710 description = "Fetching Git remote refs",
711 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
712 haltOnFailure = True,
715 # Verify that Git HEAD points to a tag or branch
716 # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
717 factory.addStep(ShellCommand(
719 description = "Ensure that Git HEAD is pointing to a branch or tag",
720 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
721 haltOnFailure = True,
724 factory.addStep(ShellCommand(
726 description = "Remove tmp folder",
727 command=["rm", "-rf", "tmp/"],
731 factory.addStep(ShellCommand(
732 name = "rmfeedlinks",
733 description = "Remove feed symlinks",
734 command=["rm", "-rf", "package/feeds/"],
737 factory.addStep(StringDownload(
739 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
740 workerdest = "../ccache_cc.sh",
744 factory.addStep(StringDownload(
746 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
747 workerdest = "../ccache_cxx.sh",
752 factory.addStep(ShellCommand(
753 name = "updatefeeds",
754 description = "Updating feeds",
755 command=["./scripts/feeds", "update"],
756 env = MakeEnv(tryccache=True),
757 haltOnFailure = True,
762 factory.addStep(ShellCommand(
763 name = "installfeeds",
764 description = "Installing feeds",
765 command=["./scripts/feeds", "install", "-a"],
766 env = MakeEnv(tryccache=True),
767 haltOnFailure = True,
771 factory.addStep(StringDownload(
772 name = "dlconfigseed",
773 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
774 workerdest = ".config",
779 factory.addStep(ShellCommand(
781 descriptionDone = ".config seeded",
782 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
785 factory.addStep(ShellCommand(
787 description = "Removing output directory",
788 command = ["rm", "-rf", "bin/"],
791 factory.addStep(ShellCommand(
793 description = "Populating .config",
794 command = ["make", "defconfig"],
798 # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
799 factory.addStep(ShellCommand(
801 description = "Checking architecture",
802 descriptionDone = "Architecture validated",
803 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
807 haltOnFailure = True,
808 flunkOnFailure = False, # this is not a build FAILURE
812 factory.addStep(SetPropertyFromCommand(
815 description = "Finding libc suffix",
816 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
820 factory.addStep(StringDownload(
821 name = "dlkeybuildpub",
822 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
823 workerdest = "key-build.pub",
825 doStepIf = IsUsignEnabled,
828 factory.addStep(StringDownload(
830 s = "# fake private key",
831 workerdest = "key-build",
833 doStepIf = IsUsignEnabled,
836 factory.addStep(StringDownload(
837 name = "dlkeybuilducert",
838 s = "# fake certificate",
839 workerdest = "key-build.ucert",
841 doStepIf = IsUsignEnabled,
845 factory.addStep(ShellCommand(
847 description = "Preparing dl/",
848 descriptionDone = "dl/ prepared",
849 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
850 workdir = Property("builddir"),
856 factory.addStep(ShellCommand(
858 description = "Pruning dl/",
859 descriptionDone = "dl/ pruned",
860 command = 'find dl/ -atime +15 -delete -print',
865 factory.addStep(ShellCommand(
867 description = "Building and installing GNU tar",
868 descriptionDone = "GNU tar built and installed",
869 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
870 env = MakeEnv(tryccache=True),
871 haltOnFailure = True,
875 factory.addStep(ShellCommand(
877 description = "Populating dl/",
878 descriptionDone = "dl/ populated",
879 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
885 factory.addStep(ShellCommand(
887 description = "Cleaning base-files",
888 command=["make", "package/base-files/clean", "V=s"],
892 factory.addStep(ShellCommand(
894 description = "Building and installing tools",
895 descriptionDone = "Tools built and installed",
896 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
897 env = MakeEnv(tryccache=True),
898 haltOnFailure = True,
901 factory.addStep(ShellCommand(
903 description = "Building and installing toolchain",
904 descriptionDone = "Toolchain built and installed",
905 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
907 haltOnFailure = True,
910 factory.addStep(ShellCommand(
912 description = "Building kmods",
913 descriptionDone = "Kmods built",
914 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
916 haltOnFailure = True,
919 # find kernel version
920 factory.addStep(SetPropertyFromCommand(
921 name = "kernelversion",
922 property = "kernelversion",
923 description = "Finding the effective Kernel version",
924 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
925 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
928 factory.addStep(ShellCommand(
930 description = "Cleaning up package build",
931 descriptionDone = "Package build cleaned up",
932 command=["make", "package/cleanup", "V=s"],
935 factory.addStep(ShellCommand(
937 description = "Building packages",
938 descriptionDone = "Packages built",
939 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
941 haltOnFailure = True,
944 factory.addStep(ShellCommand(
946 description = "Installing packages",
947 descriptionDone = "Packages installed",
948 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
950 haltOnFailure = True,
953 factory.addStep(ShellCommand(
955 description = "Indexing packages",
956 descriptionDone = "Packages indexed",
957 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
959 haltOnFailure = True,
962 factory.addStep(ShellCommand(
964 description = "Building and installing images",
965 descriptionDone = "Images built and installed",
966 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
968 haltOnFailure = True,
971 factory.addStep(ShellCommand(
973 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
974 command = "make -j1 buildinfo V=s || true",
976 haltOnFailure = True,
979 factory.addStep(ShellCommand(
980 name = "json_overview_image_info",
981 description = "Generating profiles.json in target folder",
982 command = "make -j1 json_overview_image_info V=s || true",
984 haltOnFailure = True,
987 factory.addStep(ShellCommand(
989 description = "Calculating checksums",
990 descriptionDone = "Checksums calculated",
991 command=["make", "-j1", "checksum", "V=s"],
993 haltOnFailure = True,
996 factory.addStep(ShellCommand(
998 descriptionDone = "Kmod directory created",
999 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
1000 haltOnFailure = True,
1001 doStepIf = IsKmodArchiveEnabled,
1004 factory.addStep(ShellCommand(
1005 name = "kmodprepare",
1006 description = "Preparing kmod archive",
1007 descriptionDone = "Kmod archive prepared",
1008 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
1009 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
1010 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1011 haltOnFailure = True,
1012 doStepIf = IsKmodArchiveEnabled,
1015 factory.addStep(ShellCommand(
1017 description = "Indexing kmod archive",
1018 descriptionDone = "Kmod archive indexed",
1019 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
1020 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1022 haltOnFailure = True,
1023 doStepIf = IsKmodArchiveEnabled,
1027 factory.addStep(MasterShellCommand(
1028 name = "signprepare",
1029 descriptionDone = "Temporary signing directory prepared",
1030 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1031 haltOnFailure = True,
1032 doStepIf = IsSignEnabled,
1036 factory.addStep(ShellCommand(
1038 description = "Packing files to sign",
1039 descriptionDone = "Files to sign packed",
1040 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1041 haltOnFailure = True,
1042 doStepIf = IsSignEnabled,
1045 factory.addStep(FileUpload(
1046 workersrc = "sign.tar.gz",
1047 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1048 haltOnFailure = True,
1049 doStepIf = IsSignEnabled,
1052 factory.addStep(MasterShellCommand(
1054 description = "Signing files",
1055 descriptionDone = "Files signed",
1056 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
1057 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1058 haltOnFailure = True,
1059 doStepIf = IsSignEnabled,
1062 factory.addStep(FileDownload(
1063 name = "dlsigntargz",
1064 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1065 workerdest = "sign.tar.gz",
1066 haltOnFailure = True,
1067 doStepIf = IsSignEnabled,
1070 factory.addStep(ShellCommand(
1071 name = "signunpack",
1072 description = "Unpacking signed files",
1073 descriptionDone = "Signed files unpacked",
1074 command = ["tar", "-xzf", "sign.tar.gz"],
1075 haltOnFailure = True,
1076 doStepIf = IsSignEnabled,
1080 factory.addStep(ShellCommand(
1081 name = "dirprepare",
1082 descriptionDone = "Upload directory structure prepared",
1083 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1084 haltOnFailure = True,
1087 factory.addStep(ShellCommand(
1088 name = "linkprepare",
1089 descriptionDone = "Repository symlink prepared",
1090 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1091 doStepIf = IsNoMasterBuild,
1092 haltOnFailure = True,
1095 factory.addStep(ShellCommand(
1096 name = "kmoddirprepare",
1097 descriptionDone = "Kmod archive upload directory prepared",
1098 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1099 haltOnFailure = True,
1100 doStepIf = IsKmodArchiveEnabled,
1103 factory.addStep(ShellCommand(
1105 description = "Uploading directory structure",
1106 descriptionDone = "Directory structure uploaded",
1107 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1108 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1109 haltOnFailure = True,
1112 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1115 # download remote sha256sums to 'target-sha256sums'
1116 factory.addStep(ShellCommand(
1117 name = "target-sha256sums",
1118 description = "Fetching remote sha256sums for target",
1119 descriptionDone = "Remote sha256sums for target fetched",
1120 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1121 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1123 haltOnFailure = False,
1124 flunkOnFailure = False,
1125 warnOnFailure = False,
1126 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1129 # build list of files to upload
1130 factory.addStep(FileDownload(
1131 name = "dlsha2rsyncpl",
1132 mastersrc = scripts_dir + '/sha2rsync.pl',
1133 workerdest = "../sha2rsync.pl",
1137 factory.addStep(ShellCommand(
1139 description = "Building list of files to upload",
1140 descriptionDone = "List of files to upload built",
1141 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1142 haltOnFailure = True,
1145 factory.addStep(FileDownload(
1146 name = "dlrsync.sh",
1147 mastersrc = scripts_dir + '/rsync.sh',
1148 workerdest = "../rsync.sh",
1152 # upload new files and update existing ones
1153 factory.addStep(ShellCommand(
1154 name = "targetupload",
1155 description = "Uploading target files",
1156 descriptionDone = "Target files uploaded",
1157 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1158 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1159 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1160 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1161 haltOnFailure = True,
1163 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1166 # delete files which don't exist locally
1167 factory.addStep(ShellCommand(
1168 name = "targetprune",
1169 description = "Pruning target files",
1170 descriptionDone = "Target files pruned",
1171 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1172 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1173 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1174 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1175 haltOnFailure = True,
1178 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1181 factory.addStep(ShellCommand(
1182 name = "kmodupload",
1183 description = "Uploading kmod archive",
1184 descriptionDone = "Kmod archive uploaded",
1185 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1186 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1187 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1188 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1189 haltOnFailure = True,
1192 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1195 factory.addStep(ShellCommand(
1196 name = "sourcelist",
1197 description = "Finding source archives to upload",
1198 descriptionDone = "Source archives to upload found",
1199 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1200 haltOnFailure = True,
1203 factory.addStep(ShellCommand(
1204 name = "sourceupload",
1205 description = "Uploading source archives",
1206 descriptionDone = "Source archives uploaded",
1207 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1208 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1209 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1210 haltOnFailure = True,
1213 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1216 factory.addStep(ShellCommand(
1218 description = "Reporting disk usage",
1219 command=["df", "-h", "."],
1220 env={'LC_ALL': 'C'},
1222 haltOnFailure = False,
1223 flunkOnFailure = False,
1224 warnOnFailure = False,
1228 factory.addStep(ShellCommand(
1230 description = "Reporting estimated file space usage",
1231 command=["du", "-sh", "."],
1232 env={'LC_ALL': 'C'},
1234 haltOnFailure = False,
1235 flunkOnFailure = False,
1236 warnOnFailure = False,
1240 factory.addStep(ShellCommand(
1241 name = "ccachestat",
1242 description = "Reporting ccache stats",
1243 command=["ccache", "-s"],
1244 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1246 want_stderr = False,
1247 haltOnFailure = False,
1248 flunkOnFailure = False,
1249 warnOnFailure = False,
1250 hideStepIf = lambda r, s: r==results.FAILURE,
1253 for brname in branchNames:
1254 bldrname = brname + "_" + target
1255 c['builders'].append(BuilderConfig(name=bldrname, workernames=workerNames, factory=factory, nextBuild=GetNextBuild, canStartBuild=canStartBuild))
1258 ####### STATUS TARGETS
1260 # 'status' is a list of Status Targets. The results of each build will be
1261 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1262 # including web pages, email senders, and IRC bots.
1264 if "status_bind" in inip1:
1266 'port': inip1.get("status_bind"),
1268 'waterfall_view': True,
1269 'console_view': True,
1274 if "status_user" in inip1 and "status_password" in inip1:
1275 c['www']['auth'] = util.UserPasswordAuth([
1276 (inip1.get("status_user"), inip1.get("status_password"))
1278 c['www']['authz'] = util.Authz(
1279 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1280 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1284 if ini.has_section("irc"):
1286 irc_host = iniirc.get("host", None)
1287 irc_port = iniirc.getint("port", 6667)
1288 irc_chan = iniirc.get("channel", None)
1289 irc_nick = iniirc.get("nickname", None)
1290 irc_pass = iniirc.get("password", None)
1292 if irc_host and irc_nick and irc_chan:
1293 irc = reporters.IRC(irc_host, irc_nick,
1295 password = irc_pass,
1296 channels = [ irc_chan ],
1297 notify_events = [ 'exception', 'problem', 'recovery' ]
1300 c['services'].append(irc)
1302 c['revlink'] = util.RevlinkMatch([
1303 r'https://git.openwrt.org/openwrt/(.*).git'
1305 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1310 # This specifies what database buildbot uses to store its state. You can leave
1311 # this at its default for all but the largest installations.
1312 'db_url' : "sqlite:///state.sqlite",
1315 c['buildbotNetUsageData'] = None