2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes.gitpoller import GitPoller
19 from buildbot.config import BuilderConfig
20 from buildbot.plugins import reporters
21 from buildbot.plugins import schedulers
22 from buildbot.plugins import steps
23 from buildbot.plugins import util
24 from buildbot.process import properties
25 from buildbot.process import results
26 from buildbot.process.factory import BuildFactory
27 from buildbot.process.properties import Interpolate
28 from buildbot.process.properties import Property
29 from buildbot.schedulers.basic import AnyBranchScheduler
30 from buildbot.schedulers.forcesched import BaseParameter
31 from buildbot.schedulers.forcesched import ForceScheduler
32 from buildbot.schedulers.forcesched import ValidationError
33 from buildbot.steps.master import MasterShellCommand
34 from buildbot.steps.shell import SetPropertyFromCommand
35 from buildbot.steps.shell import ShellCommand
36 from buildbot.steps.source.git import Git
37 from buildbot.steps.transfer import FileDownload
38 from buildbot.steps.transfer import FileUpload
39 from buildbot.steps.transfer import StringDownload
40 from buildbot.worker import Worker
41 from buildbot.worker.local import LocalWorker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini['repo'].get("url")
65 rsync_defopts = ["-v", "-4", "--timeout=120"]
67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
68 # rsync_bin_defopts += ["--contimeout=20"]
72 def ini_parse_branch(section):
74 name = section.get("name")
77 raise ValueError("missing 'name' in " + repr(section))
79 raise ValueError("duplicate branch name in " + repr(section))
82 b["bin_url"] = section.get("binary_url")
83 b["bin_key"] = section.get("binary_password")
85 b["src_url"] = section.get("source_url")
86 b["src_key"] = section.get("source_password")
88 b["gpg_key"] = section.get("gpg_key")
90 b["usign_key"] = section.get("usign_key")
91 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
92 b["usign_comment"] = section.get("usign_comment", usign_comment)
94 b["config_seed"] = section.get("config_seed")
96 b["kmod_archive"] = section.getboolean("kmod_archive", False)
99 log.msg("Configured branch: {}".format(name))
101 # PB port can be either a numeric port or a connection string
102 pb_port = inip1.get("port") or 9989
104 # This is the dictionary that the buildmaster pays attention to. We also use
105 # a shorter alias to save typing.
106 c = BuildmasterConfig = {}
108 ####### PROJECT IDENTITY
110 # the 'title' string will appear at the top of this buildbot
111 # installation's html.WebStatus home page (linked to the
112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
114 c['title'] = ini['general'].get("title")
115 c['titleURL'] = ini['general'].get("title_url")
117 # the 'buildbotURL' string should point to the location where the buildbot's
118 # internal web server (usually the html.WebStatus page) is visible. This
119 # typically uses the port number set in the Waterfall 'status' entry, but
120 # with an externally-visible host name which the buildbot cannot figure out
123 c['buildbotURL'] = inip1.get("buildbot_url")
127 # The 'workers' list defines the set of recognized buildworkers. Each element is
128 # a Worker object, specifying a unique worker name and password. The same
129 # worker name and password must be configured on the worker.
134 for section in ini.sections():
135 if section.startswith("branch "):
136 ini_parse_branch(ini[section])
138 if section.startswith("worker "):
139 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
140 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
141 sl_props = { 'dl_lock':None, 'ul_lock':None }
142 name = ini.get(section, "name")
143 password = ini.get(section, "password")
144 if ini.has_option(section, "dl_lock"):
145 lockname = ini.get(section, "dl_lock")
146 sl_props['dl_lock'] = lockname
147 if lockname not in NetLocks:
148 NetLocks[lockname] = locks.MasterLock(lockname)
149 if ini.has_option(section, "ul_lock"):
150 lockname = ini.get(section, "ul_lock")
151 sl_props['ul_lock'] = lockname
152 if lockname not in NetLocks:
153 NetLocks[lockname] = locks.MasterLock(lockname)
154 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
156 c['protocols'] = {'pb': {'port': pb_port}}
159 c['collapseRequests'] = True
161 # Reduce amount of backlog data
162 c['configurators'] = [util.JanitorConfigurator(
163 logHorizon=timedelta(days=3),
167 @defer.inlineCallbacks
168 def getNewestCompleteTime(bldr):
169 """Returns the complete_at of the latest completed and not SKIPPED
170 build request for this builder, or None if there are no such build
171 requests. We need to filter out SKIPPED requests because we're
172 using collapseRequests=True which is unfortunately marking all
173 previous requests as complete when new buildset is created.
175 @returns: datetime instance or None, via Deferred
178 bldrid = yield bldr.getBuilderId()
179 completed = yield bldr.master.data.get(
180 ('builders', bldrid, 'buildrequests'),
182 resultspec.Filter('complete', 'eq', [True]),
183 resultspec.Filter('results', 'ne', [results.SKIPPED]),
185 order=['-complete_at'], limit=1)
189 complete_at = completed[0]['complete_at']
191 last_build = yield bldr.master.data.get(
194 resultspec.Filter('builderid', 'eq', [bldrid]),
196 order=['-started_at'], limit=1)
198 if last_build and last_build[0]:
199 last_complete_at = last_build[0]['complete_at']
200 if last_complete_at and (last_complete_at > complete_at):
201 return last_complete_at
205 @defer.inlineCallbacks
206 def prioritizeBuilders(master, builders):
207 """Returns sorted list of builders by their last timestamp of completed and
210 @returns: list of sorted builders
213 def is_building(bldr):
214 return bool(bldr.building) or bool(bldr.old_building)
217 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
218 d.addCallback(lambda complete_at: (complete_at, bldr))
222 (complete_at, bldr) = item
224 if bldr.name == "00_force_build":
226 complete_at = date.replace(tzinfo=tzutc())
227 return (complete_at, bldr.name)
231 complete_at = date.replace(tzinfo=tzutc())
233 if is_building(bldr):
235 complete_at = date.replace(tzinfo=tzutc())
237 return (complete_at, bldr.name)
239 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
240 results.sort(key=bldr_sort)
243 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
245 return [r[1] for r in results]
247 c['prioritizeBuilders'] = prioritizeBuilders
249 ####### CHANGESOURCES
251 branchNames = [branches[b]["name"] for b in branches]
256 def populateTargets():
257 log.msg("Populating targets, this will take time")
258 sourcegit = work_dir + '/source.git'
259 for branch in branchNames:
260 if os.path.isdir(sourcegit):
261 subprocess.call(["rm", "-rf", sourcegit])
263 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
265 os.makedirs(sourcegit + '/tmp', exist_ok=True)
266 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
267 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
270 line = findtargets.stdout.readline()
273 ta = line.decode().strip().split(' ')
276 subprocess.call(["rm", "-rf", sourcegit])
280 # the 'change_source' setting tells the buildmaster how it should find out
281 # about source code changes. Here we point to the buildbot clone of pyflakes.
283 c['change_source'] = []
284 c['change_source'].append(GitPoller(
286 workdir=work_dir+'/work.git', branches=branchNames,
287 pollAtLaunch=True, pollinterval=300))
291 # Configure the Schedulers, which decide how to react to incoming changes. In this
292 # case, just kick off a 'basebuild' build
294 class TagChoiceParameter(BaseParameter):
295 spec_attributes = ["strict", "choices"]
299 def __init__(self, name, label=None, **kw):
300 super().__init__(name, label, **kw)
301 self._choice_list = []
303 def getRevTags(self, findtag=None):
307 for b in branchNames:
308 basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
310 branchvers.append(basever[1])
312 alltags = subprocess.Popen(
313 ['git', 'ls-remote', '--tags', repo_url],
314 stdout = subprocess.PIPE)
317 line = alltags.stdout.readline()
322 (rev, tag) = line.split()
324 tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
326 # only list tags matching configured branches
327 if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
328 if findtag and findtag != tagver[1]:
330 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
336 taglist = [rt['tag'] for rt in self.getRevTags()]
337 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
338 taglist.insert(0, '')
340 self._choice_list = taglist
342 return self._choice_list
344 def updateFromKwargs(self, properties, kwargs, **unused):
345 tag = self.getFromKwargs(kwargs)
346 properties[self.name] = tag
348 # find the commit matching the tag
349 findtag = self.getRevTags(tag)
352 raise ValidationError("Couldn't find tag")
354 properties['force_revision'] = findtag[0]['rev']
356 # find the branch matching the tag
358 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
359 for b in branchNames:
360 if b.endswith(branchver[1]):
364 raise ValidationError("Couldn't find branch")
366 properties['force_branch'] = branch
368 def parse_from_arg(self, s):
369 if self.strict and s not in self._choice_list:
370 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
374 c['schedulers'].append(AnyBranchScheduler(
376 change_filter = util.ChangeFilter(branch=branchNames),
377 treeStableTimer = 15*60,
378 builderNames = list(targets)))
380 c['schedulers'].append(ForceScheduler(
382 buttonName = "Force builds",
383 label = "Force build details",
384 builderNames = [ "00_force_build" ],
387 util.CodebaseParameter(
389 label = "Repository",
390 branch = util.FixedParameter(name = "branch", default = ""),
391 revision = util.FixedParameter(name = "revision", default = ""),
392 repository = util.FixedParameter(name = "repository", default = ""),
393 project = util.FixedParameter(name = "project", default = "")
397 reason = util.StringParameter(
400 default = "Trigger build",
406 util.ChoiceStringParameter(
408 label = "Build target",
410 choices = [ "all" ] + list(targets)
422 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
423 # what steps, and which workers can execute them. Note that any particular build will
424 # only take place on one worker.
426 def IsNoMasterBuild(step):
427 return step.getProperty("branch") != "master"
429 def IsUsignEnabled(step):
430 branch = step.getProperty("branch")
431 return branch and branches[branch].get("usign_key")
433 def IsSignEnabled(step):
434 branch = step.getProperty("branch")
435 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
437 def IsKmodArchiveEnabled(step):
438 branch = step.getProperty("branch")
439 return branch and branches[branch].get("kmod_archive")
441 def GetBaseVersion(branch):
442 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
443 return branch.split('-')[1]
448 def GetVersionPrefix(props):
449 branch = props.getProperty("branch")
450 basever = GetBaseVersion(branch)
451 if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
452 return "%s/" % props["tag"][1:]
453 elif basever != "master":
454 return "%s-SNAPSHOT/" % basever
459 def GetConfigSeed(props):
460 branch = props.getProperty("branch")
461 return branch and branches[branch].get("config_seed") or ""
464 def GetRsyncParams(props, srcorbin, urlorkey):
465 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
466 branch = props.getProperty("branch")
467 opt = srcorbin + "_" + urlorkey
468 return branch and branches[branch].get(opt)
471 def GetUsignKey(props):
472 branch = props.getProperty("branch")
473 return branch and branches[branch].get("usign_key")
475 def GetNextBuild(builder, requests):
478 # order tagged build first
479 if r.properties.hasProperty("tag"):
481 # then order by branch order
482 pbranch = r.properties.getProperty("branch")
483 for name in branchNames:
488 log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
491 def MakeEnv(overrides=None, tryccache=False):
493 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
494 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
497 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
498 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
499 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
501 env['CC'] = env['CCC']
502 env['CXX'] = env['CCXX']
504 if overrides is not None:
505 env.update(overrides)
509 def NetLockDl(props, extralock=None):
511 if props.hasProperty("dl_lock"):
512 lock = NetLocks[props["dl_lock"]]
514 return [lock.access('exclusive')]
519 def NetLockUl(props):
521 if props.hasProperty("ul_lock"):
522 lock = NetLocks[props["ul_lock"]]
524 return [lock.access('exclusive')]
528 def IsTargetSelected(target):
529 def CheckTargetProperty(step):
530 selected_target = step.getProperty("target", "all")
531 if selected_target != "all" and selected_target != target:
535 return CheckTargetProperty
538 def UsignSec2Pub(props):
539 branch = props.getProperty("branch")
541 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
542 seckey = branches[branch].get("usign_key")
543 seckey = base64.b64decode(seckey)
547 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
548 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
555 for worker in c['workers']:
556 workerNames.append(worker.workername)
558 # add a single LocalWorker to handle the forcebuild builder
559 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
561 force_factory = BuildFactory()
563 c['builders'].append(BuilderConfig(
564 name = "00_force_build",
565 workername = "__local_force_build",
566 factory = force_factory))
568 for target in targets:
569 ts = target.split('/')
571 factory = BuildFactory()
573 # setup shared work directory if required
574 factory.addStep(ShellCommand(
576 descriptionDone = "Shared work directory set up",
577 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
579 haltOnFailure = True,
582 # find number of cores
583 factory.addStep(SetPropertyFromCommand(
586 description = "Finding number of CPUs",
590 # find gcc and g++ compilers
591 factory.addStep(FileDownload(
592 name = "dlfindbinpl",
593 mastersrc = scripts_dir + '/findbin.pl',
594 workerdest = "../findbin.pl",
598 factory.addStep(SetPropertyFromCommand(
600 property = "cc_command",
601 description = "Finding gcc command",
602 command = ["../findbin.pl", "gcc", "", ""],
603 haltOnFailure = True,
606 factory.addStep(SetPropertyFromCommand(
608 property = "cxx_command",
609 description = "Finding g++ command",
610 command = ["../findbin.pl", "g++", "", ""],
611 haltOnFailure = True,
614 # see if ccache is available
615 factory.addStep(SetPropertyFromCommand(
617 property = "ccache_command",
618 description = "Testing for ccache command",
619 command = ["which", "ccache"],
620 haltOnFailure = False,
621 flunkOnFailure = False,
622 warnOnFailure = False,
623 hideStepIf = lambda r, s: r==results.FAILURE,
626 # check out the source
628 # if repo doesn't exist: 'git clone repourl'
629 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
630 # git cat-file -e <commit>
631 # git checkout -f <commit>
632 # git checkout -B <branch>
640 haltOnFailure = True,
644 factory.addStep(ShellCommand(
646 description = "Fetching Git remote refs",
647 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
648 haltOnFailure = True,
651 # Verify that Git HEAD points to a tag or branch
652 # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
653 factory.addStep(ShellCommand(
655 description = "Ensure that Git HEAD is pointing to a branch or tag",
656 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
657 haltOnFailure = True,
660 factory.addStep(ShellCommand(
662 description = "Remove tmp folder",
663 command=["rm", "-rf", "tmp/"],
667 factory.addStep(ShellCommand(
668 name = "rmfeedlinks",
669 description = "Remove feed symlinks",
670 command=["rm", "-rf", "package/feeds/"],
673 factory.addStep(StringDownload(
675 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
676 workerdest = "../ccache_cc.sh",
680 factory.addStep(StringDownload(
682 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
683 workerdest = "../ccache_cxx.sh",
688 factory.addStep(ShellCommand(
689 name = "updatefeeds",
690 description = "Updating feeds",
691 command=["./scripts/feeds", "update"],
692 env = MakeEnv(tryccache=True),
693 haltOnFailure = True,
698 factory.addStep(ShellCommand(
699 name = "installfeeds",
700 description = "Installing feeds",
701 command=["./scripts/feeds", "install", "-a"],
702 env = MakeEnv(tryccache=True),
703 haltOnFailure = True,
707 factory.addStep(StringDownload(
708 name = "dlconfigseed",
709 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
710 workerdest = ".config",
715 factory.addStep(ShellCommand(
717 descriptionDone = ".config seeded",
718 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
721 factory.addStep(ShellCommand(
723 description = "Removing output directory",
724 command = ["rm", "-rf", "bin/"],
727 factory.addStep(ShellCommand(
729 description = "Populating .config",
730 command = ["make", "defconfig"],
734 # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
735 factory.addStep(ShellCommand(
737 description = "Checking architecture",
738 descriptionDone = "Architecture validated",
739 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
743 haltOnFailure = True,
744 flunkOnFailure = False, # this is not a build FAILURE
748 factory.addStep(SetPropertyFromCommand(
751 description = "Finding libc suffix",
752 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
756 factory.addStep(StringDownload(
757 name = "dlkeybuildpub",
758 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
759 workerdest = "key-build.pub",
761 doStepIf = IsUsignEnabled,
764 factory.addStep(StringDownload(
766 s = "# fake private key",
767 workerdest = "key-build",
769 doStepIf = IsUsignEnabled,
772 factory.addStep(StringDownload(
773 name = "dlkeybuilducert",
774 s = "# fake certificate",
775 workerdest = "key-build.ucert",
777 doStepIf = IsUsignEnabled,
781 factory.addStep(ShellCommand(
783 description = "Preparing dl/",
784 descriptionDone = "dl/ prepared",
785 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
786 workdir = Property("builddir"),
792 factory.addStep(ShellCommand(
794 description = "Pruning dl/",
795 descriptionDone = "dl/ pruned",
796 command = 'find dl/ -atime +15 -delete -print',
801 factory.addStep(ShellCommand(
803 description = "Building and installing GNU tar",
804 descriptionDone = "GNU tar built and installed",
805 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
806 env = MakeEnv(tryccache=True),
807 haltOnFailure = True,
811 factory.addStep(ShellCommand(
813 description = "Populating dl/",
814 descriptionDone = "dl/ populated",
815 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
821 factory.addStep(ShellCommand(
823 description = "Cleaning base-files",
824 command=["make", "package/base-files/clean", "V=s"],
828 factory.addStep(ShellCommand(
830 description = "Building and installing tools",
831 descriptionDone = "Tools built and installed",
832 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
833 env = MakeEnv(tryccache=True),
834 haltOnFailure = True,
837 factory.addStep(ShellCommand(
839 description = "Building and installing toolchain",
840 descriptionDone = "Toolchain built and installed",
841 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
843 haltOnFailure = True,
846 factory.addStep(ShellCommand(
848 description = "Building kmods",
849 descriptionDone = "Kmods built",
850 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
852 haltOnFailure = True,
855 # find kernel version
856 factory.addStep(SetPropertyFromCommand(
857 name = "kernelversion",
858 property = "kernelversion",
859 description = "Finding the effective Kernel version",
860 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
861 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
864 factory.addStep(ShellCommand(
866 description = "Cleaning up package build",
867 descriptionDone = "Package build cleaned up",
868 command=["make", "package/cleanup", "V=s"],
871 factory.addStep(ShellCommand(
873 description = "Building packages",
874 descriptionDone = "Packages built",
875 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
877 haltOnFailure = True,
880 factory.addStep(ShellCommand(
882 description = "Installing packages",
883 descriptionDone = "Packages installed",
884 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
886 haltOnFailure = True,
889 factory.addStep(ShellCommand(
891 description = "Indexing packages",
892 descriptionDone = "Packages indexed",
893 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
895 haltOnFailure = True,
898 factory.addStep(ShellCommand(
900 description = "Building and installing images",
901 descriptionDone = "Images built and installed",
902 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
904 haltOnFailure = True,
907 factory.addStep(ShellCommand(
909 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
910 command = "make -j1 buildinfo V=s || true",
912 haltOnFailure = True,
915 factory.addStep(ShellCommand(
916 name = "json_overview_image_info",
917 description = "Generating profiles.json in target folder",
918 command = "make -j1 json_overview_image_info V=s || true",
920 haltOnFailure = True,
923 factory.addStep(ShellCommand(
925 description = "Calculating checksums",
926 descriptionDone = "Checksums calculated",
927 command=["make", "-j1", "checksum", "V=s"],
929 haltOnFailure = True,
932 factory.addStep(ShellCommand(
934 descriptionDone = "Kmod directory created",
935 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
936 haltOnFailure = True,
937 doStepIf = IsKmodArchiveEnabled,
940 factory.addStep(ShellCommand(
941 name = "kmodprepare",
942 description = "Preparing kmod archive",
943 descriptionDone = "Kmod archive prepared",
944 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
945 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
946 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
947 haltOnFailure = True,
948 doStepIf = IsKmodArchiveEnabled,
951 factory.addStep(ShellCommand(
953 description = "Indexing kmod archive",
954 descriptionDone = "Kmod archive indexed",
955 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
956 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
958 haltOnFailure = True,
959 doStepIf = IsKmodArchiveEnabled,
963 factory.addStep(MasterShellCommand(
964 name = "signprepare",
965 descriptionDone = "Temporary signing directory prepared",
966 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
967 haltOnFailure = True,
968 doStepIf = IsSignEnabled,
972 factory.addStep(ShellCommand(
974 description = "Packing files to sign",
975 descriptionDone = "Files to sign packed",
976 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
977 haltOnFailure = True,
978 doStepIf = IsSignEnabled,
981 factory.addStep(FileUpload(
982 workersrc = "sign.tar.gz",
983 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
984 haltOnFailure = True,
985 doStepIf = IsSignEnabled,
988 factory.addStep(MasterShellCommand(
990 description = "Signing files",
991 descriptionDone = "Files signed",
992 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
993 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
994 haltOnFailure = True,
995 doStepIf = IsSignEnabled,
998 factory.addStep(FileDownload(
999 name = "dlsigntargz",
1000 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1001 workerdest = "sign.tar.gz",
1002 haltOnFailure = True,
1003 doStepIf = IsSignEnabled,
1006 factory.addStep(ShellCommand(
1007 name = "signunpack",
1008 description = "Unpacking signed files",
1009 descriptionDone = "Signed files unpacked",
1010 command = ["tar", "-xzf", "sign.tar.gz"],
1011 haltOnFailure = True,
1012 doStepIf = IsSignEnabled,
1016 factory.addStep(ShellCommand(
1017 name = "dirprepare",
1018 descriptionDone = "Upload directory structure prepared",
1019 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1020 haltOnFailure = True,
1023 factory.addStep(ShellCommand(
1024 name = "linkprepare",
1025 descriptionDone = "Repository symlink prepared",
1026 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1027 doStepIf = IsNoMasterBuild,
1028 haltOnFailure = True,
1031 factory.addStep(ShellCommand(
1032 name = "kmoddirprepare",
1033 descriptionDone = "Kmod archive upload directory prepared",
1034 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1035 haltOnFailure = True,
1036 doStepIf = IsKmodArchiveEnabled,
1039 factory.addStep(ShellCommand(
1041 description = "Uploading directory structure",
1042 descriptionDone = "Directory structure uploaded",
1043 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1044 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1045 haltOnFailure = True,
1048 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1051 # download remote sha256sums to 'target-sha256sums'
1052 factory.addStep(ShellCommand(
1053 name = "target-sha256sums",
1054 description = "Fetching remote sha256sums for target",
1055 descriptionDone = "Remote sha256sums for target fetched",
1056 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1057 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1059 haltOnFailure = False,
1060 flunkOnFailure = False,
1061 warnOnFailure = False,
1062 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1065 # build list of files to upload
1066 factory.addStep(FileDownload(
1067 name = "dlsha2rsyncpl",
1068 mastersrc = scripts_dir + '/sha2rsync.pl',
1069 workerdest = "../sha2rsync.pl",
1073 factory.addStep(ShellCommand(
1075 description = "Building list of files to upload",
1076 descriptionDone = "List of files to upload built",
1077 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1078 haltOnFailure = True,
1081 factory.addStep(FileDownload(
1082 name = "dlrsync.sh",
1083 mastersrc = scripts_dir + '/rsync.sh',
1084 workerdest = "../rsync.sh",
1088 # upload new files and update existing ones
1089 factory.addStep(ShellCommand(
1090 name = "targetupload",
1091 description = "Uploading target files",
1092 descriptionDone = "Target files uploaded",
1093 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1094 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1095 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1096 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1097 haltOnFailure = True,
1099 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1102 # delete files which don't exist locally
1103 factory.addStep(ShellCommand(
1104 name = "targetprune",
1105 description = "Pruning target files",
1106 descriptionDone = "Target files pruned",
1107 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1108 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1109 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1110 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1111 haltOnFailure = True,
1114 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1117 factory.addStep(ShellCommand(
1118 name = "kmodupload",
1119 description = "Uploading kmod archive",
1120 descriptionDone = "Kmod archive uploaded",
1121 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1122 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1123 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1124 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1125 haltOnFailure = True,
1128 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1131 factory.addStep(ShellCommand(
1132 name = "sourcelist",
1133 description = "Finding source archives to upload",
1134 descriptionDone = "Source archives to upload found",
1135 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1136 haltOnFailure = True,
1139 factory.addStep(ShellCommand(
1140 name = "sourceupload",
1141 description = "Uploading source archives",
1142 descriptionDone = "Source archives uploaded",
1143 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1144 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1145 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1146 haltOnFailure = True,
1149 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1152 factory.addStep(ShellCommand(
1154 description = "Reporting disk usage",
1155 command=["df", "-h", "."],
1156 env={'LC_ALL': 'C'},
1158 haltOnFailure = False,
1159 flunkOnFailure = False,
1160 warnOnFailure = False,
1164 factory.addStep(ShellCommand(
1166 description = "Reporting estimated file space usage",
1167 command=["du", "-sh", "."],
1168 env={'LC_ALL': 'C'},
1170 haltOnFailure = False,
1171 flunkOnFailure = False,
1172 warnOnFailure = False,
1176 factory.addStep(ShellCommand(
1177 name = "ccachestat",
1178 description = "Reporting ccache stats",
1179 command=["ccache", "-s"],
1180 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1182 want_stderr = False,
1183 haltOnFailure = False,
1184 flunkOnFailure = False,
1185 warnOnFailure = False,
1186 hideStepIf = lambda r, s: r==results.FAILURE,
1189 c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1191 c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1192 force_factory.addStep(steps.Trigger(
1193 name = "trigger_%s" % target,
1194 description = "Triggering %s build" % target,
1195 schedulerNames = [ "trigger_%s" % target ],
1196 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
1197 set_properties = { "reason": Property("reason"), "tag": Property("tag"), },
1198 doStepIf = IsTargetSelected(target),
1202 ####### STATUS TARGETS
1204 # 'status' is a list of Status Targets. The results of each build will be
1205 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1206 # including web pages, email senders, and IRC bots.
1208 if "status_bind" in inip1:
1210 'port': inip1.get("status_bind"),
1212 'waterfall_view': True,
1213 'console_view': True,
1218 if "status_user" in inip1 and "status_password" in inip1:
1219 c['www']['auth'] = util.UserPasswordAuth([
1220 (inip1.get("status_user"), inip1.get("status_password"))
1222 c['www']['authz'] = util.Authz(
1223 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1224 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1228 if ini.has_section("irc"):
1230 irc_host = iniirc.get("host", None)
1231 irc_port = iniirc.getint("port", 6667)
1232 irc_chan = iniirc.get("channel", None)
1233 irc_nick = iniirc.get("nickname", None)
1234 irc_pass = iniirc.get("password", None)
1236 if irc_host and irc_nick and irc_chan:
1237 irc = reporters.IRC(irc_host, irc_nick,
1239 password = irc_pass,
1240 channels = [ irc_chan ],
1241 notify_events = [ 'exception', 'problem', 'recovery' ]
1244 c['services'].append(irc)
1246 c['revlink'] = util.RevlinkMatch([
1247 r'https://git.openwrt.org/openwrt/(.*).git'
1249 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1254 # This specifies what database buildbot uses to store its state. You can leave
1255 # this at its default for all but the largest installations.
1256 'db_url' : "sqlite:///state.sqlite",
1259 c['buildbotNetUsageData'] = None