2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes.gitpoller import GitPoller
19 from buildbot.config import BuilderConfig
20 from buildbot.plugins import reporters
21 from buildbot.plugins import schedulers
22 from buildbot.plugins import steps
23 from buildbot.plugins import util
24 from buildbot.process import properties
25 from buildbot.process import results
26 from buildbot.process.factory import BuildFactory
27 from buildbot.process.properties import Interpolate
28 from buildbot.process.properties import Property
29 from buildbot.schedulers.basic import AnyBranchScheduler
30 from buildbot.schedulers.forcesched import BaseParameter
31 from buildbot.schedulers.forcesched import ForceScheduler
32 from buildbot.schedulers.forcesched import ValidationError
33 from buildbot.steps.master import MasterShellCommand
34 from buildbot.steps.shell import SetPropertyFromCommand
35 from buildbot.steps.shell import ShellCommand
36 from buildbot.steps.source.git import Git
37 from buildbot.steps.transfer import FileDownload
38 from buildbot.steps.transfer import FileUpload
39 from buildbot.steps.transfer import StringDownload
40 from buildbot.worker import Worker
43 if not os.path.exists("twistd.pid"):
44 with open("twistd.pid", "w") as pidfile:
45 pidfile.write("{}".format(os.getpid()))
47 # This is a sample buildmaster config file. It must be installed as
48 # 'master.cfg' in your buildmaster's base directory.
50 ini = configparser.ConfigParser()
51 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
53 if "general" not in ini or "phase1" not in ini:
54 raise ValueError("Fix your configuration")
59 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
60 scripts_dir = os.path.abspath("../scripts")
62 repo_url = ini['repo'].get("url")
64 rsync_defopts = ["-v", "-4", "--timeout=120"]
66 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
67 # rsync_bin_defopts += ["--contimeout=20"]
71 def ini_parse_branch(section):
73 name = section.get("name")
76 raise ValueError("missing 'name' in " + repr(section))
78 raise ValueError("duplicate branch name in " + repr(section))
81 b["bin_url"] = section.get("binary_url")
82 b["bin_key"] = section.get("binary_password")
84 b["src_url"] = section.get("source_url")
85 b["src_key"] = section.get("source_password")
87 b["gpg_key"] = section.get("gpg_key")
89 b["usign_key"] = section.get("usign_key")
90 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
91 b["usign_comment"] = section.get("usign_comment", usign_comment)
93 b["config_seed"] = section.get("config_seed")
95 b["kmod_archive"] = section.getboolean("kmod_archive", False)
98 log.msg("Configured branch: {}".format(name))
100 # PB port can be either a numeric port or a connection string
101 pb_port = inip1.get("port") or 9989
103 # This is the dictionary that the buildmaster pays attention to. We also use
104 # a shorter alias to save typing.
105 c = BuildmasterConfig = {}
107 ####### PROJECT IDENTITY
109 # the 'title' string will appear at the top of this buildbot
110 # installation's html.WebStatus home page (linked to the
111 # 'titleURL') and is embedded in the title of the waterfall HTML page.
113 c['title'] = ini['general'].get("title")
114 c['titleURL'] = ini['general'].get("title_url")
116 # the 'buildbotURL' string should point to the location where the buildbot's
117 # internal web server (usually the html.WebStatus page) is visible. This
118 # typically uses the port number set in the Waterfall 'status' entry, but
119 # with an externally-visible host name which the buildbot cannot figure out
122 c['buildbotURL'] = inip1.get("buildbot_url")
126 # The 'workers' list defines the set of recognized buildworkers. Each element is
127 # a Worker object, specifying a unique worker name and password. The same
128 # worker name and password must be configured on the worker.
133 for section in ini.sections():
134 if section.startswith("branch "):
135 ini_parse_branch(ini[section])
137 if section.startswith("worker "):
138 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
139 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
140 sl_props = { 'dl_lock':None, 'ul_lock':None }
141 name = ini.get(section, "name")
142 password = ini.get(section, "password")
143 if ini.has_option(section, "dl_lock"):
144 lockname = ini.get(section, "dl_lock")
145 sl_props['dl_lock'] = lockname
146 if lockname not in NetLocks:
147 NetLocks[lockname] = locks.MasterLock(lockname)
148 if ini.has_option(section, "ul_lock"):
149 lockname = ini.get(section, "ul_lock")
150 sl_props['ul_lock'] = lockname
151 if lockname not in NetLocks:
152 NetLocks[lockname] = locks.MasterLock(lockname)
153 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
155 c['protocols'] = {'pb': {'port': pb_port}}
158 c['collapseRequests'] = True
160 # Reduce amount of backlog data
161 c['configurators'] = [util.JanitorConfigurator(
162 logHorizon=timedelta(days=3),
166 @defer.inlineCallbacks
167 def getNewestCompleteTime(bldr):
168 """Returns the complete_at of the latest completed and not SKIPPED
169 build request for this builder, or None if there are no such build
170 requests. We need to filter out SKIPPED requests because we're
171 using collapseRequests=True which is unfortunately marking all
172 previous requests as complete when new buildset is created.
174 @returns: datetime instance or None, via Deferred
177 bldrid = yield bldr.getBuilderId()
178 completed = yield bldr.master.data.get(
179 ('builders', bldrid, 'buildrequests'),
181 resultspec.Filter('complete', 'eq', [True]),
182 resultspec.Filter('results', 'ne', [results.SKIPPED]),
184 order=['-complete_at'], limit=1)
188 complete_at = completed[0]['complete_at']
190 last_build = yield bldr.master.data.get(
193 resultspec.Filter('builderid', 'eq', [bldrid]),
195 order=['-started_at'], limit=1)
197 if last_build and last_build[0]:
198 last_complete_at = last_build[0]['complete_at']
199 if last_complete_at and (last_complete_at > complete_at):
200 return last_complete_at
204 @defer.inlineCallbacks
205 def prioritizeBuilders(master, builders):
206 """Returns sorted list of builders by their last timestamp of completed and
209 @returns: list of sorted builders
212 def is_building(bldr):
213 return bool(bldr.building) or bool(bldr.old_building)
216 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
217 d.addCallback(lambda complete_at: (complete_at, bldr))
221 (complete_at, bldr) = item
223 if bldr.name == "00_force_build":
225 complete_at = date.replace(tzinfo=tzutc())
226 return (complete_at, bldr.name)
230 complete_at = date.replace(tzinfo=tzutc())
232 if is_building(bldr):
234 complete_at = date.replace(tzinfo=tzutc())
236 return (complete_at, bldr.name)
238 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
239 results.sort(key=bldr_sort)
242 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
244 return [r[1] for r in results]
246 c['prioritizeBuilders'] = prioritizeBuilders
248 ####### CHANGESOURCES
250 branchNames = [branches[b]["name"] for b in branches]
255 def populateTargets():
256 log.msg("Populating targets, this will take time")
257 sourcegit = work_dir + '/source.git'
258 for branch in branchNames:
259 if os.path.isdir(sourcegit):
260 subprocess.call(["rm", "-rf", sourcegit])
262 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
264 os.makedirs(sourcegit + '/tmp', exist_ok=True)
265 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
266 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
269 line = findtargets.stdout.readline()
272 ta = line.decode().strip().split(' ')
275 subprocess.call(["rm", "-rf", sourcegit])
279 # the 'change_source' setting tells the buildmaster how it should find out
280 # about source code changes. Here we point to the buildbot clone of pyflakes.
282 c['change_source'] = []
283 c['change_source'].append(GitPoller(
285 workdir=work_dir+'/work.git', branches=branchNames,
286 pollAtLaunch=True, pollinterval=300))
290 # Configure the Schedulers, which decide how to react to incoming changes. In this
291 # case, just kick off a 'basebuild' build
293 class TagChoiceParameter(BaseParameter):
294 spec_attributes = ["strict", "choices"]
298 def __init__(self, name, label=None, **kw):
299 super().__init__(name, label, **kw)
300 self._choice_list = []
307 for b in branchNames:
308 basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
310 branchvers.append(basever[1])
312 alltags = subprocess.Popen(
313 ['git', 'ls-remote', '--tags', repo_url],
314 stdout = subprocess.PIPE)
317 line = alltags.stdout.readline()
322 (ref, tag) = line.split()
324 tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
326 # only list tags matching configured branches
327 if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
328 taglist.append(tagver[1])
330 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
331 taglist.insert(0, '')
333 self._choice_list = taglist
335 return self._choice_list
337 def updateFromKwargs(self, properties, kwargs, **unused):
338 tag = self.getFromKwargs(kwargs)
339 properties[self.name] = tag
341 # find the commit matching the tag
342 findrev = subprocess.Popen(['git', 'rev-parse', 'tags/'+tag], stdout=subprocess.PIPE, cwd=work_dir+'/work.git')
343 findrev.wait(timeout=10)
344 line = findrev.stdout.readline()
346 if findrev.returncode!=0 or not line:
347 raise ValidationError("Couldn't find tag")
349 properties['force_revision'] = line.decode().strip()
351 # find the branch matching the tag
353 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
354 for b in branchNames:
355 if b.endswith(branchver[1]):
359 raise ValidationError("Couldn't find branch")
361 properties['force_branch'] = branch
363 def parse_from_arg(self, s):
364 if self.strict and s not in self._choice_list:
365 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
369 c['schedulers'].append(AnyBranchScheduler(
371 change_filter = util.ChangeFilter(branch=branchNames),
372 treeStableTimer = 15*60,
373 builderNames = list(targets)))
375 c['schedulers'].append(ForceScheduler(
377 buttonName = "Force builds",
378 label = "Force build details",
379 builderNames = [ "00_force_build" ],
382 util.CodebaseParameter(
384 label = "Repository",
385 branch = util.FixedParameter(name = "branch", default = ""),
386 revision = util.FixedParameter(name = "revision", default = ""),
387 repository = util.FixedParameter(name = "repository", default = ""),
388 project = util.FixedParameter(name = "project", default = "")
392 reason = util.StringParameter(
395 default = "Trigger build",
401 util.ChoiceStringParameter(
403 label = "Build target",
405 choices = [ "all" ] + list(targets)
417 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
418 # what steps, and which workers can execute them. Note that any particular build will
419 # only take place on one worker.
421 def IsNoMasterBuild(step):
422 return step.getProperty("branch") != "master"
424 def IsUsignEnabled(step):
425 branch = step.getProperty("branch")
426 return branch and branches[branch].get("usign_key")
428 def IsSignEnabled(step):
429 branch = step.getProperty("branch")
430 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
432 def IsKmodArchiveEnabled(step):
433 branch = step.getProperty("branch")
434 return branch and branches[branch].get("kmod_archive")
436 def GetBaseVersion(branch):
437 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
438 return branch.split('-')[1]
443 def GetVersionPrefix(props):
444 branch = props.getProperty("branch")
445 basever = GetBaseVersion(branch)
446 if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
447 return "%s/" % props["tag"][1:]
448 elif basever != "master":
449 return "%s-SNAPSHOT/" % basever
454 def GetConfigSeed(props):
455 branch = props.getProperty("branch")
456 return branch and branches[branch].get("config_seed") or ""
459 def GetRsyncParams(props, srcorbin, urlorkey):
460 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
461 branch = props.getProperty("branch")
462 opt = srcorbin + "_" + urlorkey
463 return branch and branches[branch].get(opt)
466 def GetUsignKey(props):
467 branch = props.getProperty("branch")
468 return branch and branches[branch].get("usign_key")
470 def GetNextBuild(builder, requests):
473 # order tagged build first
474 if r.properties.hasProperty("tag"):
476 # then order by branch order
477 pbranch = r.properties.getProperty("branch")
478 for name in branchNames:
483 log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
486 def MakeEnv(overrides=None, tryccache=False):
488 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
489 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
492 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
493 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
494 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
496 env['CC'] = env['CCC']
497 env['CXX'] = env['CCXX']
499 if overrides is not None:
500 env.update(overrides)
504 def NetLockDl(props, extralock=None):
507 if props.hasProperty("dl_lock"):
508 lock = NetLocks[props["dl_lock"]]
510 locks.append(lock.access('exclusive'))
511 if extralock is not None:
512 locks.append(extralock)
516 def NetLockUl(props):
518 if props.hasProperty("ul_lock"):
519 lock = NetLocks[props["ul_lock"]]
521 return [lock.access('exclusive')]
525 def IsTargetSelected(target):
526 def CheckTargetProperty(step):
527 selected_target = step.getProperty("target", "all")
528 if selected_target != "all" and selected_target != target:
532 return CheckTargetProperty
535 def UsignSec2Pub(props):
536 branch = props.getProperty("branch")
538 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
539 seckey = branches[branch].get("usign_key")
540 seckey = base64.b64decode(seckey)
544 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
545 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
550 dlLock = locks.WorkerLock("worker_dl")
554 for worker in c['workers']:
555 workerNames.append(worker.workername)
557 force_factory = BuildFactory()
559 c['builders'].append(BuilderConfig(
560 name = "00_force_build",
561 workernames = workerNames,
562 factory = force_factory))
564 for target in targets:
565 ts = target.split('/')
567 factory = BuildFactory()
569 # setup shared work directory if required
570 factory.addStep(ShellCommand(
572 descriptionDone = "Shared work directory set up",
573 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
575 haltOnFailure = True,
578 # find number of cores
579 factory.addStep(SetPropertyFromCommand(
582 description = "Finding number of CPUs",
586 # find gcc and g++ compilers
587 factory.addStep(FileDownload(
588 name = "dlfindbinpl",
589 mastersrc = scripts_dir + '/findbin.pl',
590 workerdest = "../findbin.pl",
594 factory.addStep(SetPropertyFromCommand(
596 property = "cc_command",
597 description = "Finding gcc command",
598 command = ["../findbin.pl", "gcc", "", ""],
599 haltOnFailure = True,
602 factory.addStep(SetPropertyFromCommand(
604 property = "cxx_command",
605 description = "Finding g++ command",
606 command = ["../findbin.pl", "g++", "", ""],
607 haltOnFailure = True,
610 # see if ccache is available
611 factory.addStep(SetPropertyFromCommand(
613 property = "ccache_command",
614 description = "Testing for ccache command",
615 command = ["which", "ccache"],
616 haltOnFailure = False,
617 flunkOnFailure = False,
618 warnOnFailure = False,
619 hideStepIf = lambda r, s: r==results.FAILURE,
622 # check out the source
624 # if repo doesn't exist: 'git clone repourl'
625 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
626 # git cat-file -e <commit>
627 # git checkout -f <commit>
628 # git checkout -B <branch>
636 haltOnFailure = True,
640 factory.addStep(ShellCommand(
642 description = "Fetching Git remote refs",
643 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
644 haltOnFailure = True,
647 # Verify that Git HEAD points to a tag or branch
648 # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
649 factory.addStep(ShellCommand(
651 description = "Ensure that Git HEAD is pointing to a branch or tag",
652 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
653 haltOnFailure = True,
656 factory.addStep(ShellCommand(
658 description = "Remove tmp folder",
659 command=["rm", "-rf", "tmp/"],
663 factory.addStep(ShellCommand(
664 name = "rmfeedlinks",
665 description = "Remove feed symlinks",
666 command=["rm", "-rf", "package/feeds/"],
669 factory.addStep(StringDownload(
671 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
672 workerdest = "../ccache_cc.sh",
676 factory.addStep(StringDownload(
678 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
679 workerdest = "../ccache_cxx.sh",
684 factory.addStep(ShellCommand(
685 name = "updatefeeds",
686 description = "Updating feeds",
687 command=["./scripts/feeds", "update"],
688 env = MakeEnv(tryccache=True),
689 haltOnFailure = True,
694 factory.addStep(ShellCommand(
695 name = "installfeeds",
696 description = "Installing feeds",
697 command=["./scripts/feeds", "install", "-a"],
698 env = MakeEnv(tryccache=True),
699 haltOnFailure = True,
703 factory.addStep(StringDownload(
704 name = "dlconfigseed",
705 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
706 workerdest = ".config",
711 factory.addStep(ShellCommand(
713 descriptionDone = ".config seeded",
714 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
717 factory.addStep(ShellCommand(
719 description = "Removing output directory",
720 command = ["rm", "-rf", "bin/"],
723 factory.addStep(ShellCommand(
725 description = "Populating .config",
726 command = ["make", "defconfig"],
730 # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
731 factory.addStep(ShellCommand(
733 description = "Checking architecture",
734 descriptionDone = "Architecture validated",
735 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
739 haltOnFailure = True,
740 flunkOnFailure = False, # this is not a build FAILURE
744 factory.addStep(SetPropertyFromCommand(
747 description = "Finding libc suffix",
748 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
752 factory.addStep(StringDownload(
753 name = "dlkeybuildpub",
754 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
755 workerdest = "key-build.pub",
757 doStepIf = IsUsignEnabled,
760 factory.addStep(StringDownload(
762 s = "# fake private key",
763 workerdest = "key-build",
765 doStepIf = IsUsignEnabled,
768 factory.addStep(StringDownload(
769 name = "dlkeybuilducert",
770 s = "# fake certificate",
771 workerdest = "key-build.ucert",
773 doStepIf = IsUsignEnabled,
777 factory.addStep(ShellCommand(
779 description = "Preparing dl/",
780 descriptionDone = "dl/ prepared",
781 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
782 workdir = Property("builddir"),
788 factory.addStep(ShellCommand(
790 description = "Building and installing GNU tar",
791 descriptionDone = "GNU tar built and installed",
792 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
793 env = MakeEnv(tryccache=True),
794 haltOnFailure = True,
798 factory.addStep(ShellCommand(
800 description = "Populating dl/",
801 descriptionDone = "dl/ populated",
802 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
805 locks = NetLockDl.withArgs(dlLock.access('exclusive')),
808 factory.addStep(ShellCommand(
810 description = "Cleaning base-files",
811 command=["make", "package/base-files/clean", "V=s"],
815 factory.addStep(ShellCommand(
817 description = "Building and installing tools",
818 descriptionDone = "Tools built and installed",
819 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
820 env = MakeEnv(tryccache=True),
821 haltOnFailure = True,
824 factory.addStep(ShellCommand(
826 description = "Building and installing toolchain",
827 descriptionDone = "Toolchain built and installed",
828 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
830 haltOnFailure = True,
833 factory.addStep(ShellCommand(
835 description = "Building kmods",
836 descriptionDone = "Kmods built",
837 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
839 haltOnFailure = True,
842 # find kernel version
843 factory.addStep(SetPropertyFromCommand(
844 name = "kernelversion",
845 property = "kernelversion",
846 description = "Finding the effective Kernel version",
847 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
848 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
851 factory.addStep(ShellCommand(
853 description = "Cleaning up package build",
854 descriptionDone = "Package build cleaned up",
855 command=["make", "package/cleanup", "V=s"],
858 factory.addStep(ShellCommand(
860 description = "Building packages",
861 descriptionDone = "Packages built",
862 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
864 haltOnFailure = True,
867 factory.addStep(ShellCommand(
869 description = "Installing packages",
870 descriptionDone = "Packages installed",
871 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
873 haltOnFailure = True,
876 factory.addStep(ShellCommand(
878 description = "Indexing packages",
879 descriptionDone = "Packages indexed",
880 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
882 haltOnFailure = True,
885 factory.addStep(ShellCommand(
887 description = "Building and installing images",
888 descriptionDone = "Images built and installed",
889 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
891 haltOnFailure = True,
894 factory.addStep(ShellCommand(
896 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
897 command = "make -j1 buildinfo V=s || true",
899 haltOnFailure = True,
902 factory.addStep(ShellCommand(
903 name = "json_overview_image_info",
904 description = "Generating profiles.json in target folder",
905 command = "make -j1 json_overview_image_info V=s || true",
907 haltOnFailure = True,
910 factory.addStep(ShellCommand(
912 description = "Calculating checksums",
913 descriptionDone = "Checksums calculated",
914 command=["make", "-j1", "checksum", "V=s"],
916 haltOnFailure = True,
919 factory.addStep(ShellCommand(
921 descriptionDone = "Kmod directory created",
922 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
923 haltOnFailure = True,
924 doStepIf = IsKmodArchiveEnabled,
927 factory.addStep(ShellCommand(
928 name = "kmodprepare",
929 description = "Preparing kmod archive",
930 descriptionDone = "Kmod archive prepared",
931 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
932 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
933 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
934 haltOnFailure = True,
935 doStepIf = IsKmodArchiveEnabled,
938 factory.addStep(ShellCommand(
940 description = "Indexing kmod archive",
941 descriptionDone = "Kmod archive indexed",
942 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
943 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
945 haltOnFailure = True,
946 doStepIf = IsKmodArchiveEnabled,
950 factory.addStep(MasterShellCommand(
951 name = "signprepare",
952 descriptionDone = "Temporary signing directory prepared",
953 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
954 haltOnFailure = True,
955 doStepIf = IsSignEnabled,
959 factory.addStep(ShellCommand(
961 description = "Packing files to sign",
962 descriptionDone = "Files to sign packed",
963 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
964 haltOnFailure = True,
965 doStepIf = IsSignEnabled,
968 factory.addStep(FileUpload(
969 workersrc = "sign.tar.gz",
970 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
971 haltOnFailure = True,
972 doStepIf = IsSignEnabled,
975 factory.addStep(MasterShellCommand(
977 description = "Signing files",
978 descriptionDone = "Files signed",
979 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
980 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
981 haltOnFailure = True,
982 doStepIf = IsSignEnabled,
985 factory.addStep(FileDownload(
986 name = "dlsigntargz",
987 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
988 workerdest = "sign.tar.gz",
989 haltOnFailure = True,
990 doStepIf = IsSignEnabled,
993 factory.addStep(ShellCommand(
995 description = "Unpacking signed files",
996 descriptionDone = "Signed files unpacked",
997 command = ["tar", "-xzf", "sign.tar.gz"],
998 haltOnFailure = True,
999 doStepIf = IsSignEnabled,
1003 factory.addStep(ShellCommand(
1004 name = "dirprepare",
1005 descriptionDone = "Upload directory structure prepared",
1006 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1007 haltOnFailure = True,
1010 factory.addStep(ShellCommand(
1011 name = "linkprepare",
1012 descriptionDone = "Repository symlink prepared",
1013 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1014 doStepIf = IsNoMasterBuild,
1015 haltOnFailure = True,
1018 factory.addStep(ShellCommand(
1019 name = "kmoddirprepare",
1020 descriptionDone = "Kmod archive upload directory prepared",
1021 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1022 haltOnFailure = True,
1023 doStepIf = IsKmodArchiveEnabled,
1026 factory.addStep(ShellCommand(
1028 description = "Uploading directory structure",
1029 descriptionDone = "Directory structure uploaded",
1030 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1031 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1032 haltOnFailure = True,
1035 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1038 # download remote sha256sums to 'target-sha256sums'
1039 factory.addStep(ShellCommand(
1040 name = "target-sha256sums",
1041 description = "Fetching remote sha256sums for target",
1042 descriptionDone = "Remote sha256sums for target fetched",
1043 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1044 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1046 haltOnFailure = False,
1047 flunkOnFailure = False,
1048 warnOnFailure = False,
1049 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1052 # build list of files to upload
1053 factory.addStep(FileDownload(
1054 name = "dlsha2rsyncpl",
1055 mastersrc = scripts_dir + '/sha2rsync.pl',
1056 workerdest = "../sha2rsync.pl",
1060 factory.addStep(ShellCommand(
1062 description = "Building list of files to upload",
1063 descriptionDone = "List of files to upload built",
1064 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1065 haltOnFailure = True,
1068 factory.addStep(FileDownload(
1069 name = "dlrsync.sh",
1070 mastersrc = scripts_dir + '/rsync.sh',
1071 workerdest = "../rsync.sh",
1075 # upload new files and update existing ones
1076 factory.addStep(ShellCommand(
1077 name = "targetupload",
1078 description = "Uploading target files",
1079 descriptionDone = "Target files uploaded",
1080 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1081 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1082 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1083 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1084 haltOnFailure = True,
1086 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1089 # delete files which don't exist locally
1090 factory.addStep(ShellCommand(
1091 name = "targetprune",
1092 description = "Pruning target files",
1093 descriptionDone = "Target files pruned",
1094 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1095 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1096 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1097 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1098 haltOnFailure = True,
1101 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1104 factory.addStep(ShellCommand(
1105 name = "kmodupload",
1106 description = "Uploading kmod archive",
1107 descriptionDone = "Kmod archive uploaded",
1108 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1109 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1110 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1111 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1112 haltOnFailure = True,
1115 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1118 factory.addStep(ShellCommand(
1119 name = "sourcelist",
1120 description = "Finding source archives to upload",
1121 descriptionDone = "Source archives to upload found",
1122 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1123 haltOnFailure = True,
1126 factory.addStep(ShellCommand(
1127 name = "sourceupload",
1128 description = "Uploading source archives",
1129 descriptionDone = "Source archives uploaded",
1130 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1131 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1132 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1133 haltOnFailure = True,
1136 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1139 factory.addStep(ShellCommand(
1141 description = "Reporting disk usage",
1142 command=["df", "-h", "."],
1143 env={'LC_ALL': 'C'},
1145 haltOnFailure = False,
1146 flunkOnFailure = False,
1147 warnOnFailure = False,
1151 factory.addStep(ShellCommand(
1153 description = "Reporting estimated file space usage",
1154 command=["du", "-sh", "."],
1155 env={'LC_ALL': 'C'},
1157 haltOnFailure = False,
1158 flunkOnFailure = False,
1159 warnOnFailure = False,
1163 factory.addStep(ShellCommand(
1164 name = "ccachestat",
1165 description = "Reporting ccache stats",
1166 command=["ccache", "-s"],
1167 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1169 want_stderr = False,
1170 haltOnFailure = False,
1171 flunkOnFailure = False,
1172 warnOnFailure = False,
1173 hideStepIf = lambda r, s: r==results.FAILURE,
1176 c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1178 c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1179 force_factory.addStep(steps.Trigger(
1180 name = "trigger_%s" % target,
1181 description = "Triggering %s build" % target,
1182 schedulerNames = [ "trigger_%s" % target ],
1183 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
1184 set_properties = { "reason": Property("reason"), "tag": Property("tag"), },
1185 doStepIf = IsTargetSelected(target),
1189 ####### STATUS TARGETS
1191 # 'status' is a list of Status Targets. The results of each build will be
1192 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1193 # including web pages, email senders, and IRC bots.
1195 if "status_bind" in inip1:
1197 'port': inip1.get("status_bind"),
1199 'waterfall_view': True,
1200 'console_view': True,
1205 if "status_user" in inip1 and "status_password" in inip1:
1206 c['www']['auth'] = util.UserPasswordAuth([
1207 (inip1.get("status_user"), inip1.get("status_password"))
1209 c['www']['authz'] = util.Authz(
1210 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1211 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1215 if ini.has_section("irc"):
1217 irc_host = iniirc.get("host", None)
1218 irc_port = iniirc.getint("port", 6667)
1219 irc_chan = iniirc.get("channel", None)
1220 irc_nick = iniirc.get("nickname", None)
1221 irc_pass = iniirc.get("password", None)
1223 if irc_host and irc_nick and irc_chan:
1224 irc = reporters.IRC(irc_host, irc_nick,
1226 password = irc_pass,
1227 channels = [ irc_chan ],
1228 notify_events = [ 'exception', 'problem', 'recovery' ]
1231 c['services'].append(irc)
1233 c['revlink'] = util.RevlinkMatch([
1234 r'https://git.openwrt.org/openwrt/(.*).git'
1236 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1241 # This specifies what database buildbot uses to store its state. You can leave
1242 # this at its default for all but the largest installations.
1243 'db_url' : "sqlite:///state.sqlite",
1246 c['buildbotNetUsageData'] = None