2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes.gitpoller import GitPoller
19 from buildbot.config import BuilderConfig
20 from buildbot.plugins import reporters
21 from buildbot.plugins import schedulers
22 from buildbot.plugins import steps
23 from buildbot.plugins import util
24 from buildbot.process import properties
25 from buildbot.process import results
26 from buildbot.process.factory import BuildFactory
27 from buildbot.process.properties import Interpolate
28 from buildbot.process.properties import Property
29 from buildbot.schedulers.basic import AnyBranchScheduler
30 from buildbot.schedulers.forcesched import BaseParameter
31 from buildbot.schedulers.forcesched import ForceScheduler
32 from buildbot.schedulers.forcesched import ValidationError
33 from buildbot.steps.master import MasterShellCommand
34 from buildbot.steps.shell import SetPropertyFromCommand
35 from buildbot.steps.shell import ShellCommand
36 from buildbot.steps.source.git import Git
37 from buildbot.steps.transfer import FileDownload
38 from buildbot.steps.transfer import FileUpload
39 from buildbot.steps.transfer import StringDownload
40 from buildbot.worker import Worker
41 from buildbot.worker.local import LocalWorker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini['repo'].get("url")
65 rsync_defopts = ["-v", "--timeout=120"]
67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
68 # rsync_bin_defopts += ["--contimeout=20"]
72 def ini_parse_branch(section):
74 name = section.get("name")
77 raise ValueError("missing 'name' in " + repr(section))
79 raise ValueError("duplicate branch name in " + repr(section))
82 b["bin_url"] = section.get("binary_url")
83 b["bin_key"] = section.get("binary_password")
85 b["src_url"] = section.get("source_url")
86 b["src_key"] = section.get("source_password")
88 b["gpg_key"] = section.get("gpg_key")
90 b["usign_key"] = section.get("usign_key")
91 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
92 b["usign_comment"] = section.get("usign_comment", usign_comment)
94 b["config_seed"] = section.get("config_seed")
96 b["kmod_archive"] = section.getboolean("kmod_archive", False)
99 log.msg("Configured branch: {}".format(name))
101 # PB port can be either a numeric port or a connection string
102 pb_port = inip1.get("port") or 9989
104 # This is the dictionary that the buildmaster pays attention to. We also use
105 # a shorter alias to save typing.
106 c = BuildmasterConfig = {}
108 ####### PROJECT IDENTITY
110 # the 'title' string will appear at the top of this buildbot
111 # installation's html.WebStatus home page (linked to the
112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
114 c['title'] = ini['general'].get("title")
115 c['titleURL'] = ini['general'].get("title_url")
117 # the 'buildbotURL' string should point to the location where the buildbot's
118 # internal web server (usually the html.WebStatus page) is visible. This
119 # typically uses the port number set in the Waterfall 'status' entry, but
120 # with an externally-visible host name which the buildbot cannot figure out
123 c['buildbotURL'] = inip1.get("buildbot_url")
127 # The 'workers' list defines the set of recognized buildworkers. Each element is
128 # a Worker object, specifying a unique worker name and password. The same
129 # worker name and password must be configured on the worker.
134 def ini_parse_workers(section):
135 name = section.get("name")
136 password = section.get("password")
137 phase = section.getint("phase")
138 tagonly = section.getboolean("tag_only")
139 rsyncipv4 = section.getboolean("rsync_ipv4")
141 if not name or not password or not phase == 1:
142 log.msg("invalid worker configuration ignored: {}".format(repr(section)))
145 sl_props = { 'tag_only':tagonly }
146 if "dl_lock" in section:
147 lockname = section.get("dl_lock")
148 sl_props['dl_lock'] = lockname
149 if lockname not in NetLocks:
150 NetLocks[lockname] = locks.MasterLock(lockname)
151 if "ul_lock" in section:
152 lockname = section.get("ul_lock")
153 sl_props['ul_lock'] = lockname
154 if lockname not in NetLocks:
155 NetLocks[lockname] = locks.MasterLock(lockname)
157 sl_props['rsync_ipv4'] = True # only set prop if required, we use '+' Interpolate substitution
159 log.msg("Configured worker: {}".format(name))
160 # NB: phase1 build factory requires workers to be single-build only
161 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
164 for section in ini.sections():
165 if section.startswith("branch "):
166 ini_parse_branch(ini[section])
168 if section.startswith("worker "):
169 ini_parse_workers(ini[section])
171 # list of branches in build-priority order
172 branchNames = [branches[b]["name"] for b in branches]
174 c['protocols'] = {'pb': {'port': pb_port}}
177 c['collapseRequests'] = True
179 # Reduce amount of backlog data
180 c['configurators'] = [util.JanitorConfigurator(
181 logHorizon=timedelta(days=3),
185 @defer.inlineCallbacks
186 def getNewestCompleteTime(bldr):
187 """Returns the complete_at of the latest completed and not SKIPPED
188 build request for this builder, or None if there are no such build
189 requests. We need to filter out SKIPPED requests because we're
190 using collapseRequests=True which is unfortunately marking all
191 previous requests as complete when new buildset is created.
193 @returns: datetime instance or None, via Deferred
196 bldrid = yield bldr.getBuilderId()
197 completed = yield bldr.master.data.get(
198 ('builders', bldrid, 'buildrequests'),
200 resultspec.Filter('complete', 'eq', [True]),
201 resultspec.Filter('results', 'ne', [results.SKIPPED]),
203 order=['-complete_at'], limit=1)
207 complete_at = completed[0]['complete_at']
209 last_build = yield bldr.master.data.get(
212 resultspec.Filter('builderid', 'eq', [bldrid]),
214 order=['-started_at'], limit=1)
216 if last_build and last_build[0]:
217 last_complete_at = last_build[0]['complete_at']
218 if last_complete_at and (last_complete_at > complete_at):
219 return last_complete_at
223 @defer.inlineCallbacks
224 def prioritizeBuilders(master, builders):
225 """Returns sorted list of builders by their last timestamp of completed and
226 not skipped build, ordered first by branch name.
228 @returns: list of sorted builders
231 bldrNamePrio = { "__Janitor": 0, "00_force_build": 0 }
233 for bname in branchNames:
234 bldrNamePrio[bname] = i
237 def is_building(bldr):
238 return bool(bldr.building) or bool(bldr.old_building)
241 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
242 d.addCallback(lambda complete_at: (complete_at, bldr))
246 (complete_at, bldr) = item
249 for (name, prio) in bldrNamePrio.items():
250 if bldr.name.startswith(name):
256 complete_at = date.replace(tzinfo=tzutc())
258 if is_building(bldr):
260 complete_at = date.replace(tzinfo=tzutc())
262 return (pos, complete_at, bldr.name)
264 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
265 results.sort(key=bldr_sort)
268 # log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
270 return [r[1] for r in results]
272 c['prioritizeBuilders'] = prioritizeBuilders
274 ####### CHANGESOURCES
279 def populateTargets():
280 """ fetch a shallow clone of each configured branch in turn:
281 execute dump-target-info.pl and collate the results to ensure
282 targets that only exist in specific branches get built.
283 This takes a while during master startup but is executed only once.
285 log.msg("Populating targets, this will take time")
286 sourcegit = work_dir + '/source.git'
287 for branch in branchNames:
288 if os.path.isdir(sourcegit):
289 subprocess.call(["rm", "-rf", sourcegit])
291 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
293 os.makedirs(sourcegit + '/tmp', exist_ok=True)
294 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
295 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
298 line = findtargets.stdout.readline()
301 ta = line.decode().strip().split(' ')
304 subprocess.call(["rm", "-rf", sourcegit])
308 # the 'change_source' setting tells the buildmaster how it should find out
309 # about source code changes.
311 c['change_source'] = []
312 c['change_source'].append(GitPoller(
314 workdir=work_dir+'/work.git', branches=branchNames,
315 pollAtLaunch=True, pollinterval=300))
319 # Configure the Schedulers, which decide how to react to incoming changes.
321 # Selector for known valid tags
322 class TagChoiceParameter(BaseParameter):
323 spec_attributes = ["strict", "choices"]
327 def __init__(self, name, label=None, **kw):
328 super().__init__(name, label, **kw)
329 self._choice_list = []
331 def getRevTags(self, findtag=None):
335 # we will filter out tags that do no match the configured branches
336 for b in branchNames:
337 basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
339 branchvers.append(basever[1])
341 # grab tags from remote repository
342 alltags = subprocess.Popen(
343 ['git', 'ls-remote', '--tags', repo_url],
344 stdout = subprocess.PIPE)
347 line = alltags.stdout.readline()
352 (rev, tag) = line.split()
354 # does it match known format? ('vNN.NN.NN(-rcN)')
355 tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
357 # only list valid tags matching configured branches
358 if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
359 # if we want a specific tag, ignore all that don't match
360 if findtag and findtag != tagver[1]:
362 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
368 taglist = [rt['tag'] for rt in self.getRevTags()]
369 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
370 taglist.insert(0, '')
372 self._choice_list = taglist
374 return self._choice_list
376 def updateFromKwargs(self, properties, kwargs, **unused):
377 tag = self.getFromKwargs(kwargs)
378 properties[self.name] = tag
380 # find the commit matching the tag
381 findtag = self.getRevTags(tag)
384 raise ValidationError("Couldn't find tag")
386 properties['force_revision'] = findtag[0]['rev']
388 # find the branch matching the tag
390 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
391 for b in branchNames:
392 if b.endswith(branchver[1]):
396 raise ValidationError("Couldn't find branch")
398 properties['force_branch'] = branch
400 def parse_from_arg(self, s):
401 if self.strict and s not in self._choice_list:
402 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
406 @defer.inlineCallbacks
407 def builderNames(props):
408 """ since we have per branch and per target builders,
409 address the relevant builder for each new buildrequest
410 based on the request's desired branch and target.
412 branch = props.getProperty("branch")
413 target = props.getProperty("target", "")
418 # if that didn't work, try sourcestamp to find a branch
420 # match builders with target branch
421 ss = props.sourcestamps[0]
423 branch = ss['branch']
425 log.msg("couldn't find builder")
426 return [] # nothing works
428 bname = branch + "_" + target
431 for b in (yield props.master.data.get(('builders',))):
432 if not b['name'].startswith(bname):
434 builders.append(b['name'])
439 c['schedulers'].append(AnyBranchScheduler(
441 change_filter = util.ChangeFilter(branch=branchNames),
442 treeStableTimer = 15*60,
443 builderNames = builderNames))
445 c['schedulers'].append(ForceScheduler(
447 buttonName = "Force builds",
448 label = "Force build details",
449 builderNames = [ "00_force_build" ],
452 util.CodebaseParameter(
454 label = "Repository",
455 branch = util.FixedParameter(name = "branch", default = ""),
456 revision = util.FixedParameter(name = "revision", default = ""),
457 repository = util.FixedParameter(name = "repository", default = ""),
458 project = util.FixedParameter(name = "project", default = "")
462 reason = util.StringParameter(
465 default = "Trigger build",
471 # NB: avoid nesting to simplify processing of properties
472 util.ChoiceStringParameter(
474 label = "Build target",
476 choices = [ "all" ] + list(targets)
486 c['schedulers'].append(schedulers.Triggerable(name="trigger", builderNames=builderNames))
490 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
491 # what steps, and which workers can execute them. Note that any particular build will
492 # only take place on one worker.
494 def IsNoMasterBuild(step):
495 return step.getProperty("branch") != "master"
497 def IsUsignEnabled(step):
498 branch = step.getProperty("branch")
499 return branch and branches[branch].get("usign_key")
501 def IsSignEnabled(step):
502 branch = step.getProperty("branch")
503 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
505 def IsKmodArchiveEnabled(step):
506 branch = step.getProperty("branch")
507 return branch and branches[branch].get("kmod_archive")
509 def IsKmodArchiveAndRsyncEnabled(step):
510 branch = step.getProperty("branch")
511 return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url"))
513 def GetBaseVersion(branch):
514 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
515 return branch.split('-')[1]
520 def GetVersionPrefix(props):
521 branch = props.getProperty("branch")
522 basever = GetBaseVersion(branch)
523 if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
524 return "%s/" % props["tag"][1:]
525 elif basever != "master":
526 return "%s-SNAPSHOT/" % basever
531 def GetConfigSeed(props):
532 branch = props.getProperty("branch")
533 return branch and branches[branch].get("config_seed") or ""
536 def GetRsyncParams(props, srcorbin, urlorkey):
537 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
538 branch = props.getProperty("branch")
539 opt = srcorbin + "_" + urlorkey
540 return branch and branches[branch].get(opt)
543 def GetUsignKey(props):
544 branch = props.getProperty("branch")
545 return branch and branches[branch].get("usign_key")
547 def GetNextBuild(builder, requests):
550 # order tagged build first
551 if r.properties.hasProperty("tag"):
555 #log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
558 def MakeEnv(overrides=None, tryccache=False):
560 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
561 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
564 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
565 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
566 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
568 env['CC'] = env['CCC']
569 env['CXX'] = env['CCXX']
571 if overrides is not None:
572 env.update(overrides)
576 def NetLockDl(props, extralock=None):
578 if props.hasProperty("dl_lock"):
579 lock = NetLocks[props["dl_lock"]]
581 return [lock.access('exclusive')]
586 def NetLockUl(props):
588 if props.hasProperty("ul_lock"):
589 lock = NetLocks[props["ul_lock"]]
591 return [lock.access('exclusive')]
595 def IsTargetSelected(target):
596 def CheckTargetProperty(step):
597 selected_target = step.getProperty("target", "all")
598 if selected_target != "all" and selected_target != target:
602 return CheckTargetProperty
605 def UsignSec2Pub(props):
606 branch = props.getProperty("branch")
608 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
609 seckey = branches[branch].get("usign_key")
610 seckey = base64.b64decode(seckey)
614 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
615 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
618 def canStartBuild(builder, wfb, request):
619 """ filter out non tag requests for tag_only workers. """
620 wtagonly = wfb.worker.properties.getProperty('tag_only')
621 tag = request.properties.getProperty('tag')
623 if wtagonly and not tag:
632 for worker in c['workers']:
633 workerNames.append(worker.workername)
635 # add a single LocalWorker to handle the forcebuild builder
636 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
638 force_factory = BuildFactory()
639 force_factory.addStep(steps.Trigger(
640 name = "trigger_build",
641 schedulerNames = [ "trigger" ],
642 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
643 set_properties = { "reason": Property("reason"), "tag": Property("tag"), "target": Property("target") },
646 c['builders'].append(BuilderConfig(
647 name = "00_force_build",
648 workername = "__local_force_build",
649 factory = force_factory))
652 # NB the phase1 build factory assumes workers are single-build only
653 for target in targets:
654 ts = target.split('/')
656 factory = BuildFactory()
658 # setup shared work directory if required
659 factory.addStep(ShellCommand(
661 descriptionDone = "Shared work directory set up",
662 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
664 haltOnFailure = True,
667 # find number of cores
668 factory.addStep(SetPropertyFromCommand(
671 description = "Finding number of CPUs",
675 # find gcc and g++ compilers
676 factory.addStep(FileDownload(
677 name = "dlfindbinpl",
678 mastersrc = scripts_dir + '/findbin.pl',
679 workerdest = "../findbin.pl",
683 factory.addStep(SetPropertyFromCommand(
685 property = "cc_command",
686 description = "Finding gcc command",
687 command = ["../findbin.pl", "gcc", "", ""],
688 haltOnFailure = True,
691 factory.addStep(SetPropertyFromCommand(
693 property = "cxx_command",
694 description = "Finding g++ command",
695 command = ["../findbin.pl", "g++", "", ""],
696 haltOnFailure = True,
699 # see if ccache is available
700 factory.addStep(SetPropertyFromCommand(
702 property = "ccache_command",
703 description = "Testing for ccache command",
704 command = ["which", "ccache"],
705 haltOnFailure = False,
706 flunkOnFailure = False,
707 warnOnFailure = False,
708 hideStepIf = lambda r, s: r==results.FAILURE,
711 # check out the source
713 # if repo doesn't exist: 'git clone repourl'
714 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
715 # git cat-file -e <commit>
716 # git checkout -f <commit>
717 # git checkout -B <branch>
725 haltOnFailure = True,
729 factory.addStep(ShellCommand(
731 description = "Fetching Git remote refs",
732 descriptionDone = "Git remote refs fetched",
733 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
734 haltOnFailure = True,
737 # Verify that Git HEAD points to a tag or branch
738 # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
739 factory.addStep(ShellCommand(
741 description = "Ensuring that Git HEAD is pointing to a branch or tag",
742 descriptionDone = "Git HEAD is sane",
743 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
744 haltOnFailure = True,
747 factory.addStep(ShellCommand(
749 description = "Remove tmp folder",
750 command=["rm", "-rf", "tmp/"],
754 factory.addStep(ShellCommand(
755 name = "rmfeedlinks",
756 description = "Remove feed symlinks",
757 command=["rm", "-rf", "package/feeds/"],
760 factory.addStep(StringDownload(
762 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
763 workerdest = "../ccache_cc.sh",
767 factory.addStep(StringDownload(
769 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
770 workerdest = "../ccache_cxx.sh",
775 factory.addStep(ShellCommand(
776 name = "updatefeeds",
777 description = "Updating feeds",
778 command=["./scripts/feeds", "update"],
779 env = MakeEnv(tryccache=True),
780 haltOnFailure = True,
785 factory.addStep(ShellCommand(
786 name = "installfeeds",
787 description = "Installing feeds",
788 command=["./scripts/feeds", "install", "-a"],
789 env = MakeEnv(tryccache=True),
790 haltOnFailure = True,
794 factory.addStep(StringDownload(
795 name = "dlconfigseed",
796 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
797 workerdest = ".config",
802 factory.addStep(ShellCommand(
804 descriptionDone = ".config seeded",
805 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
808 factory.addStep(ShellCommand(
810 description = "Removing output directory",
811 command = ["rm", "-rf", "bin/"],
814 factory.addStep(ShellCommand(
816 description = "Populating .config",
817 command = ["make", "defconfig"],
821 # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
822 factory.addStep(ShellCommand(
824 description = "Checking architecture",
825 descriptionDone = "Architecture validated",
826 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
830 haltOnFailure = True,
831 flunkOnFailure = False, # this is not a build FAILURE - TODO mark build as SKIPPED
835 factory.addStep(SetPropertyFromCommand(
838 description = "Finding libc suffix",
839 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
843 factory.addStep(StringDownload(
844 name = "dlkeybuildpub",
845 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
846 workerdest = "key-build.pub",
848 doStepIf = IsUsignEnabled,
851 factory.addStep(StringDownload(
853 s = "# fake private key",
854 workerdest = "key-build",
856 doStepIf = IsUsignEnabled,
859 factory.addStep(StringDownload(
860 name = "dlkeybuilducert",
861 s = "# fake certificate",
862 workerdest = "key-build.ucert",
864 doStepIf = IsUsignEnabled,
868 factory.addStep(ShellCommand(
870 description = "Preparing dl/",
871 descriptionDone = "dl/ prepared",
872 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
873 workdir = Property("builddir"),
879 factory.addStep(ShellCommand(
881 description = "Pruning dl/",
882 descriptionDone = "dl/ pruned",
883 command = 'find dl/ -atime +15 -delete -print',
888 factory.addStep(ShellCommand(
890 description = "Building and installing GNU tar",
891 descriptionDone = "GNU tar built and installed",
892 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
893 env = MakeEnv(tryccache=True),
894 haltOnFailure = True,
898 factory.addStep(ShellCommand(
900 description = "Populating dl/",
901 descriptionDone = "dl/ populated",
902 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
908 factory.addStep(ShellCommand(
910 description = "Cleaning base-files",
911 command=["make", "package/base-files/clean", "V=s"],
915 factory.addStep(ShellCommand(
917 description = "Building and installing tools",
918 descriptionDone = "Tools built and installed",
919 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
920 env = MakeEnv(tryccache=True),
921 haltOnFailure = True,
924 factory.addStep(ShellCommand(
926 description = "Building and installing toolchain",
927 descriptionDone = "Toolchain built and installed",
928 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
930 haltOnFailure = True,
933 factory.addStep(ShellCommand(
935 description = "Building kmods",
936 descriptionDone = "Kmods built",
937 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
939 haltOnFailure = True,
942 # find kernel version
943 factory.addStep(SetPropertyFromCommand(
944 name = "kernelversion",
945 property = "kernelversion",
946 description = "Finding the effective Kernel version",
947 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
948 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
951 factory.addStep(ShellCommand(
953 description = "Cleaning up package build",
954 descriptionDone = "Package build cleaned up",
955 command=["make", "package/cleanup", "V=s"],
958 factory.addStep(ShellCommand(
960 description = "Building packages",
961 descriptionDone = "Packages built",
962 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
964 haltOnFailure = True,
967 factory.addStep(ShellCommand(
969 description = "Installing packages",
970 descriptionDone = "Packages installed",
971 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
973 haltOnFailure = True,
976 factory.addStep(ShellCommand(
978 description = "Indexing packages",
979 descriptionDone = "Packages indexed",
980 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
982 haltOnFailure = True,
985 factory.addStep(ShellCommand(
987 description = "Building and installing images",
988 descriptionDone = "Images built and installed",
989 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
991 haltOnFailure = True,
994 factory.addStep(ShellCommand(
996 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
997 command = "make -j1 buildinfo V=s || true",
999 haltOnFailure = True,
1002 factory.addStep(ShellCommand(
1003 name = "json_overview_image_info",
1004 description = "Generating profiles.json in target folder",
1005 command = "make -j1 json_overview_image_info V=s || true",
1007 haltOnFailure = True,
1010 factory.addStep(ShellCommand(
1012 description = "Calculating checksums",
1013 descriptionDone = "Checksums calculated",
1014 command=["make", "-j1", "checksum", "V=s"],
1016 haltOnFailure = True,
1019 factory.addStep(ShellCommand(
1021 descriptionDone = "Kmod directory created",
1022 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
1023 haltOnFailure = True,
1024 doStepIf = IsKmodArchiveEnabled,
1027 factory.addStep(ShellCommand(
1028 name = "kmodprepare",
1029 description = "Preparing kmod archive",
1030 descriptionDone = "Kmod archive prepared",
1031 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
1032 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
1033 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1034 haltOnFailure = True,
1035 doStepIf = IsKmodArchiveEnabled,
1038 factory.addStep(ShellCommand(
1040 description = "Indexing kmod archive",
1041 descriptionDone = "Kmod archive indexed",
1042 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
1043 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1045 haltOnFailure = True,
1046 doStepIf = IsKmodArchiveEnabled,
1050 factory.addStep(MasterShellCommand(
1051 name = "signprepare",
1052 descriptionDone = "Temporary signing directory prepared",
1053 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1054 haltOnFailure = True,
1055 doStepIf = IsSignEnabled,
1059 factory.addStep(ShellCommand(
1061 description = "Packing files to sign",
1062 descriptionDone = "Files to sign packed",
1063 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1064 haltOnFailure = True,
1065 doStepIf = IsSignEnabled,
1068 factory.addStep(FileUpload(
1069 workersrc = "sign.tar.gz",
1070 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1071 haltOnFailure = True,
1072 doStepIf = IsSignEnabled,
1075 factory.addStep(MasterShellCommand(
1077 description = "Signing files",
1078 descriptionDone = "Files signed",
1079 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
1080 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1081 haltOnFailure = True,
1082 doStepIf = IsSignEnabled,
1085 factory.addStep(FileDownload(
1086 name = "dlsigntargz",
1087 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1088 workerdest = "sign.tar.gz",
1089 haltOnFailure = True,
1090 doStepIf = IsSignEnabled,
1093 factory.addStep(ShellCommand(
1094 name = "signunpack",
1095 description = "Unpacking signed files",
1096 descriptionDone = "Signed files unpacked",
1097 command = ["tar", "-xzf", "sign.tar.gz"],
1098 haltOnFailure = True,
1099 doStepIf = IsSignEnabled,
1103 factory.addStep(ShellCommand(
1104 name = "dirprepare",
1105 descriptionDone = "Upload directory structure prepared",
1106 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1107 haltOnFailure = True,
1110 factory.addStep(ShellCommand(
1111 name = "linkprepare",
1112 descriptionDone = "Repository symlink prepared",
1113 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1114 doStepIf = IsNoMasterBuild,
1115 haltOnFailure = True,
1118 factory.addStep(ShellCommand(
1119 name = "kmoddirprepare",
1120 descriptionDone = "Kmod archive upload directory prepared",
1121 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1122 haltOnFailure = True,
1123 doStepIf = IsKmodArchiveEnabled,
1126 factory.addStep(ShellCommand(
1128 description = "Uploading directory structure",
1129 descriptionDone = "Directory structure uploaded",
1130 command = ["rsync", Interpolate("-az%(prop:rsync_ipv4:+4)s")] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1131 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1132 haltOnFailure = True,
1135 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1138 # download remote sha256sums to 'target-sha256sums'
1139 factory.addStep(ShellCommand(
1140 name = "target-sha256sums",
1141 description = "Fetching remote sha256sums for target",
1142 descriptionDone = "Remote sha256sums for target fetched",
1143 command = ["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1144 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1146 haltOnFailure = False,
1147 flunkOnFailure = False,
1148 warnOnFailure = False,
1149 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1152 # build list of files to upload
1153 factory.addStep(FileDownload(
1154 name = "dlsha2rsyncpl",
1155 mastersrc = scripts_dir + '/sha2rsync.pl',
1156 workerdest = "../sha2rsync.pl",
1160 factory.addStep(ShellCommand(
1162 description = "Building list of files to upload",
1163 descriptionDone = "List of files to upload built",
1164 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1165 haltOnFailure = True,
1168 factory.addStep(FileDownload(
1169 name = "dlrsync.sh",
1170 mastersrc = scripts_dir + '/rsync.sh',
1171 workerdest = "../rsync.sh",
1175 # upload new files and update existing ones
1176 factory.addStep(ShellCommand(
1177 name = "targetupload",
1178 description = "Uploading target files",
1179 descriptionDone = "Target files uploaded",
1180 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1181 [Interpolate("-a%(prop:rsync_ipv4:+4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1182 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1183 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1184 haltOnFailure = True,
1186 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1189 # delete files which don't exist locally
1190 factory.addStep(ShellCommand(
1191 name = "targetprune",
1192 description = "Pruning target files",
1193 descriptionDone = "Target files pruned",
1194 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1195 [Interpolate("-a%(prop:rsync_ipv4:+4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1196 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1197 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1198 haltOnFailure = True,
1201 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1204 factory.addStep(ShellCommand(
1205 name = "kmodupload",
1206 description = "Uploading kmod archive",
1207 descriptionDone = "Kmod archive uploaded",
1208 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1209 [Interpolate("-a%(prop:rsync_ipv4:+4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1210 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1211 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1212 haltOnFailure = True,
1215 doStepIf = IsKmodArchiveAndRsyncEnabled,
1218 factory.addStep(ShellCommand(
1219 name = "sourcelist",
1220 description = "Finding source archives to upload",
1221 descriptionDone = "Source archives to upload found",
1222 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1223 haltOnFailure = True,
1226 factory.addStep(ShellCommand(
1227 name = "sourceupload",
1228 description = "Uploading source archives",
1229 descriptionDone = "Source archives uploaded",
1230 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1231 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), Interpolate("-a%(prop:rsync_ipv4:+4)s"), "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1232 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1233 haltOnFailure = True,
1236 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1239 factory.addStep(ShellCommand(
1241 description = "Reporting disk usage",
1242 command=["df", "-h", "."],
1243 env={'LC_ALL': 'C'},
1245 haltOnFailure = False,
1246 flunkOnFailure = False,
1247 warnOnFailure = False,
1251 factory.addStep(ShellCommand(
1253 description = "Reporting estimated file space usage",
1254 command=["du", "-sh", "."],
1255 env={'LC_ALL': 'C'},
1257 haltOnFailure = False,
1258 flunkOnFailure = False,
1259 warnOnFailure = False,
1263 factory.addStep(ShellCommand(
1264 name = "ccachestat",
1265 description = "Reporting ccache stats",
1266 command=["ccache", "-s"],
1268 want_stderr = False,
1269 haltOnFailure = False,
1270 flunkOnFailure = False,
1271 warnOnFailure = False,
1272 doStepIf = util.Transform(bool, Property("ccache_command")),
1275 for brname in branchNames:
1276 bldrname = brname + "_" + target
1277 c['builders'].append(BuilderConfig(name=bldrname, workernames=workerNames, factory=factory, tags=[brname,], nextBuild=GetNextBuild, canStartBuild=canStartBuild))
1280 ####### STATUS TARGETS
1282 # 'status' is a list of Status Targets. The results of each build will be
1283 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1284 # including web pages, email senders, and IRC bots.
1286 if "status_bind" in inip1:
1288 'port': inip1.get("status_bind"),
1290 'waterfall_view': True,
1291 'console_view': True,
1296 if "status_user" in inip1 and "status_password" in inip1:
1297 c['www']['auth'] = util.UserPasswordAuth([
1298 (inip1.get("status_user"), inip1.get("status_password"))
1300 c['www']['authz'] = util.Authz(
1301 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1302 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1306 if ini.has_section("irc"):
1308 irc_host = iniirc.get("host", None)
1309 irc_port = iniirc.getint("port", 6667)
1310 irc_chan = iniirc.get("channel", None)
1311 irc_nick = iniirc.get("nickname", None)
1312 irc_pass = iniirc.get("password", None)
1314 if irc_host and irc_nick and irc_chan:
1315 irc = reporters.IRC(irc_host, irc_nick,
1317 password = irc_pass,
1318 channels = [ irc_chan ],
1319 notify_events = [ 'exception', 'problem', 'recovery' ]
1322 c['services'].append(irc)
1324 c['revlink'] = util.RevlinkMatch([
1325 r'https://git.openwrt.org/openwrt/(.*).git'
1327 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1332 # This specifies what database buildbot uses to store its state. You can leave
1333 # this at its default for all but the largest installations.
1334 'db_url' : "sqlite:///state.sqlite",
1337 c['buildbotNetUsageData'] = None