2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes.gitpoller import GitPoller
19 from buildbot.config import BuilderConfig
20 from buildbot.plugins import reporters
21 from buildbot.plugins import schedulers
22 from buildbot.plugins import steps
23 from buildbot.plugins import util
24 from buildbot.process import properties
25 from buildbot.process import results
26 from buildbot.process.factory import BuildFactory
27 from buildbot.process.properties import Interpolate
28 from buildbot.process.properties import Property
29 from buildbot.schedulers.basic import AnyBranchScheduler
30 from buildbot.schedulers.forcesched import BaseParameter
31 from buildbot.schedulers.forcesched import ForceScheduler
32 from buildbot.schedulers.forcesched import ValidationError
33 from buildbot.steps.master import MasterShellCommand
34 from buildbot.steps.shell import SetPropertyFromCommand
35 from buildbot.steps.shell import ShellCommand
36 from buildbot.steps.source.git import Git
37 from buildbot.steps.transfer import FileDownload
38 from buildbot.steps.transfer import FileUpload
39 from buildbot.steps.transfer import StringDownload
40 from buildbot.worker import Worker
41 from buildbot.worker.local import LocalWorker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini["general"].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini["repo"].get("url")
65 rsync_defopts = ["-v", "--timeout=120"]
67 # if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
68 # rsync_bin_defopts += ["--contimeout=20"]
73 def ini_parse_branch(section):
75 name = section.get("name")
78 raise ValueError("missing 'name' in " + repr(section))
80 raise ValueError("duplicate branch name in " + repr(section))
83 b["bin_url"] = section.get("binary_url")
84 b["bin_key"] = section.get("binary_password")
86 b["src_url"] = section.get("source_url")
87 b["src_key"] = section.get("source_password")
89 b["gpg_key"] = section.get("gpg_key")
91 b["usign_key"] = section.get("usign_key")
92 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
93 b["usign_comment"] = section.get("usign_comment", usign_comment)
95 b["config_seed"] = section.get("config_seed")
97 b["kmod_archive"] = section.getboolean("kmod_archive", False)
100 log.msg("Configured branch: {}".format(name))
103 # PB port can be either a numeric port or a connection string
104 pb_port = inip1.get("port") or 9989
106 # This is the dictionary that the buildmaster pays attention to. We also use
107 # a shorter alias to save typing.
108 c = BuildmasterConfig = {}
110 ####### PROJECT IDENTITY
112 # the 'title' string will appear at the top of this buildbot
113 # installation's html.WebStatus home page (linked to the
114 # 'titleURL') and is embedded in the title of the waterfall HTML page.
116 c["title"] = ini["general"].get("title")
117 c["titleURL"] = ini["general"].get("title_url")
119 # the 'buildbotURL' string should point to the location where the buildbot's
120 # internal web server (usually the html.WebStatus page) is visible. This
121 # typically uses the port number set in the Waterfall 'status' entry, but
122 # with an externally-visible host name which the buildbot cannot figure out
125 c["buildbotURL"] = inip1.get("buildbot_url")
129 # The 'workers' list defines the set of recognized buildworkers. Each element is
130 # a Worker object, specifying a unique worker name and password. The same
131 # worker name and password must be configured on the worker.
137 def ini_parse_workers(section):
138 name = section.get("name")
139 password = section.get("password")
140 phase = section.getint("phase")
141 tagonly = section.getboolean("tag_only")
142 rsyncipv4 = section.getboolean("rsync_ipv4")
144 if not name or not password or not phase == 1:
145 log.msg("invalid worker configuration ignored: {}".format(repr(section)))
148 sl_props = {"tag_only": tagonly}
149 if "dl_lock" in section:
150 lockname = section.get("dl_lock")
151 sl_props["dl_lock"] = lockname
152 if lockname not in NetLocks:
153 NetLocks[lockname] = locks.MasterLock(lockname)
154 if "ul_lock" in section:
155 lockname = section.get("ul_lock")
156 sl_props["ul_lock"] = lockname
157 if lockname not in NetLocks:
158 NetLocks[lockname] = locks.MasterLock(lockname)
162 ] = True # only set prop if required, we use '+' Interpolate substitution
164 log.msg("Configured worker: {}".format(name))
165 # NB: phase1 build factory requires workers to be single-build only
166 c["workers"].append(Worker(name, password, max_builds=1, properties=sl_props))
169 for section in ini.sections():
170 if section.startswith("branch "):
171 ini_parse_branch(ini[section])
173 if section.startswith("worker "):
174 ini_parse_workers(ini[section])
176 # list of branches in build-priority order
177 branchNames = [branches[b]["name"] for b in branches]
179 c["protocols"] = {"pb": {"port": pb_port}}
182 c["collapseRequests"] = True
184 # Reduce amount of backlog data
185 c["configurators"] = [
186 util.JanitorConfigurator(
187 logHorizon=timedelta(days=3),
193 @defer.inlineCallbacks
194 def getNewestCompleteTime(bldr):
195 """Returns the complete_at of the latest completed and not SKIPPED
196 build request for this builder, or None if there are no such build
197 requests. We need to filter out SKIPPED requests because we're
198 using collapseRequests=True which is unfortunately marking all
199 previous requests as complete when new buildset is created.
201 @returns: datetime instance or None, via Deferred
204 bldrid = yield bldr.getBuilderId()
205 completed = yield bldr.master.data.get(
206 ("builders", bldrid, "buildrequests"),
208 resultspec.Filter("complete", "eq", [True]),
209 resultspec.Filter("results", "ne", [results.SKIPPED]),
211 order=["-complete_at"],
217 complete_at = completed[0]["complete_at"]
219 last_build = yield bldr.master.data.get(
222 resultspec.Filter("builderid", "eq", [bldrid]),
224 order=["-started_at"],
228 if last_build and last_build[0]:
229 last_complete_at = last_build[0]["complete_at"]
230 if last_complete_at and (last_complete_at > complete_at):
231 return last_complete_at
236 @defer.inlineCallbacks
237 def prioritizeBuilders(master, builders):
238 """Returns sorted list of builders by their last timestamp of completed and
239 not skipped build, ordered first by branch name.
241 @returns: list of sorted builders
244 bldrNamePrio = {"__Janitor": 0, "00_force_build": 0}
246 for bname in branchNames:
247 bldrNamePrio[bname] = i
250 def is_building(bldr):
251 return bool(bldr.building) or bool(bldr.old_building)
254 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
255 d.addCallback(lambda complete_at: (complete_at, bldr))
259 (complete_at, bldr) = item
262 for name, prio in bldrNamePrio.items():
263 if bldr.name.startswith(name):
269 complete_at = date.replace(tzinfo=tzutc())
271 if is_building(bldr):
273 complete_at = date.replace(tzinfo=tzutc())
275 return (pos, complete_at, bldr.name)
277 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
278 results.sort(key=bldr_sort)
281 # log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
283 return [r[1] for r in results]
286 c["prioritizeBuilders"] = prioritizeBuilders
288 ####### CHANGESOURCES
294 def populateTargets():
295 """fetch a shallow clone of each configured branch in turn:
296 execute dump-target-info.pl and collate the results to ensure
297 targets that only exist in specific branches get built.
298 This takes a while during master startup but is executed only once.
300 sourcegit = work_dir + "/source.git"
301 for branch in branchNames:
302 log.msg(f"Populating targets for {branch}, this will take time")
304 if os.path.isdir(sourcegit):
305 subprocess.call(["rm", "-rf", sourcegit])
313 "--branch=" + branch,
319 os.makedirs(sourcegit + "/tmp", exist_ok=True)
320 findtargets = subprocess.Popen(
321 ["./scripts/dump-target-info.pl", "targets"],
322 stdout=subprocess.PIPE,
323 stderr=subprocess.DEVNULL,
327 targets[branch] = set()
329 line = findtargets.stdout.readline()
332 ta = line.decode().strip().split(" ")
333 targets[branch].add(ta[0])
335 subprocess.call(["rm", "-rf", sourcegit])
340 # the 'change_source' setting tells the buildmaster how it should find out
341 # about source code changes.
343 c["change_source"] = []
344 c["change_source"].append(
347 workdir=work_dir + "/work.git",
348 branches=branchNames,
356 # Configure the Schedulers, which decide how to react to incoming changes.
359 # Selector for known valid tags
360 class TagChoiceParameter(BaseParameter):
361 spec_attributes = ["strict", "choices"]
365 def __init__(self, name, label=None, **kw):
366 super().__init__(name, label, **kw)
367 self._choice_list = []
369 def getRevTags(self, findtag=None):
373 # we will filter out tags that do no match the configured branches
374 for b in branchNames:
375 basever = re.search(r"-([0-9]+\.[0-9]+)$", b)
377 branchvers.append(basever[1])
379 # grab tags from remote repository
380 alltags = subprocess.Popen(
381 ["git", "ls-remote", "--tags", repo_url], stdout=subprocess.PIPE
385 line = alltags.stdout.readline()
390 (rev, tag) = line.split()
392 # does it match known format? ('vNN.NN.NN(-rcN)')
394 r"\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$",
395 tag.decode().strip(),
398 # only list valid tags matching configured branches
399 if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
400 # if we want a specific tag, ignore all that don't match
401 if findtag and findtag != tagver[1]:
403 taglist.append({"rev": rev.decode().strip(), "tag": tagver[1]})
409 taglist = [rt["tag"] for rt in self.getRevTags()]
412 key=lambda tag: tag if re.search(r"-rc[0-9]+$", tag) else tag + "-z",
414 taglist.insert(0, "")
416 self._choice_list = taglist
418 return self._choice_list
420 def updateFromKwargs(self, properties, kwargs, **unused):
421 tag = self.getFromKwargs(kwargs)
422 properties[self.name] = tag
424 # find the commit matching the tag
425 findtag = self.getRevTags(tag)
428 raise ValidationError("Couldn't find tag")
430 properties["force_revision"] = findtag[0]["rev"]
432 # find the branch matching the tag
434 branchver = re.search(r"v([0-9]+\.[0-9]+)", tag)
435 for b in branchNames:
436 if b.endswith(branchver[1]):
440 raise ValidationError("Couldn't find branch")
442 properties["force_branch"] = branch
444 def parse_from_arg(self, s):
445 if self.strict and s not in self._choice_list:
446 raise ValidationError(
447 "'%s' does not belong to list of available choices '%s'"
448 % (s, self._choice_list)
454 @defer.inlineCallbacks
455 def builderNames(props):
456 """since we have per branch and per target builders,
457 address the relevant builder for each new buildrequest
458 based on the request's desired branch and target.
460 branch = props.getProperty("branch")
461 target = props.getProperty("target", "")
466 # if that didn't work, try sourcestamp to find a branch
468 # match builders with target branch
469 ss = props.sourcestamps[0]
471 branch = ss["branch"]
473 log.msg("couldn't find builder")
474 return [] # nothing works
476 bname = branch + "_" + target
479 for b in (yield props.master.data.get(("builders",))):
480 if not b["name"].startswith(bname):
482 builders.append(b["name"])
488 c["schedulers"].append(
491 change_filter=util.ChangeFilter(branch=branchNames),
492 treeStableTimer=15 * 60,
493 builderNames=builderNames,
497 c["schedulers"].append(
500 buttonName="Force builds",
501 label="Force build details",
502 builderNames=["00_force_build"],
504 util.CodebaseParameter(
507 branch=util.FixedParameter(name="branch", default=""),
508 revision=util.FixedParameter(name="revision", default=""),
509 repository=util.FixedParameter(name="repository", default=""),
510 project=util.FixedParameter(name="project", default=""),
513 reason=util.StringParameter(
516 default="Trigger build",
521 # NB: avoid nesting to simplify processing of properties
522 util.ChoiceStringParameter(
524 label="Build target",
526 choices=["all"] + [t for b in branchNames for t in targets[b]],
528 TagChoiceParameter(name="tag", label="Build tag", default=""),
533 c["schedulers"].append(
534 schedulers.Triggerable(name="trigger", builderNames=builderNames)
539 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
540 # what steps, and which workers can execute them. Note that any particular build will
541 # only take place on one worker.
544 def IsNoMasterBuild(step):
545 return step.getProperty("branch") != "master"
548 def IsUsignEnabled(step):
549 branch = step.getProperty("branch")
550 return branch and branches[branch].get("usign_key")
553 def IsSignEnabled(step):
554 branch = step.getProperty("branch")
555 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
558 def IsKmodArchiveEnabled(step):
559 branch = step.getProperty("branch")
560 return branch and branches[branch].get("kmod_archive")
563 def IsKmodArchiveAndRsyncEnabled(step):
564 branch = step.getProperty("branch")
565 return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url"))
568 def GetBaseVersion(branch):
569 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
570 return branch.split("-")[1]
576 def GetVersionPrefix(props):
577 branch = props.getProperty("branch")
578 basever = GetBaseVersion(branch)
579 if props.hasProperty("tag") and re.match(
580 r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]
582 return "%s/" % props["tag"][1:]
583 elif basever != "master":
584 return "%s-SNAPSHOT/" % basever
590 def GetConfigSeed(props):
591 branch = props.getProperty("branch")
592 return branch and branches[branch].get("config_seed") or ""
596 def GetRsyncParams(props, srcorbin, urlorkey):
597 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
598 branch = props.getProperty("branch")
599 opt = srcorbin + "_" + urlorkey
600 return branch and branches[branch].get(opt)
604 def GetUsignKey(props):
605 branch = props.getProperty("branch")
606 return branch and branches[branch].get("usign_key")
609 def GetNextBuild(builder, requests):
612 # order tagged build first
613 if r.properties.hasProperty("tag"):
617 # log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
621 def MakeEnv(overrides=None, tryccache=False):
623 "CCC": Interpolate("%(prop:cc_command:-gcc)s"),
624 "CCXX": Interpolate("%(prop:cxx_command:-g++)s"),
627 env["CC"] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
628 env["CXX"] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
629 env["CCACHE"] = Interpolate("%(prop:ccache_command:-)s")
631 env["CC"] = env["CCC"]
632 env["CXX"] = env["CCXX"]
634 if overrides is not None:
635 env.update(overrides)
640 def NetLockDl(props, extralock=None):
642 if props.hasProperty("dl_lock"):
643 lock = NetLocks[props["dl_lock"]]
645 return [lock.access("exclusive")]
651 def NetLockUl(props):
653 if props.hasProperty("ul_lock"):
654 lock = NetLocks[props["ul_lock"]]
656 return [lock.access("exclusive")]
661 def IsTargetSelected(target):
662 def CheckTargetProperty(step):
663 selected_target = step.getProperty("target", "all")
664 if selected_target != "all" and selected_target != target:
668 return CheckTargetProperty
672 def UsignSec2Pub(props):
673 branch = props.getProperty("branch")
676 branches[branch].get("usign_comment") or "untrusted comment: secret key"
678 seckey = branches[branch].get("usign_key")
679 seckey = base64.b64decode(seckey)
683 return "{}\n{}".format(
684 re.sub(r"\bsecret key$", "public key", comment),
685 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]),
689 def canStartBuild(builder, wfb, request):
690 """filter out non tag requests for tag_only workers."""
691 wtagonly = wfb.worker.properties.getProperty("tag_only")
692 tag = request.properties.getProperty("tag")
694 if wtagonly and not tag:
704 for worker in c["workers"]:
705 workerNames.append(worker.workername)
707 # add a single LocalWorker to handle the forcebuild builder
708 c["workers"].append(LocalWorker("__local_force_build", max_builds=1))
710 force_factory = BuildFactory()
711 force_factory.addStep(
713 name="trigger_build",
714 schedulerNames=["trigger"],
718 "branch": Property("force_branch"),
719 "revision": Property("force_revision"),
720 "repository": repo_url,
725 "reason": Property("reason"),
726 "tag": Property("tag"),
727 "target": Property("target"),
732 c["builders"].append(
734 name="00_force_build", workername="__local_force_build", factory=force_factory
739 # NB the phase1 build factory assumes workers are single-build only
740 def prepareFactory(target):
741 ts = target.split("/")
743 factory = BuildFactory()
745 # setup shared work directory if required
749 descriptionDone="Shared work directory set up",
750 command='test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
756 # find number of cores
758 SetPropertyFromCommand(
761 description="Finding number of CPUs",
766 # find gcc and g++ compilers
770 mastersrc=scripts_dir + "/findbin.pl",
771 workerdest="../findbin.pl",
777 SetPropertyFromCommand(
779 property="cc_command",
780 description="Finding gcc command",
781 command=["../findbin.pl", "gcc", "", ""],
787 SetPropertyFromCommand(
789 property="cxx_command",
790 description="Finding g++ command",
791 command=["../findbin.pl", "g++", "", ""],
796 # see if ccache is available
798 SetPropertyFromCommand(
800 property="ccache_command",
801 description="Testing for ccache command",
802 command=["which", "ccache"],
804 flunkOnFailure=False,
806 hideStepIf=lambda r, s: r == results.FAILURE,
810 # check out the source
812 # if repo doesn't exist: 'git clone repourl'
813 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
814 # git cat-file -e <commit>
815 # git checkout -f <commit>
816 # git checkout -B <branch>
829 # workaround for https://github.com/openwrt/buildbot/issues/5
832 name="git me once more please",
845 description="Fetching Git remote refs",
846 descriptionDone="Git remote refs fetched",
852 "+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s"
859 # getver.sh requires local branches to track upstream otherwise version computation fails.
860 # Git() does not set tracking branches when cloning or switching, so work around this here
863 name="trackupstream",
864 description="Setting upstream branch",
865 descriptionDone="getver.sh is happy now",
866 command=["git", "branch", "-u", Interpolate("origin/%(prop:branch)s")],
871 # Verify that Git HEAD points to a tag or branch
872 # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
876 description="Ensuring that Git HEAD is pointing to a branch or tag",
877 descriptionDone="Git HEAD is sane",
878 command='git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
886 s='#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
887 workerdest="../ccache_cc.sh",
895 s='#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
896 workerdest="../ccache_cxx.sh",
905 description="Updating feeds",
906 command=["./scripts/feeds", "update"],
907 env=MakeEnv(tryccache=True),
917 description="Installing feeds",
918 command=["./scripts/feeds", "install", "-a"],
919 env=MakeEnv(tryccache=True),
928 s=Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
929 workerdest=".config",
938 descriptionDone=".config seeded",
940 "printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config",
951 description="Populating .config",
952 command=["make", "defconfig"],
957 # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
961 description="Checking architecture",
962 descriptionDone="Architecture validated",
963 command='grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config'
969 flunkOnFailure=False, # this is not a build FAILURE - TODO mark build as SKIPPED
975 SetPropertyFromCommand(
978 description="Finding libc suffix",
982 '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }',
991 name="dlkeybuildpub",
992 s=Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
993 workerdest="key-build.pub",
995 doStepIf=IsUsignEnabled,
1002 s="# fake private key",
1003 workerdest="key-build",
1005 doStepIf=IsUsignEnabled,
1011 name="dlkeybuilducert",
1012 s="# fake certificate",
1013 workerdest="key-build.ucert",
1015 doStepIf=IsUsignEnabled,
1023 description="Preparing dl/",
1024 descriptionDone="dl/ prepared",
1025 command='mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
1026 workdir=Property("builddir"),
1036 description="Pruning dl/",
1037 descriptionDone="dl/ pruned",
1038 command="find dl/ -mindepth 1 -atime +15 -delete -print",
1047 description="Building and installing GNU tar",
1048 descriptionDone="GNU tar built and installed",
1051 Interpolate("-j%(prop:nproc:-1)s"),
1052 "tools/tar/compile",
1055 env=MakeEnv(tryccache=True),
1064 description="Populating dl/",
1065 descriptionDone="dl/ populated",
1066 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
1076 description="Cleaning base-files",
1077 command=["make", "package/base-files/clean", "V=s"],
1085 description="Building and installing tools",
1086 descriptionDone="Tools built and installed",
1089 Interpolate("-j%(prop:nproc:-1)s"),
1093 env=MakeEnv(tryccache=True),
1101 description="Building and installing toolchain",
1102 descriptionDone="Toolchain built and installed",
1105 Interpolate("-j%(prop:nproc:-1)s"),
1106 "toolchain/install",
1117 description="Building kmods",
1118 descriptionDone="Kmods built",
1121 Interpolate("-j%(prop:nproc:-1)s"),
1124 "IGNORE_ERRORS=n m",
1132 # find kernel version
1134 SetPropertyFromCommand(
1135 name="kernelversion",
1136 property="kernelversion",
1137 description="Finding the effective Kernel version",
1138 command="make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
1139 env={"TOPDIR": Interpolate("%(prop:builddir)s/build")},
1146 description="Cleaning up package build",
1147 descriptionDone="Package build cleaned up",
1148 command=["make", "package/cleanup", "V=s"],
1155 description="Building packages",
1156 descriptionDone="Packages built",
1159 Interpolate("-j%(prop:nproc:-1)s"),
1162 "IGNORE_ERRORS=n m",
1173 description="Installing packages",
1174 descriptionDone="Packages installed",
1177 Interpolate("-j%(prop:nproc:-1)s"),
1189 description="Indexing packages",
1190 descriptionDone="Packages indexed",
1193 Interpolate("-j%(prop:nproc:-1)s"),
1196 "CONFIG_SIGNED_PACKAGES=",
1206 description="Building and installing images",
1207 descriptionDone="Images built and installed",
1210 Interpolate("-j%(prop:nproc:-1)s"),
1222 description="Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
1223 command="make -j1 buildinfo V=s || true",
1231 name="json_overview_image_info",
1232 description="Generating profiles.json in target folder",
1233 command="make -j1 json_overview_image_info V=s || true",
1242 description="Calculating checksums",
1243 descriptionDone="Checksums calculated",
1244 command=["make", "-j1", "checksum", "V=s"],
1253 descriptionDone="Kmod directory created",
1258 "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s",
1264 doStepIf=IsKmodArchiveEnabled,
1271 description="Preparing kmod archive",
1272 descriptionDone="Kmod archive prepared",
1275 "--include=/kmod-*.ipk",
1279 "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/",
1284 "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1290 doStepIf=IsKmodArchiveEnabled,
1297 description="Indexing kmod archive",
1298 descriptionDone="Kmod archive indexed",
1301 Interpolate("-j%(prop:nproc:-1)s"),
1304 "CONFIG_SIGNED_PACKAGES=",
1306 "PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1313 doStepIf=IsKmodArchiveEnabled,
1321 descriptionDone="Temporary signing directory prepared",
1322 command=["mkdir", "-p", "%s/signing" % (work_dir)],
1324 doStepIf=IsSignEnabled,
1331 description="Packing files to sign",
1332 descriptionDone="Files to sign packed",
1333 command=Interpolate(
1334 "find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz",
1339 doStepIf=IsSignEnabled,
1345 workersrc="sign.tar.gz",
1346 masterdest="%s/signing/%s.%s.tar.gz" % (work_dir, ts[0], ts[1]),
1348 doStepIf=IsSignEnabled,
1355 description="Signing files",
1356 descriptionDone="Files signed",
1358 "%s/signall.sh" % (scripts_dir),
1359 "%s/signing/%s.%s.tar.gz" % (work_dir, ts[0], ts[1]),
1360 Interpolate("%(prop:branch)s"),
1362 env={"CONFIG_INI": os.getenv("BUILDMASTER_CONFIG", "./config.ini")},
1364 doStepIf=IsSignEnabled,
1371 mastersrc="%s/signing/%s.%s.tar.gz" % (work_dir, ts[0], ts[1]),
1372 workerdest="sign.tar.gz",
1374 doStepIf=IsSignEnabled,
1381 description="Unpacking signed files",
1382 descriptionDone="Signed files unpacked",
1383 command=["tar", "-xzf", "sign.tar.gz"],
1385 doStepIf=IsSignEnabled,
1393 descriptionDone="Upload directory structure prepared",
1398 "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s",
1401 prefix=GetVersionPrefix,
1411 descriptionDone="Repository symlink prepared",
1417 "../packages-%(kw:basever)s",
1418 basever=util.Transform(GetBaseVersion, Property("branch")),
1421 "tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix
1424 doStepIf=IsNoMasterBuild,
1431 name="kmoddirprepare",
1432 descriptionDone="Kmod archive upload directory prepared",
1437 "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s",
1440 prefix=GetVersionPrefix,
1444 doStepIf=IsKmodArchiveEnabled,
1451 description="Uploading directory structure",
1452 descriptionDone="Directory structure uploaded",
1453 command=["rsync", Interpolate("-az%(prop:rsync_ipv4:+4)s")]
1457 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url")),
1460 "RSYNC_PASSWORD": Interpolate(
1461 "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1467 doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1471 # download remote sha256sums to 'target-sha256sums'
1474 name="target-sha256sums",
1475 description="Fetching remote sha256sums for target",
1476 descriptionDone="Remote sha256sums for target fetched",
1477 command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
1481 "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
1482 url=GetRsyncParams.withArgs("bin", "url"),
1485 prefix=GetVersionPrefix,
1487 "target-sha256sums",
1490 "RSYNC_PASSWORD": Interpolate(
1491 "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1495 haltOnFailure=False,
1496 flunkOnFailure=False,
1497 warnOnFailure=False,
1498 doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1502 # build list of files to upload
1505 name="dlsha2rsyncpl",
1506 mastersrc=scripts_dir + "/sha2rsync.pl",
1507 workerdest="../sha2rsync.pl",
1515 description="Building list of files to upload",
1516 descriptionDone="List of files to upload built",
1519 "target-sha256sums",
1521 "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
1534 mastersrc=scripts_dir + "/rsync.sh",
1535 workerdest="../rsync.sh",
1540 # upload new files and update existing ones
1543 name="targetupload",
1544 description="Uploading target files",
1545 descriptionDone="Target files uploaded",
1548 "--exclude=/kmods/",
1549 "--files-from=rsynclist",
1551 "--partial-dir=.~tmp~%s~%s" % (ts[0], ts[1]),
1555 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1557 "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1562 "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1563 url=GetRsyncParams.withArgs("bin", "url"),
1566 prefix=GetVersionPrefix,
1570 "RSYNC_PASSWORD": Interpolate(
1571 "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1576 doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1580 # delete files which don't exist locally
1584 description="Pruning target files",
1585 descriptionDone="Target files pruned",
1588 "--exclude=/kmods/",
1591 "--ignore-existing",
1593 "--partial-dir=.~tmp~%s~%s" % (ts[0], ts[1]),
1597 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1599 "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1604 "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1605 url=GetRsyncParams.withArgs("bin", "url"),
1608 prefix=GetVersionPrefix,
1612 "RSYNC_PASSWORD": Interpolate(
1613 "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1619 doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1626 description="Uploading kmod archive",
1627 descriptionDone="Kmod archive uploaded",
1632 "--partial-dir=.~tmp~%s~%s" % (ts[0], ts[1]),
1636 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1638 "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1643 "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/",
1644 url=GetRsyncParams.withArgs("bin", "url"),
1647 prefix=GetVersionPrefix,
1651 "RSYNC_PASSWORD": Interpolate(
1652 "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1658 doStepIf=IsKmodArchiveAndRsyncEnabled,
1665 description="Finding source archives to upload",
1666 descriptionDone="Source archives to upload found",
1667 command="find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1674 name="sourceupload",
1675 description="Uploading source archives",
1676 descriptionDone="Source archives uploaded",
1679 "--files-from=sourcelist",
1686 "--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s",
1690 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1692 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url")),
1695 "RSYNC_PASSWORD": Interpolate(
1696 "%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")
1702 doStepIf=util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1709 description="Reporting disk usage",
1710 command=["df", "-h", "."],
1711 env={"LC_ALL": "C"},
1713 haltOnFailure=False,
1714 flunkOnFailure=False,
1715 warnOnFailure=False,
1723 description="Reporting estimated file space usage",
1724 command=["du", "-sh", "."],
1725 env={"LC_ALL": "C"},
1727 haltOnFailure=False,
1728 flunkOnFailure=False,
1729 warnOnFailure=False,
1737 description="Reporting ccache stats",
1738 command=["ccache", "-s"],
1741 haltOnFailure=False,
1742 flunkOnFailure=False,
1743 warnOnFailure=False,
1744 doStepIf=util.Transform(bool, Property("ccache_command")),
1751 for brname in branchNames:
1752 for target in targets[brname]:
1753 bldrname = brname + "_" + target
1754 c["builders"].append(
1757 workernames=workerNames,
1758 factory=prepareFactory(target),
1762 nextBuild=GetNextBuild,
1763 canStartBuild=canStartBuild,
1768 ####### STATUS TARGETS
1770 # 'status' is a list of Status Targets. The results of each build will be
1771 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1772 # including web pages, email senders, and IRC bots.
1774 if "status_bind" in inip1:
1776 "port": inip1.get("status_bind"),
1777 "plugins": {"waterfall_view": True, "console_view": True, "grid_view": True},
1780 if "status_user" in inip1 and "status_password" in inip1:
1781 c["www"]["auth"] = util.UserPasswordAuth(
1782 [(inip1.get("status_user"), inip1.get("status_password"))]
1784 c["www"]["authz"] = util.Authz(
1785 allowRules=[util.AnyControlEndpointMatcher(role="admins")],
1787 util.RolesFromUsername(
1788 roles=["admins"], usernames=[inip1.get("status_user")]
1794 if ini.has_section("irc"):
1796 irc_host = iniirc.get("host", None)
1797 irc_port = iniirc.getint("port", 6667)
1798 irc_chan = iniirc.get("channel", None)
1799 irc_nick = iniirc.get("nickname", None)
1800 irc_pass = iniirc.get("password", None)
1802 if irc_host and irc_nick and irc_chan:
1803 irc = reporters.IRC(
1808 channels=[irc_chan],
1809 notify_events=["exception", "problem", "recovery"],
1812 c["services"].append(irc)
1814 c["revlink"] = util.RevlinkMatch(
1815 [r"https://git.openwrt.org/openwrt/(.*).git"],
1816 r"https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s",
1822 # This specifies what database buildbot uses to store its state. You can leave
1823 # this at its default for all but the largest installations.
1824 "db_url": "sqlite:///state.sqlite",
1827 c["buildbotNetUsageData"] = None