2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes import filter
19 from buildbot.changes.gitpoller import GitPoller
20 from buildbot.config import BuilderConfig
21 from buildbot.plugins import reporters
22 from buildbot.plugins import schedulers
23 from buildbot.plugins import steps
24 from buildbot.plugins import util
25 from buildbot.process import properties
26 from buildbot.process import results
27 from buildbot.process.factory import BuildFactory
28 from buildbot.process.properties import Interpolate
29 from buildbot.process.properties import Property
30 from buildbot.schedulers.basic import SingleBranchScheduler
31 from buildbot.schedulers.forcesched import BaseParameter
32 from buildbot.schedulers.forcesched import ForceScheduler
33 from buildbot.schedulers.forcesched import ValidationError
34 from buildbot.steps.master import MasterShellCommand
35 from buildbot.steps.shell import SetPropertyFromCommand
36 from buildbot.steps.shell import ShellCommand
37 from buildbot.steps.source.git import Git
38 from buildbot.steps.transfer import FileDownload
39 from buildbot.steps.transfer import FileUpload
40 from buildbot.steps.transfer import StringDownload
41 from buildbot.worker import Worker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini['repo'].get("url")
64 repo_branch = ini['repo'].get("branch", "master")
66 rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
68 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
69 # rsync_bin_defopts += ["--contimeout=20"]
71 rsync_src_defopts = ["-v", "-4", "--timeout=120"]
73 #if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
74 # rsync_src_defopts += ["--contimeout=20"]
78 def ini_parse_branch(section):
80 name = section.get("name")
83 raise ValueError("missing 'name' in " + repr(section))
85 raise ValueError("duplicate branch name in " + repr(section))
88 b["bin_url"] = section.get("binary_url")
89 b["bin_key"] = section.get("binary_password")
91 b["src_url"] = section.get("source_url")
92 b["src_key"] = section.get("source_password")
94 b["gpg_key"] = section.get("gpg_key")
96 b["usign_key"] = section.get("usign_key")
97 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
98 b["usign_comment"] = section.get("usign_comment", usign_comment)
100 b["config_seed"] = section.get("config_seed")
102 b["kmod_archive"] = section.getboolean("kmod_archive", False)
105 log.msg("Configured branch: {}".format(name))
107 # PB port can be either a numeric port or a connection string
108 pb_port = inip1.get("port") or 9989
110 # This is the dictionary that the buildmaster pays attention to. We also use
111 # a shorter alias to save typing.
112 c = BuildmasterConfig = {}
114 ####### PROJECT IDENTITY
116 # the 'title' string will appear at the top of this buildbot
117 # installation's html.WebStatus home page (linked to the
118 # 'titleURL') and is embedded in the title of the waterfall HTML page.
120 c['title'] = ini['general'].get("title")
121 c['titleURL'] = ini['general'].get("title_url")
123 # the 'buildbotURL' string should point to the location where the buildbot's
124 # internal web server (usually the html.WebStatus page) is visible. This
125 # typically uses the port number set in the Waterfall 'status' entry, but
126 # with an externally-visible host name which the buildbot cannot figure out
129 c['buildbotURL'] = inip1.get("buildbot_url")
133 # The 'workers' list defines the set of recognized buildworkers. Each element is
134 # a Worker object, specifying a unique worker name and password. The same
135 # worker name and password must be configured on the worker.
140 for section in ini.sections():
141 if section.startswith("branch "):
142 ini_parse_branch(ini[section])
144 if section.startswith("worker "):
145 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
146 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
147 sl_props = { 'dl_lock':None, 'ul_lock':None }
148 name = ini.get(section, "name")
149 password = ini.get(section, "password")
150 if ini.has_option(section, "dl_lock"):
151 lockname = ini.get(section, "dl_lock")
152 sl_props['dl_lock'] = lockname
153 if lockname not in NetLocks:
154 NetLocks[lockname] = locks.MasterLock(lockname)
155 if ini.has_option(section, "ul_lock"):
156 lockname = ini.get(section, "ul_lock")
157 sl_props['ul_lock'] = lockname
158 if lockname not in NetLocks:
159 NetLocks[lockname] = locks.MasterLock(lockname)
160 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
162 c['protocols'] = {'pb': {'port': pb_port}}
165 c['collapseRequests'] = True
167 # Reduce amount of backlog data
168 c['configurators'] = [util.JanitorConfigurator(
169 logHorizon=timedelta(days=3),
173 @defer.inlineCallbacks
174 def getNewestCompleteTime(bldr):
175 """Returns the complete_at of the latest completed and not SKIPPED
176 build request for this builder, or None if there are no such build
177 requests. We need to filter out SKIPPED requests because we're
178 using collapseRequests=True which is unfortunately marking all
179 previous requests as complete when new buildset is created.
181 @returns: datetime instance or None, via Deferred
184 bldrid = yield bldr.getBuilderId()
185 completed = yield bldr.master.data.get(
186 ('builders', bldrid, 'buildrequests'),
188 resultspec.Filter('complete', 'eq', [True]),
189 resultspec.Filter('results', 'ne', [results.SKIPPED]),
191 order=['-complete_at'], limit=1)
195 complete_at = completed[0]['complete_at']
197 last_build = yield bldr.master.data.get(
200 resultspec.Filter('builderid', 'eq', [bldrid]),
202 order=['-started_at'], limit=1)
204 if last_build and last_build[0]:
205 last_complete_at = last_build[0]['complete_at']
206 if last_complete_at and (last_complete_at > complete_at):
207 return last_complete_at
211 @defer.inlineCallbacks
212 def prioritizeBuilders(master, builders):
213 """Returns sorted list of builders by their last timestamp of completed and
216 @returns: list of sorted builders
219 def is_building(bldr):
220 return bool(bldr.building) or bool(bldr.old_building)
223 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
224 d.addCallback(lambda complete_at: (complete_at, bldr))
228 (complete_at, bldr) = item
232 complete_at = date.replace(tzinfo=tzutc())
234 if is_building(bldr):
236 complete_at = date.replace(tzinfo=tzutc())
238 return (complete_at, bldr.name)
240 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
241 results.sort(key=bldr_sort)
244 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
246 return [r[1] for r in results]
248 c['prioritizeBuilders'] = prioritizeBuilders
250 ####### CHANGESOURCES
256 def populateTargets():
257 sourcegit = work_dir + '/source.git'
258 if os.path.isdir(sourcegit):
259 subprocess.call(["rm", "-rf", sourcegit])
261 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, sourcegit])
263 os.makedirs(sourcegit + '/tmp', exist_ok=True)
264 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
265 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
268 line = findtargets.stdout.readline()
271 ta = line.decode().strip().split(' ')
272 targets.append(ta[0])
274 subprocess.call(["rm", "-rf", sourcegit])
278 # the 'change_source' setting tells the buildmaster how it should find out
279 # about source code changes. Here we point to the buildbot clone of pyflakes.
281 c['change_source'] = []
282 c['change_source'].append(GitPoller(
284 workdir=work_dir+'/work.git', branch=repo_branch,
289 # Configure the Schedulers, which decide how to react to incoming changes. In this
290 # case, just kick off a 'basebuild' build
292 class TagChoiceParameter(BaseParameter):
293 spec_attributes = ["strict", "choices"]
297 def __init__(self, name, label=None, **kw):
298 super().__init__(name, label, **kw)
299 self._choice_list = []
304 basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch)
307 findtags = subprocess.Popen(
308 ['git', 'ls-remote', '--tags', repo_url],
309 stdout = subprocess.PIPE)
312 line = findtags.stdout.readline()
317 tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
319 if tagver and tagver[1].find(basever[1]) == 0:
320 taglist.append(tagver[1])
322 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
323 taglist.insert(0, '')
325 self._choice_list = taglist
327 return self._choice_list
329 def parse_from_arg(self, s):
330 if self.strict and s not in self._choice_list:
331 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
335 c['schedulers'].append(SingleBranchScheduler(
337 change_filter = filter.ChangeFilter(branch=repo_branch),
338 treeStableTimer = 60,
339 builderNames = targets))
341 c['schedulers'].append(ForceScheduler(
343 buttonName = "Force builds",
344 label = "Force build details",
345 builderNames = [ "00_force_build" ],
348 util.CodebaseParameter(
350 label = "Repository",
351 branch = util.FixedParameter(name = "branch", default = ""),
352 revision = util.FixedParameter(name = "revision", default = ""),
353 repository = util.FixedParameter(name = "repository", default = ""),
354 project = util.FixedParameter(name = "project", default = "")
358 reason = util.StringParameter(
361 default = "Trigger build",
367 util.NestedParameter(
369 label="Build Options",
372 util.ChoiceStringParameter(
374 label = "Build target",
376 choices = [ "all" ] + targets
390 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
391 # what steps, and which workers can execute them. Note that any particular build will
392 # only take place on one worker.
394 def IsTaggingRequested(step):
395 tag = step.getProperty("tag")
396 return tag and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", tag)
398 def IsNoMasterBuild(step):
399 return step.getProperty("branch") != "master"
401 def IsUsignEnabled(step):
402 branch = step.getProperty("branch")
403 return branch and branches[branch].get("usign_key")
405 def IsSignEnabled(step):
406 branch = step.getProperty("branch")
407 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
409 def IsKmodArchiveEnabled(step):
410 branch = step.getProperty("branch")
411 return branch and branches[branch].get("kmod_archive")
413 def GetBaseVersion(branch):
414 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
415 return branch.split('-')[1]
420 def GetVersionPrefix(props):
421 branch = props.getProperty("branch")
422 basever = GetBaseVersion(branch)
423 if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
424 return "%s/" % props["tag"]
425 elif basever != "master":
426 return "%s-SNAPSHOT/" % basever
431 def GetConfigSeed(props):
432 branch = props.getProperty("branch")
433 return branch and branches[branch].get("config_seed") or ""
436 def GetRsyncParams(props, srcorbin, urlorkey):
437 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
438 branch = props.getProperty("branch")
439 opt = srcorbin + "_" + urlorkey
440 return branch and branches[branch].get(opt)
443 def GetUsignKey(props):
444 branch = props.getProperty("branch")
445 return branch and branches[branch].get("usign_key")
447 def GetNextBuild(builder, requests):
449 if r.properties and r.properties.hasProperty("tag"):
453 log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
456 def MakeEnv(overrides=None, tryccache=False):
458 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
459 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
462 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
463 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
464 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
466 env['CC'] = env['CCC']
467 env['CXX'] = env['CCXX']
469 if overrides is not None:
470 env.update(overrides)
474 def NetLockDl(props):
476 if props.hasProperty("dl_lock"):
477 lock = NetLocks[props["dl_lock"]]
479 return [lock.access('exclusive')]
484 def NetLockUl(props):
486 if props.hasProperty("ul_lock"):
487 lock = NetLocks[props["ul_lock"]]
489 return [lock.access('exclusive')]
494 def TagPropertyValue(props):
495 if props.hasProperty("options"):
496 options = props.getProperty("options")
497 if type(options) is dict:
498 return options.get("tag")
501 def IsTargetSelected(target):
502 def CheckTargetProperty(step):
504 options = step.getProperty("options")
505 if type(options) is dict:
506 selected_target = options.get("target", "all")
507 if selected_target != "all" and selected_target != target:
514 return CheckTargetProperty
517 def UsignSec2Pub(props):
518 branch = props.getProperty("branch")
520 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
521 seckey = branches[branch].get("usign_key")
522 seckey = base64.b64decode(seckey)
526 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
527 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
532 dlLock = locks.WorkerLock("worker_dl")
536 for worker in c['workers']:
537 workerNames.append(worker.workername)
539 force_factory = BuildFactory()
541 c['builders'].append(BuilderConfig(
542 name = "00_force_build",
543 workernames = workerNames,
544 factory = force_factory))
546 for target in targets:
547 ts = target.split('/')
549 factory = BuildFactory()
551 # setup shared work directory if required
552 factory.addStep(ShellCommand(
554 description = "Setting up shared work directory",
555 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
557 haltOnFailure = True))
559 # find number of cores
560 factory.addStep(SetPropertyFromCommand(
563 description = "Finding number of CPUs",
564 command = ["nproc"]))
566 # find gcc and g++ compilers
567 factory.addStep(FileDownload(
568 name = "dlfindbinpl",
569 mastersrc = scripts_dir + '/findbin.pl',
570 workerdest = "../findbin.pl",
573 factory.addStep(SetPropertyFromCommand(
575 property = "cc_command",
576 description = "Finding gcc command",
578 "../findbin.pl", "gcc", "", "",
580 haltOnFailure = True))
582 factory.addStep(SetPropertyFromCommand(
584 property = "cxx_command",
585 description = "Finding g++ command",
587 "../findbin.pl", "g++", "", "",
589 haltOnFailure = True))
591 # see if ccache is available
592 factory.addStep(SetPropertyFromCommand(
593 property = "ccache_command",
594 command = ["which", "ccache"],
595 description = "Testing for ccache command",
596 haltOnFailure = False,
597 flunkOnFailure = False,
598 warnOnFailure = False,
601 # Workaround bug when switching from a checked out tag back to a branch
602 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
603 factory.addStep(ShellCommand(
604 name = "gitcheckout",
605 description = "Ensure that Git HEAD is sane",
606 command = Interpolate("if [ -d .git ]; then git checkout -f %(prop:branch)s && git branch --set-upstream-to origin/%(prop:branch)s || rm -fr .git; else exit 0; fi"),
607 haltOnFailure = True))
609 # check out the source
611 # if repo doesn't exist: 'git clone repourl'
612 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
613 # 'git fetch -t repourl branch; git reset --hard revision'
620 haltOnFailure = True,
624 factory.addStep(ShellCommand(
626 description = "Fetching Git remote refs",
627 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
632 factory.addStep(ShellCommand(
634 description = "Checking out Git tag",
635 command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
636 haltOnFailure = True,
637 doStepIf = IsTaggingRequested
640 # Verify that Git HEAD points to a tag or branch
641 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
642 factory.addStep(ShellCommand(
644 description = "Ensure that Git HEAD is pointing to a branch or tag",
645 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
646 haltOnFailure = True))
648 factory.addStep(ShellCommand(
650 description = "Remove tmp folder",
651 command=["rm", "-rf", "tmp/"]))
654 factory.addStep(ShellCommand(
655 name = "rmfeedlinks",
656 description = "Remove feed symlinks",
657 command=["rm", "-rf", "package/feeds/"]))
659 factory.addStep(StringDownload(
661 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
662 workerdest = "../ccache_cc.sh",
666 factory.addStep(StringDownload(
668 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
669 workerdest = "../ccache_cxx.sh",
674 factory.addStep(ShellCommand(
675 name = "updatefeeds",
676 description = "Updating feeds",
677 command=["./scripts/feeds", "update"],
678 env = MakeEnv(tryccache=True),
679 haltOnFailure = True,
684 factory.addStep(ShellCommand(
685 name = "installfeeds",
686 description = "Installing feeds",
687 command=["./scripts/feeds", "install", "-a"],
688 env = MakeEnv(tryccache=True),
693 factory.addStep(StringDownload(
694 name = "dlconfigseed",
695 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
696 workerdest = ".config",
701 factory.addStep(ShellCommand(
703 description = "Seeding .config",
704 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey)
707 factory.addStep(ShellCommand(
709 description = "Removing output directory",
710 command = ["rm", "-rf", "bin/"]
713 factory.addStep(ShellCommand(
715 description = "Populating .config",
716 command = ["make", "defconfig"],
721 factory.addStep(ShellCommand(
723 description = "Checking architecture",
724 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
732 factory.addStep(SetPropertyFromCommand(
735 description = "Finding libc suffix",
736 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
739 factory.addStep(StringDownload(
740 name = "dlkeybuildpub",
741 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
742 workerdest = "key-build.pub",
744 doStepIf = IsUsignEnabled,
747 factory.addStep(StringDownload(
749 s = "# fake private key",
750 workerdest = "key-build",
752 doStepIf = IsUsignEnabled,
755 factory.addStep(StringDownload(
756 name = "dlkeybuilducert",
757 s = "# fake certificate",
758 workerdest = "key-build.ucert",
760 doStepIf = IsUsignEnabled,
764 factory.addStep(ShellCommand(
766 description = "Preparing dl/",
767 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
773 factory.addStep(ShellCommand(
775 description = "Building and installing GNU tar",
776 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
777 env = MakeEnv(tryccache=True),
782 factory.addStep(ShellCommand(
784 description = "Populating dl/",
785 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
788 locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
791 factory.addStep(ShellCommand(
793 description = "Cleaning base-files",
794 command=["make", "package/base-files/clean", "V=s"]
798 factory.addStep(ShellCommand(
800 description = "Building and installing tools",
801 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
802 env = MakeEnv(tryccache=True),
806 factory.addStep(ShellCommand(
808 description = "Building and installing toolchain",
809 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
814 factory.addStep(ShellCommand(
816 description = "Building kmods",
817 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
822 # find kernel version
823 factory.addStep(SetPropertyFromCommand(
824 name = "kernelversion",
825 property = "kernelversion",
826 description = "Finding the effective Kernel version",
827 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
828 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") }
831 factory.addStep(ShellCommand(
833 description = "Cleaning up package build",
834 command=["make", "package/cleanup", "V=s"]
837 factory.addStep(ShellCommand(
839 description = "Building packages",
840 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
845 factory.addStep(ShellCommand(
847 description = "Installing packages",
848 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
853 factory.addStep(ShellCommand(
855 description = "Indexing packages",
856 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
861 factory.addStep(ShellCommand(
863 description = "Building and installing images",
864 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
869 factory.addStep(ShellCommand(
871 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
872 command = "make -j1 buildinfo V=s || true",
877 factory.addStep(ShellCommand(
878 name = "json_overview_image_info",
879 description = "Generate profiles.json in target folder",
880 command = "make -j1 json_overview_image_info V=s || true",
885 factory.addStep(ShellCommand(
887 description = "Calculating checksums",
888 command=["make", "-j1", "checksum", "V=s"],
893 factory.addStep(ShellCommand(
895 description = "Creating kmod directory",
896 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
897 haltOnFailure = True,
898 doStepIf = IsKmodArchiveEnabled,
901 factory.addStep(ShellCommand(
902 name = "kmodprepare",
903 description = "Preparing kmod archive",
904 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
905 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
906 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
907 haltOnFailure = True,
908 doStepIf = IsKmodArchiveEnabled,
911 factory.addStep(ShellCommand(
913 description = "Indexing kmod archive",
914 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
915 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
917 haltOnFailure = True,
918 doStepIf = IsKmodArchiveEnabled,
922 factory.addStep(MasterShellCommand(
923 name = "signprepare",
924 description = "Preparing temporary signing directory",
925 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
926 haltOnFailure = True,
927 doStepIf = IsSignEnabled,
931 factory.addStep(ShellCommand(
933 description = "Packing files to sign",
934 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
935 haltOnFailure = True,
936 doStepIf = IsSignEnabled,
939 factory.addStep(FileUpload(
940 workersrc = "sign.tar.gz",
941 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
942 haltOnFailure = True,
943 doStepIf = IsSignEnabled,
946 factory.addStep(MasterShellCommand(
948 description = "Signing files",
949 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
950 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
951 haltOnFailure = True,
952 doStepIf = IsSignEnabled,
955 factory.addStep(FileDownload(
956 name = "dlsigntargz",
957 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
958 workerdest = "sign.tar.gz",
959 haltOnFailure = True,
960 doStepIf = IsSignEnabled,
963 factory.addStep(ShellCommand(
965 description = "Unpacking signed files",
966 command = ["tar", "-xzf", "sign.tar.gz"],
967 haltOnFailure = True,
968 doStepIf = IsSignEnabled,
972 factory.addStep(ShellCommand(
974 description = "Preparing upload directory structure",
975 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
979 factory.addStep(ShellCommand(
980 name = "linkprepare",
981 description = "Preparing repository symlink",
982 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
983 doStepIf = IsNoMasterBuild,
987 factory.addStep(ShellCommand(
988 name = "kmoddirprepare",
989 description = "Preparing kmod archive upload directory",
990 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
991 haltOnFailure = True,
992 doStepIf = IsKmodArchiveEnabled,
995 factory.addStep(ShellCommand(
997 description = "Uploading directory structure",
998 command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
999 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1000 haltOnFailure = True,
1005 # download remote sha256sums to 'target-sha256sums'
1006 factory.addStep(ShellCommand(
1007 name = "target-sha256sums",
1008 description = "Fetching remote sha256sums for target",
1009 command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1010 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1012 haltOnFailure = False,
1013 flunkOnFailure = False,
1014 warnOnFailure = False,
1017 # build list of files to upload
1018 factory.addStep(FileDownload(
1019 name = "dlsha2rsyncpl",
1020 mastersrc = scripts_dir + '/sha2rsync.pl',
1021 workerdest = "../sha2rsync.pl",
1025 factory.addStep(ShellCommand(
1027 description = "Building list of files to upload",
1028 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1029 haltOnFailure = True,
1032 factory.addStep(FileDownload(
1033 name = "dlrsync.sh",
1034 mastersrc = scripts_dir + '/rsync.sh',
1035 workerdest = "../rsync.sh",
1039 # upload new files and update existing ones
1040 factory.addStep(ShellCommand(
1041 name = "targetupload",
1042 description = "Uploading target files",
1043 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1044 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1045 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1046 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1047 haltOnFailure = True,
1051 # delete files which don't exist locally
1052 factory.addStep(ShellCommand(
1053 name = "targetprune",
1054 description = "Pruning target files",
1055 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1056 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1057 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1058 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1059 haltOnFailure = True,
1064 factory.addStep(ShellCommand(
1065 name = "kmodupload",
1066 description = "Uploading kmod archive",
1067 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1068 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1069 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1070 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1071 haltOnFailure = True,
1074 doStepIf = IsKmodArchiveEnabled,
1077 factory.addStep(ShellCommand(
1078 name = "sourcelist",
1079 description = "Finding source archives to upload",
1080 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1081 haltOnFailure = True
1084 factory.addStep(ShellCommand(
1085 name = "sourceupload",
1086 description = "Uploading source archives",
1087 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
1088 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1089 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1090 haltOnFailure = True,
1095 factory.addStep(ShellCommand(
1097 description = "Reporting disk usage",
1098 command=["df", "-h", "."],
1099 env={'LC_ALL': 'C'},
1100 haltOnFailure = False,
1101 flunkOnFailure = False,
1102 warnOnFailure = False,
1106 factory.addStep(ShellCommand(
1108 description = "Reporting estimated file space usage",
1109 command=["du", "-sh", "."],
1110 env={'LC_ALL': 'C'},
1111 haltOnFailure = False,
1112 flunkOnFailure = False,
1113 warnOnFailure = False,
1117 factory.addStep(ShellCommand(
1118 name = "ccachestat",
1119 description = "Reporting ccache stats",
1120 command=["ccache", "-s"],
1121 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1122 want_stderr = False,
1123 haltOnFailure = False,
1124 flunkOnFailure = False,
1125 warnOnFailure = False,
1129 c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1131 c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1132 force_factory.addStep(steps.Trigger(
1133 name = "trigger_%s" % target,
1134 description = "Triggering %s build" % target,
1135 schedulerNames = [ "trigger_%s" % target ],
1136 set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
1137 doStepIf = IsTargetSelected(target)
1141 ####### STATUS TARGETS
1143 # 'status' is a list of Status Targets. The results of each build will be
1144 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1145 # including web pages, email senders, and IRC bots.
1147 if "status_bind" in inip1:
1149 'port': inip1.get("status_bind"),
1151 'waterfall_view': True,
1152 'console_view': True,
1157 if "status_user" in inip1 and "status_password" in inip1:
1158 c['www']['auth'] = util.UserPasswordAuth([
1159 (inip1.get("status_user"), inip1.get("status_password"))
1161 c['www']['authz'] = util.Authz(
1162 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1163 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1167 if ini.has_section("irc"):
1169 irc_host = iniirc.get("host", None)
1170 irc_port = iniirc.getint("port", 6667)
1171 irc_chan = iniirc.get("channel", None)
1172 irc_nick = iniirc.get("nickname", None)
1173 irc_pass = iniirc.get("password", None)
1175 if irc_host and irc_nick and irc_chan:
1176 irc = reporters.IRC(irc_host, irc_nick,
1178 password = irc_pass,
1179 channels = [ irc_chan ],
1180 notify_events = [ 'exception', 'problem', 'recovery' ]
1183 c['services'].append(irc)
1185 c['revlink'] = util.RevlinkMatch([
1186 r'https://git.openwrt.org/openwrt/(.*).git'
1188 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1193 # This specifies what database buildbot uses to store its state. You can leave
1194 # this at its default for all but the largest installations.
1195 'db_url' : "sqlite:///state.sqlite",
1198 c['buildbotNetUsageData'] = None