2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes import filter
19 from buildbot.changes.gitpoller import GitPoller
20 from buildbot.config import BuilderConfig
21 from buildbot.plugins import reporters
22 from buildbot.plugins import schedulers
23 from buildbot.plugins import steps
24 from buildbot.plugins import util
25 from buildbot.process import properties
26 from buildbot.process import results
27 from buildbot.process.factory import BuildFactory
28 from buildbot.process.properties import Interpolate
29 from buildbot.process.properties import Property
30 from buildbot.schedulers.basic import SingleBranchScheduler
31 from buildbot.schedulers.forcesched import BaseParameter
32 from buildbot.schedulers.forcesched import ForceScheduler
33 from buildbot.schedulers.forcesched import ValidationError
34 from buildbot.steps.master import MasterShellCommand
35 from buildbot.steps.shell import SetPropertyFromCommand
36 from buildbot.steps.shell import ShellCommand
37 from buildbot.steps.source.git import Git
38 from buildbot.steps.transfer import FileDownload
39 from buildbot.steps.transfer import FileUpload
40 from buildbot.steps.transfer import StringDownload
41 from buildbot.worker import Worker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini or "rsync" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini['repo'].get("url")
64 repo_branch = ini['repo'].get("branch", "master")
66 rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
68 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
69 # rsync_bin_defopts += ["--contimeout=20"]
71 rsync_src_defopts = ["-v", "-4", "--timeout=120"]
73 #if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
74 # rsync_src_defopts += ["--contimeout=20"]
78 def ini_parse_branch(section):
80 name = section.get("name")
83 raise ValueError("missing 'name' in " + repr(section))
85 raise ValueError("duplicate branch name in " + repr(section))
88 b["bin_url"] = section.get("binary_url")
89 b["bin_key"] = section.get("binary_password")
91 b["src_url"] = section.get("source_url")
92 b["src_key"] = section.get("source_password")
94 b["gpg_key"] = section.get("gpg_key")
96 b["usign_key"] = section.get("usign_key")
97 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
98 b["usign_comment"] = section.get("usign_comment", usign_comment)
100 b["config_seed"] = section.get("config_seed")
102 b["kmod_archive"] = section.getboolean("kmod_archive", False)
105 log.msg("Configured branch: {}".format(name))
107 # PB port can be either a numeric port or a connection string
108 pb_port = inip1.get("port") or 9989
110 # This is the dictionary that the buildmaster pays attention to. We also use
111 # a shorter alias to save typing.
112 c = BuildmasterConfig = {}
114 ####### PROJECT IDENTITY
116 # the 'title' string will appear at the top of this buildbot
117 # installation's html.WebStatus home page (linked to the
118 # 'titleURL') and is embedded in the title of the waterfall HTML page.
120 c['title'] = ini['general'].get("title")
121 c['titleURL'] = ini['general'].get("title_url")
123 # the 'buildbotURL' string should point to the location where the buildbot's
124 # internal web server (usually the html.WebStatus page) is visible. This
125 # typically uses the port number set in the Waterfall 'status' entry, but
126 # with an externally-visible host name which the buildbot cannot figure out
129 c['buildbotURL'] = inip1.get("buildbot_url")
133 # The 'workers' list defines the set of recognized buildworkers. Each element is
134 # a Worker object, specifying a unique worker name and password. The same
135 # worker name and password must be configured on the worker.
140 for section in ini.sections():
141 if section.startswith("branch "):
142 ini_parse_branch(ini[section])
144 if section.startswith("worker "):
145 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
146 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
147 sl_props = { 'dl_lock':None, 'ul_lock':None }
148 name = ini.get(section, "name")
149 password = ini.get(section, "password")
150 if ini.has_option(section, "dl_lock"):
151 lockname = ini.get(section, "dl_lock")
152 sl_props['dl_lock'] = lockname
153 if lockname not in NetLocks:
154 NetLocks[lockname] = locks.MasterLock(lockname)
155 if ini.has_option(section, "ul_lock"):
156 lockname = ini.get(section, "ul_lock")
157 sl_props['ul_lock'] = lockname
158 if lockname not in NetLocks:
159 NetLocks[lockname] = locks.MasterLock(lockname)
160 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
162 c['protocols'] = {'pb': {'port': pb_port}}
165 c['collapseRequests'] = True
167 # Reduce amount of backlog data
168 c['configurators'] = [util.JanitorConfigurator(
169 logHorizon=timedelta(days=3),
173 @defer.inlineCallbacks
174 def getNewestCompleteTime(bldr):
175 """Returns the complete_at of the latest completed and not SKIPPED
176 build request for this builder, or None if there are no such build
177 requests. We need to filter out SKIPPED requests because we're
178 using collapseRequests=True which is unfortunately marking all
179 previous requests as complete when new buildset is created.
181 @returns: datetime instance or None, via Deferred
184 bldrid = yield bldr.getBuilderId()
185 completed = yield bldr.master.data.get(
186 ('builders', bldrid, 'buildrequests'),
188 resultspec.Filter('complete', 'eq', [True]),
189 resultspec.Filter('results', 'ne', [results.SKIPPED]),
191 order=['-complete_at'], limit=1)
195 complete_at = completed[0]['complete_at']
197 last_build = yield bldr.master.data.get(
200 resultspec.Filter('builderid', 'eq', [bldrid]),
202 order=['-started_at'], limit=1)
204 if last_build and last_build[0]:
205 last_complete_at = last_build[0]['complete_at']
206 if last_complete_at and (last_complete_at > complete_at):
207 return last_complete_at
211 @defer.inlineCallbacks
212 def prioritizeBuilders(master, builders):
213 """Returns sorted list of builders by their last timestamp of completed and
216 @returns: list of sorted builders
219 def is_building(bldr):
220 return bool(bldr.building) or bool(bldr.old_building)
223 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
224 d.addCallback(lambda complete_at: (complete_at, bldr))
228 (complete_at, bldr) = item
232 complete_at = date.replace(tzinfo=tzutc())
234 if is_building(bldr):
236 complete_at = date.replace(tzinfo=tzutc())
238 return (complete_at, bldr.name)
240 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
241 results.sort(key=bldr_sort)
244 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
246 return [r[1] for r in results]
248 c['prioritizeBuilders'] = prioritizeBuilders
250 ####### CHANGESOURCES
256 def populateTargets():
257 sourcegit = work_dir + '/source.git'
258 if os.path.isdir(sourcegit):
259 subprocess.call(["rm", "-rf", sourcegit])
261 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, sourcegit])
263 os.makedirs(sourcegit + '/tmp', exist_ok=True)
264 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
265 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
268 line = findtargets.stdout.readline()
271 ta = line.decode().strip().split(' ')
272 targets.append(ta[0])
274 subprocess.call(["rm", "-rf", sourcegit])
278 # the 'change_source' setting tells the buildmaster how it should find out
279 # about source code changes. Here we point to the buildbot clone of pyflakes.
281 c['change_source'] = []
282 c['change_source'].append(GitPoller(
284 workdir=work_dir+'/work.git', branch=repo_branch,
289 # Configure the Schedulers, which decide how to react to incoming changes. In this
290 # case, just kick off a 'basebuild' build
292 class TagChoiceParameter(BaseParameter):
293 spec_attributes = ["strict", "choices"]
297 def __init__(self, name, label=None, **kw):
298 super().__init__(name, label, **kw)
299 self._choice_list = []
304 basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch)
307 findtags = subprocess.Popen(
308 ['git', 'ls-remote', '--tags', repo_url],
309 stdout = subprocess.PIPE)
312 line = findtags.stdout.readline()
317 tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
319 if tagver and tagver[1].find(basever[1]) == 0:
320 taglist.append(tagver[1])
322 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
323 taglist.insert(0, '')
325 self._choice_list = taglist
327 return self._choice_list
329 def parse_from_arg(self, s):
330 if self.strict and s not in self._choice_list:
331 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
335 c['schedulers'].append(SingleBranchScheduler(
337 change_filter = filter.ChangeFilter(branch=repo_branch),
338 treeStableTimer = 60,
339 builderNames = targets))
341 c['schedulers'].append(ForceScheduler(
343 buttonName = "Force builds",
344 label = "Force build details",
345 builderNames = [ "00_force_build" ],
348 util.CodebaseParameter(
350 label = "Repository",
351 branch = util.FixedParameter(name = "branch", default = ""),
352 revision = util.FixedParameter(name = "revision", default = ""),
353 repository = util.FixedParameter(name = "repository", default = ""),
354 project = util.FixedParameter(name = "project", default = "")
358 reason = util.StringParameter(
361 default = "Trigger build",
367 util.NestedParameter(
369 label="Build Options",
372 util.ChoiceStringParameter(
374 label = "Build target",
376 choices = [ "all" ] + targets
390 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
391 # what steps, and which workers can execute them. Note that any particular build will
392 # only take place on one worker.
394 def IsTaggingRequested(step):
395 val = step.getProperty("tag")
396 if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
401 def IsNoMasterBuild(step):
402 return step.getProperty("branch") != "master"
404 def GetBaseVersion(branch):
405 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
406 return branch.split('-')[1]
411 def GetVersionPrefix(props):
412 branch = props.getProperty("branch")
413 basever = GetBaseVersion(branch)
414 if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
415 return "%s/" % props["tag"]
416 elif basever != "master":
417 return "%s-SNAPSHOT/" % basever
421 def GetNextBuild(builder, requests):
423 if r.properties and r.properties.hasProperty("tag"):
427 log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
430 def MakeEnv(overrides=None, tryccache=False):
432 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
433 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
436 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
437 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
438 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
440 env['CC'] = env['CCC']
441 env['CXX'] = env['CCXX']
443 if overrides is not None:
444 env.update(overrides)
448 def NetLockDl(props):
450 if props.hasProperty("dl_lock"):
451 lock = NetLocks[props["dl_lock"]]
453 return [lock.access('exclusive')]
458 def NetLockUl(props):
460 if props.hasProperty("ul_lock"):
461 lock = NetLocks[props["ul_lock"]]
463 return [lock.access('exclusive')]
468 def TagPropertyValue(props):
469 if props.hasProperty("options"):
470 options = props.getProperty("options")
471 if type(options) is dict:
472 return options.get("tag")
475 def IsTargetSelected(target):
476 def CheckTargetProperty(step):
478 options = step.getProperty("options")
479 if type(options) is dict:
480 selected_target = options.get("target", "all")
481 if selected_target != "all" and selected_target != target:
488 return CheckTargetProperty
490 def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
492 seckey = base64.b64decode(seckey)
496 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
497 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
502 dlLock = locks.WorkerLock("worker_dl")
506 for worker in c['workers']:
507 workerNames.append(worker.workername)
509 force_factory = BuildFactory()
511 c['builders'].append(BuilderConfig(
512 name = "00_force_build",
513 workernames = workerNames,
514 factory = force_factory))
516 for target in targets:
517 ts = target.split('/')
519 factory = BuildFactory()
521 # setup shared work directory if required
522 factory.addStep(ShellCommand(
524 description = "Setting up shared work directory",
525 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
527 haltOnFailure = True))
529 # find number of cores
530 factory.addStep(SetPropertyFromCommand(
533 description = "Finding number of CPUs",
534 command = ["nproc"]))
536 # find gcc and g++ compilers
537 factory.addStep(FileDownload(
538 name = "dlfindbinpl",
539 mastersrc = scripts_dir + '/findbin.pl',
540 workerdest = "../findbin.pl",
543 factory.addStep(SetPropertyFromCommand(
545 property = "cc_command",
546 description = "Finding gcc command",
548 "../findbin.pl", "gcc", "", "",
550 haltOnFailure = True))
552 factory.addStep(SetPropertyFromCommand(
554 property = "cxx_command",
555 description = "Finding g++ command",
557 "../findbin.pl", "g++", "", "",
559 haltOnFailure = True))
561 # see if ccache is available
562 factory.addStep(SetPropertyFromCommand(
563 property = "ccache_command",
564 command = ["which", "ccache"],
565 description = "Testing for ccache command",
566 haltOnFailure = False,
567 flunkOnFailure = False,
568 warnOnFailure = False,
571 # Workaround bug when switching from a checked out tag back to a branch
572 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
573 factory.addStep(ShellCommand(
574 name = "gitcheckout",
575 description = "Ensure that Git HEAD is sane",
576 command = Interpolate("if [ -d .git ]; then git checkout -f %(prop:branch)s && git branch --set-upstream-to origin/%(prop:branch)s || rm -fr .git; else exit 0; fi"),
577 haltOnFailure = True))
579 # check out the source
581 # if repo doesn't exist: 'git clone repourl'
582 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
583 # 'git fetch -t repourl branch; git reset --hard revision'
590 haltOnFailure = True,
594 factory.addStep(ShellCommand(
596 description = "Fetching Git remote refs",
597 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
602 factory.addStep(ShellCommand(
604 description = "Checking out Git tag",
605 command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
606 haltOnFailure = True,
607 doStepIf = IsTaggingRequested
610 # Verify that Git HEAD points to a tag or branch
611 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
612 factory.addStep(ShellCommand(
614 description = "Ensure that Git HEAD is pointing to a branch or tag",
615 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
616 haltOnFailure = True))
618 factory.addStep(ShellCommand(
620 description = "Remove tmp folder",
621 command=["rm", "-rf", "tmp/"]))
624 factory.addStep(ShellCommand(
625 name = "rmfeedlinks",
626 description = "Remove feed symlinks",
627 command=["rm", "-rf", "package/feeds/"]))
629 factory.addStep(StringDownload(
631 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
632 workerdest = "../ccache_cc.sh",
636 factory.addStep(StringDownload(
638 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
639 workerdest = "../ccache_cxx.sh",
644 factory.addStep(ShellCommand(
645 name = "updatefeeds",
646 description = "Updating feeds",
647 command=["./scripts/feeds", "update"],
648 env = MakeEnv(tryccache=True),
649 haltOnFailure = True,
654 factory.addStep(ShellCommand(
655 name = "installfeeds",
656 description = "Installing feeds",
657 command=["./scripts/feeds", "install", "-a"],
658 env = MakeEnv(tryccache=True),
663 if config_seed is not None:
664 factory.addStep(StringDownload(
665 name = "dlconfigseed",
666 s = config_seed + '\n',
667 workerdest = ".config",
672 factory.addStep(ShellCommand(
674 description = "Seeding .config",
675 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\nCONFIG_SIGNED_PACKAGES=%s\\n' >> .config" %(ts[0], ts[0], ts[1], 'y' if usign_key is not None else 'n')
678 factory.addStep(ShellCommand(
680 description = "Removing output directory",
681 command = ["rm", "-rf", "bin/"]
684 factory.addStep(ShellCommand(
686 description = "Populating .config",
687 command = ["make", "defconfig"],
692 factory.addStep(ShellCommand(
694 description = "Checking architecture",
695 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
703 factory.addStep(SetPropertyFromCommand(
706 description = "Finding libc suffix",
707 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
710 if usign_key is not None:
711 factory.addStep(StringDownload(
712 name = "dlkeybuildpub",
713 s = UsignSec2Pub(usign_key, usign_comment),
714 workerdest = "key-build.pub",
718 factory.addStep(StringDownload(
720 s = "# fake private key",
721 workerdest = "key-build",
725 factory.addStep(StringDownload(
726 name = "dlkeybuilducert",
727 s = "# fake certificate",
728 workerdest = "key-build.ucert",
733 factory.addStep(ShellCommand(
735 description = "Preparing dl/",
736 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
742 factory.addStep(ShellCommand(
744 description = "Building and installing GNU tar",
745 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
746 env = MakeEnv(tryccache=True),
751 factory.addStep(ShellCommand(
753 description = "Populating dl/",
754 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
757 locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
760 factory.addStep(ShellCommand(
762 description = "Cleaning base-files",
763 command=["make", "package/base-files/clean", "V=s"]
767 factory.addStep(ShellCommand(
769 description = "Building and installing tools",
770 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
771 env = MakeEnv(tryccache=True),
775 factory.addStep(ShellCommand(
777 description = "Building and installing toolchain",
778 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
783 factory.addStep(ShellCommand(
785 description = "Building kmods",
786 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
791 # find kernel version
792 factory.addStep(SetPropertyFromCommand(
793 name = "kernelversion",
794 property = "kernelversion",
795 description = "Finding the effective Kernel version",
796 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
797 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") }
800 factory.addStep(ShellCommand(
802 description = "Cleaning up package build",
803 command=["make", "package/cleanup", "V=s"]
806 factory.addStep(ShellCommand(
808 description = "Building packages",
809 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
814 factory.addStep(ShellCommand(
816 description = "Installing packages",
817 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
822 factory.addStep(ShellCommand(
824 description = "Indexing packages",
825 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
830 factory.addStep(ShellCommand(
832 description = "Building and installing images",
833 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
838 factory.addStep(ShellCommand(
840 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
841 command = "make -j1 buildinfo V=s || true",
846 factory.addStep(ShellCommand(
847 name = "json_overview_image_info",
848 description = "Generate profiles.json in target folder",
849 command = "make -j1 json_overview_image_info V=s || true",
854 factory.addStep(ShellCommand(
856 description = "Calculating checksums",
857 command=["make", "-j1", "checksum", "V=s"],
862 if enable_kmod_archive:
863 factory.addStep(ShellCommand(
865 description = "Creating kmod directory",
866 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
870 factory.addStep(ShellCommand(
871 name = "kmodprepare",
872 description = "Preparing kmod archive",
873 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
874 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
875 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
879 factory.addStep(ShellCommand(
881 description = "Indexing kmod archive",
882 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
883 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
889 if ini.has_option("gpg", "key") or usign_key is not None:
890 factory.addStep(MasterShellCommand(
891 name = "signprepare",
892 description = "Preparing temporary signing directory",
893 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
897 factory.addStep(ShellCommand(
899 description = "Packing files to sign",
900 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
904 factory.addStep(FileUpload(
905 workersrc = "sign.tar.gz",
906 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
910 factory.addStep(MasterShellCommand(
912 description = "Signing files",
913 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
914 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
918 factory.addStep(FileDownload(
919 name = "dlsigntargz",
920 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
921 workerdest = "sign.tar.gz",
925 factory.addStep(ShellCommand(
927 description = "Unpacking signed files",
928 command = ["tar", "-xzf", "sign.tar.gz"],
933 factory.addStep(ShellCommand(
935 description = "Preparing upload directory structure",
936 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
940 factory.addStep(ShellCommand(
941 name = "linkprepare",
942 description = "Preparing repository symlink",
943 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
944 doStepIf = IsNoMasterBuild,
948 if enable_kmod_archive:
949 factory.addStep(ShellCommand(
950 name = "kmoddirprepare",
951 description = "Preparing kmod archive upload directory",
952 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
956 factory.addStep(ShellCommand(
958 description = "Uploading directory structure",
959 command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
960 env={'RSYNC_PASSWORD': rsync_bin_key},
961 haltOnFailure = True,
966 # download remote sha256sums to 'target-sha256sums'
967 factory.addStep(ShellCommand(
968 name = "target-sha256sums",
969 description = "Fetching remote sha256sums for target",
970 command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
971 env={'RSYNC_PASSWORD': rsync_bin_key},
973 haltOnFailure = False,
974 flunkOnFailure = False,
975 warnOnFailure = False,
978 # build list of files to upload
979 factory.addStep(FileDownload(
980 name = "dlsha2rsyncpl",
981 mastersrc = scripts_dir + '/sha2rsync.pl',
982 workerdest = "../sha2rsync.pl",
986 factory.addStep(ShellCommand(
988 description = "Building list of files to upload",
989 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
990 haltOnFailure = True,
993 factory.addStep(FileDownload(
995 mastersrc = scripts_dir + '/rsync.sh',
996 workerdest = "../rsync.sh",
1000 # upload new files and update existing ones
1001 factory.addStep(ShellCommand(
1002 name = "targetupload",
1003 description = "Uploading target files",
1004 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1005 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1006 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1007 env={'RSYNC_PASSWORD': rsync_bin_key},
1008 haltOnFailure = True,
1012 # delete files which don't exist locally
1013 factory.addStep(ShellCommand(
1014 name = "targetprune",
1015 description = "Pruning target files",
1016 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1017 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1018 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1019 env={'RSYNC_PASSWORD': rsync_bin_key},
1020 haltOnFailure = True,
1025 if enable_kmod_archive:
1026 factory.addStep(ShellCommand(
1027 name = "kmodupload",
1028 description = "Uploading kmod archive",
1029 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1030 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1031 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1032 env={'RSYNC_PASSWORD': rsync_bin_key},
1033 haltOnFailure = True,
1038 if rsync_src_url is not None:
1039 factory.addStep(ShellCommand(
1040 name = "sourcelist",
1041 description = "Finding source archives to upload",
1042 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1043 haltOnFailure = True
1046 factory.addStep(ShellCommand(
1047 name = "sourceupload",
1048 description = "Uploading source archives",
1049 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
1050 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
1051 env={'RSYNC_PASSWORD': rsync_src_key},
1052 haltOnFailure = True,
1057 factory.addStep(ShellCommand(
1059 description = "Reporting disk usage",
1060 command=["df", "-h", "."],
1061 env={'LC_ALL': 'C'},
1062 haltOnFailure = False,
1063 flunkOnFailure = False,
1064 warnOnFailure = False,
1068 factory.addStep(ShellCommand(
1070 description = "Reporting estimated file space usage",
1071 command=["du", "-sh", "."],
1072 env={'LC_ALL': 'C'},
1073 haltOnFailure = False,
1074 flunkOnFailure = False,
1075 warnOnFailure = False,
1079 factory.addStep(ShellCommand(
1080 name = "ccachestat",
1081 description = "Reporting ccache stats",
1082 command=["ccache", "-s"],
1083 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1084 want_stderr = False,
1085 haltOnFailure = False,
1086 flunkOnFailure = False,
1087 warnOnFailure = False,
1091 c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1093 c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1094 force_factory.addStep(steps.Trigger(
1095 name = "trigger_%s" % target,
1096 description = "Triggering %s build" % target,
1097 schedulerNames = [ "trigger_%s" % target ],
1098 set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
1099 doStepIf = IsTargetSelected(target)
1103 ####### STATUS TARGETS
1105 # 'status' is a list of Status Targets. The results of each build will be
1106 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1107 # including web pages, email senders, and IRC bots.
1109 if "status_bind" in inip1:
1111 'port': inip1.get("status_bind"),
1113 'waterfall_view': True,
1114 'console_view': True,
1119 if "status_user" in inip1 and "status_password" in inip1:
1120 c['www']['auth'] = util.UserPasswordAuth([
1121 (inip1.get("status_user"), inip1.get("status_password"))
1123 c['www']['authz'] = util.Authz(
1124 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1125 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1129 if ini.has_section("irc"):
1131 irc_host = iniirc.get("host", None)
1132 irc_port = iniirc.getint("port", 6667)
1133 irc_chan = iniirc.get("channel", None)
1134 irc_nick = iniirc.get("nickname", None)
1135 irc_pass = iniirc.get("password", None)
1137 if irc_host and irc_nick and irc_chan:
1138 irc = reporters.IRC(irc_host, irc_nick,
1140 password = irc_pass,
1141 channels = [ irc_chan ],
1142 notify_events = [ 'exception', 'problem', 'recovery' ]
1145 c['services'].append(irc)
1147 c['revlink'] = util.RevlinkMatch([
1148 r'https://git.openwrt.org/openwrt/(.*).git'
1150 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1155 # This specifies what database buildbot uses to store its state. You can leave
1156 # this at its default for all but the largest installations.
1157 'db_url' : "sqlite:///state.sqlite",
1160 c['buildbotNetUsageData'] = None