phase1: use branch config in factory
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import base64
7 import subprocess
8 import configparser
9
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
12
13 from twisted.internet import defer
14 from twisted.python import log
15
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes import filter
19 from buildbot.changes.gitpoller import GitPoller
20 from buildbot.config import BuilderConfig
21 from buildbot.plugins import reporters
22 from buildbot.plugins import schedulers
23 from buildbot.plugins import steps
24 from buildbot.plugins import util
25 from buildbot.process import properties
26 from buildbot.process import results
27 from buildbot.process.factory import BuildFactory
28 from buildbot.process.properties import Interpolate
29 from buildbot.process.properties import Property
30 from buildbot.schedulers.basic import SingleBranchScheduler
31 from buildbot.schedulers.forcesched import BaseParameter
32 from buildbot.schedulers.forcesched import ForceScheduler
33 from buildbot.schedulers.forcesched import ValidationError
34 from buildbot.steps.master import MasterShellCommand
35 from buildbot.steps.shell import SetPropertyFromCommand
36 from buildbot.steps.shell import ShellCommand
37 from buildbot.steps.source.git import Git
38 from buildbot.steps.transfer import FileDownload
39 from buildbot.steps.transfer import FileUpload
40 from buildbot.steps.transfer import StringDownload
41 from buildbot.worker import Worker
42
43
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
47
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
50
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
53
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
56
57 inip1 = ini['phase1']
58
59 # Globals
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
62
63 repo_url = ini['repo'].get("url")
64 repo_branch = ini['repo'].get("branch", "master")
65
66 rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
67
68 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
69 # rsync_bin_defopts += ["--contimeout=20"]
70
71 rsync_src_defopts = ["-v", "-4", "--timeout=120"]
72
73 #if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
74 # rsync_src_defopts += ["--contimeout=20"]
75
76 branches = {}
77
78 def ini_parse_branch(section):
79 b = {}
80 name = section.get("name")
81
82 if not name:
83 raise ValueError("missing 'name' in " + repr(section))
84 if name in branches:
85 raise ValueError("duplicate branch name in " + repr(section))
86
87 b["name"] = name
88 b["bin_url"] = section.get("binary_url")
89 b["bin_key"] = section.get("binary_password")
90
91 b["src_url"] = section.get("source_url")
92 b["src_key"] = section.get("source_password")
93
94 b["gpg_key"] = section.get("gpg_key")
95
96 b["usign_key"] = section.get("usign_key")
97 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
98 b["usign_comment"] = section.get("usign_comment", usign_comment)
99
100 b["config_seed"] = section.get("config_seed")
101
102 b["kmod_archive"] = section.getboolean("kmod_archive", False)
103
104 branches[name] = b
105 log.msg("Configured branch: {}".format(name))
106
107 # PB port can be either a numeric port or a connection string
108 pb_port = inip1.get("port") or 9989
109
110 # This is the dictionary that the buildmaster pays attention to. We also use
111 # a shorter alias to save typing.
112 c = BuildmasterConfig = {}
113
114 ####### PROJECT IDENTITY
115
116 # the 'title' string will appear at the top of this buildbot
117 # installation's html.WebStatus home page (linked to the
118 # 'titleURL') and is embedded in the title of the waterfall HTML page.
119
120 c['title'] = ini['general'].get("title")
121 c['titleURL'] = ini['general'].get("title_url")
122
123 # the 'buildbotURL' string should point to the location where the buildbot's
124 # internal web server (usually the html.WebStatus page) is visible. This
125 # typically uses the port number set in the Waterfall 'status' entry, but
126 # with an externally-visible host name which the buildbot cannot figure out
127 # without some help.
128
129 c['buildbotURL'] = inip1.get("buildbot_url")
130
131 ####### BUILDWORKERS
132
133 # The 'workers' list defines the set of recognized buildworkers. Each element is
134 # a Worker object, specifying a unique worker name and password. The same
135 # worker name and password must be configured on the worker.
136
137 c['workers'] = []
138 NetLocks = dict()
139
140 for section in ini.sections():
141 if section.startswith("branch "):
142 ini_parse_branch(ini[section])
143
144 if section.startswith("worker "):
145 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
146 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
147 sl_props = { 'dl_lock':None, 'ul_lock':None }
148 name = ini.get(section, "name")
149 password = ini.get(section, "password")
150 if ini.has_option(section, "dl_lock"):
151 lockname = ini.get(section, "dl_lock")
152 sl_props['dl_lock'] = lockname
153 if lockname not in NetLocks:
154 NetLocks[lockname] = locks.MasterLock(lockname)
155 if ini.has_option(section, "ul_lock"):
156 lockname = ini.get(section, "ul_lock")
157 sl_props['ul_lock'] = lockname
158 if lockname not in NetLocks:
159 NetLocks[lockname] = locks.MasterLock(lockname)
160 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
161
162 c['protocols'] = {'pb': {'port': pb_port}}
163
164 # coalesce builds
165 c['collapseRequests'] = True
166
167 # Reduce amount of backlog data
168 c['configurators'] = [util.JanitorConfigurator(
169 logHorizon=timedelta(days=3),
170 hour=6,
171 )]
172
173 @defer.inlineCallbacks
174 def getNewestCompleteTime(bldr):
175 """Returns the complete_at of the latest completed and not SKIPPED
176 build request for this builder, or None if there are no such build
177 requests. We need to filter out SKIPPED requests because we're
178 using collapseRequests=True which is unfortunately marking all
179 previous requests as complete when new buildset is created.
180
181 @returns: datetime instance or None, via Deferred
182 """
183
184 bldrid = yield bldr.getBuilderId()
185 completed = yield bldr.master.data.get(
186 ('builders', bldrid, 'buildrequests'),
187 [
188 resultspec.Filter('complete', 'eq', [True]),
189 resultspec.Filter('results', 'ne', [results.SKIPPED]),
190 ],
191 order=['-complete_at'], limit=1)
192 if not completed:
193 return
194
195 complete_at = completed[0]['complete_at']
196
197 last_build = yield bldr.master.data.get(
198 ('builds', ),
199 [
200 resultspec.Filter('builderid', 'eq', [bldrid]),
201 ],
202 order=['-started_at'], limit=1)
203
204 if last_build and last_build[0]:
205 last_complete_at = last_build[0]['complete_at']
206 if last_complete_at and (last_complete_at > complete_at):
207 return last_complete_at
208
209 return complete_at
210
211 @defer.inlineCallbacks
212 def prioritizeBuilders(master, builders):
213 """Returns sorted list of builders by their last timestamp of completed and
214 not skipped build.
215
216 @returns: list of sorted builders
217 """
218
219 def is_building(bldr):
220 return bool(bldr.building) or bool(bldr.old_building)
221
222 def bldr_info(bldr):
223 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
224 d.addCallback(lambda complete_at: (complete_at, bldr))
225 return d
226
227 def bldr_sort(item):
228 (complete_at, bldr) = item
229
230 if not complete_at:
231 date = datetime.min
232 complete_at = date.replace(tzinfo=tzutc())
233
234 if is_building(bldr):
235 date = datetime.max
236 complete_at = date.replace(tzinfo=tzutc())
237
238 return (complete_at, bldr.name)
239
240 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
241 results.sort(key=bldr_sort)
242
243 for r in results:
244 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
245
246 return [r[1] for r in results]
247
248 c['prioritizeBuilders'] = prioritizeBuilders
249
250 ####### CHANGESOURCES
251
252
253 # find targets
254 targets = [ ]
255
256 def populateTargets():
257 sourcegit = work_dir + '/source.git'
258 if os.path.isdir(sourcegit):
259 subprocess.call(["rm", "-rf", sourcegit])
260
261 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, sourcegit])
262
263 os.makedirs(sourcegit + '/tmp', exist_ok=True)
264 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
265 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
266
267 while True:
268 line = findtargets.stdout.readline()
269 if not line:
270 break
271 ta = line.decode().strip().split(' ')
272 targets.append(ta[0])
273
274 subprocess.call(["rm", "-rf", sourcegit])
275
276 populateTargets()
277
278 # the 'change_source' setting tells the buildmaster how it should find out
279 # about source code changes. Here we point to the buildbot clone of pyflakes.
280
281 c['change_source'] = []
282 c['change_source'].append(GitPoller(
283 repo_url,
284 workdir=work_dir+'/work.git', branch=repo_branch,
285 pollinterval=300))
286
287 ####### SCHEDULERS
288
289 # Configure the Schedulers, which decide how to react to incoming changes. In this
290 # case, just kick off a 'basebuild' build
291
292 class TagChoiceParameter(BaseParameter):
293 spec_attributes = ["strict", "choices"]
294 type = "list"
295 strict = True
296
297 def __init__(self, name, label=None, **kw):
298 super().__init__(name, label, **kw)
299 self._choice_list = []
300
301 @property
302 def choices(self):
303 taglist = []
304 basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch)
305
306 if basever:
307 findtags = subprocess.Popen(
308 ['git', 'ls-remote', '--tags', repo_url],
309 stdout = subprocess.PIPE)
310
311 while True:
312 line = findtags.stdout.readline()
313
314 if not line:
315 break
316
317 tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
318
319 if tagver and tagver[1].find(basever[1]) == 0:
320 taglist.append(tagver[1])
321
322 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
323 taglist.insert(0, '')
324
325 self._choice_list = taglist
326
327 return self._choice_list
328
329 def parse_from_arg(self, s):
330 if self.strict and s not in self._choice_list:
331 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
332 return s
333
334 c['schedulers'] = []
335 c['schedulers'].append(SingleBranchScheduler(
336 name = "all",
337 change_filter = filter.ChangeFilter(branch=repo_branch),
338 treeStableTimer = 60,
339 builderNames = targets))
340
341 c['schedulers'].append(ForceScheduler(
342 name = "force",
343 buttonName = "Force builds",
344 label = "Force build details",
345 builderNames = [ "00_force_build" ],
346
347 codebases = [
348 util.CodebaseParameter(
349 "",
350 label = "Repository",
351 branch = util.FixedParameter(name = "branch", default = ""),
352 revision = util.FixedParameter(name = "revision", default = ""),
353 repository = util.FixedParameter(name = "repository", default = ""),
354 project = util.FixedParameter(name = "project", default = "")
355 )
356 ],
357
358 reason = util.StringParameter(
359 name = "reason",
360 label = "Reason",
361 default = "Trigger build",
362 required = True,
363 size = 80
364 ),
365
366 properties = [
367 util.NestedParameter(
368 name="options",
369 label="Build Options",
370 layout="vertical",
371 fields=[
372 util.ChoiceStringParameter(
373 name = "target",
374 label = "Build target",
375 default = "all",
376 choices = [ "all" ] + targets
377 ),
378 TagChoiceParameter(
379 name = "tag",
380 label = "Build tag",
381 default = ""
382 )
383 ]
384 )
385 ]
386 ))
387
388 ####### BUILDERS
389
390 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
391 # what steps, and which workers can execute them. Note that any particular build will
392 # only take place on one worker.
393
394 def IsTaggingRequested(step):
395 tag = step.getProperty("tag")
396 return tag and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", tag)
397
398 def IsNoMasterBuild(step):
399 return step.getProperty("branch") != "master"
400
401 def IsUsignEnabled(step):
402 branch = step.getProperty("branch")
403 return branch and branches[branch].get("usign_key")
404
405 def IsSignEnabled(step):
406 branch = step.getProperty("branch")
407 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
408
409 def IsKmodArchiveEnabled(step):
410 branch = step.getProperty("branch")
411 return branch and branches[branch].get("kmod_archive")
412
413 def GetBaseVersion(branch):
414 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
415 return branch.split('-')[1]
416 else:
417 return "master"
418
419 @properties.renderer
420 def GetVersionPrefix(props):
421 branch = props.getProperty("branch")
422 basever = GetBaseVersion(branch)
423 if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
424 return "%s/" % props["tag"]
425 elif basever != "master":
426 return "%s-SNAPSHOT/" % basever
427 else:
428 return ""
429
430 @util.renderer
431 def GetConfigSeed(props):
432 branch = props.getProperty("branch")
433 return branch and branches[branch].get("config_seed") or ""
434
435 @util.renderer
436 def GetRsyncParams(props, srcorbin, urlorkey):
437 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
438 branch = props.getProperty("branch")
439 opt = srcorbin + "_" + urlorkey
440 return branch and branches[branch].get(opt)
441
442 @util.renderer
443 def GetUsignKey(props):
444 branch = props.getProperty("branch")
445 return branch and branches[branch].get("usign_key")
446
447 def GetNextBuild(builder, requests):
448 for r in requests:
449 if r.properties and r.properties.hasProperty("tag"):
450 return r
451
452 r = requests[0]
453 log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
454 return r
455
456 def MakeEnv(overrides=None, tryccache=False):
457 env = {
458 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
459 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
460 }
461 if tryccache:
462 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
463 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
464 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
465 else:
466 env['CC'] = env['CCC']
467 env['CXX'] = env['CCXX']
468 env['CCACHE'] = ''
469 if overrides is not None:
470 env.update(overrides)
471 return env
472
473 @properties.renderer
474 def NetLockDl(props):
475 lock = None
476 if props.hasProperty("dl_lock"):
477 lock = NetLocks[props["dl_lock"]]
478 if lock is not None:
479 return [lock.access('exclusive')]
480 else:
481 return []
482
483 @properties.renderer
484 def NetLockUl(props):
485 lock = None
486 if props.hasProperty("ul_lock"):
487 lock = NetLocks[props["ul_lock"]]
488 if lock is not None:
489 return [lock.access('exclusive')]
490 else:
491 return []
492
493 @util.renderer
494 def TagPropertyValue(props):
495 if props.hasProperty("options"):
496 options = props.getProperty("options")
497 if type(options) is dict:
498 return options.get("tag")
499 return None
500
501 def IsTargetSelected(target):
502 def CheckTargetProperty(step):
503 try:
504 options = step.getProperty("options")
505 if type(options) is dict:
506 selected_target = options.get("target", "all")
507 if selected_target != "all" and selected_target != target:
508 return False
509 except KeyError:
510 pass
511
512 return True
513
514 return CheckTargetProperty
515
516 @util.renderer
517 def UsignSec2Pub(props):
518 branch = props.getProperty("branch")
519 try:
520 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
521 seckey = branches[branch].get("usign_key")
522 seckey = base64.b64decode(seckey)
523 except:
524 return None
525
526 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
527 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
528
529
530 c['builders'] = []
531
532 dlLock = locks.WorkerLock("worker_dl")
533
534 workerNames = [ ]
535
536 for worker in c['workers']:
537 workerNames.append(worker.workername)
538
539 force_factory = BuildFactory()
540
541 c['builders'].append(BuilderConfig(
542 name = "00_force_build",
543 workernames = workerNames,
544 factory = force_factory))
545
546 for target in targets:
547 ts = target.split('/')
548
549 factory = BuildFactory()
550
551 # setup shared work directory if required
552 factory.addStep(ShellCommand(
553 name = "sharedwd",
554 description = "Setting up shared work directory",
555 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
556 workdir = ".",
557 haltOnFailure = True))
558
559 # find number of cores
560 factory.addStep(SetPropertyFromCommand(
561 name = "nproc",
562 property = "nproc",
563 description = "Finding number of CPUs",
564 command = ["nproc"]))
565
566 # find gcc and g++ compilers
567 factory.addStep(FileDownload(
568 name = "dlfindbinpl",
569 mastersrc = scripts_dir + '/findbin.pl',
570 workerdest = "../findbin.pl",
571 mode = 0o755))
572
573 factory.addStep(SetPropertyFromCommand(
574 name = "gcc",
575 property = "cc_command",
576 description = "Finding gcc command",
577 command = [
578 "../findbin.pl", "gcc", "", "",
579 ],
580 haltOnFailure = True))
581
582 factory.addStep(SetPropertyFromCommand(
583 name = "g++",
584 property = "cxx_command",
585 description = "Finding g++ command",
586 command = [
587 "../findbin.pl", "g++", "", "",
588 ],
589 haltOnFailure = True))
590
591 # see if ccache is available
592 factory.addStep(SetPropertyFromCommand(
593 property = "ccache_command",
594 command = ["which", "ccache"],
595 description = "Testing for ccache command",
596 haltOnFailure = False,
597 flunkOnFailure = False,
598 warnOnFailure = False,
599 ))
600
601 # Workaround bug when switching from a checked out tag back to a branch
602 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
603 factory.addStep(ShellCommand(
604 name = "gitcheckout",
605 description = "Ensure that Git HEAD is sane",
606 command = Interpolate("if [ -d .git ]; then git checkout -f %(prop:branch)s && git branch --set-upstream-to origin/%(prop:branch)s || rm -fr .git; else exit 0; fi"),
607 haltOnFailure = True))
608
609 # check out the source
610 # Git() runs:
611 # if repo doesn't exist: 'git clone repourl'
612 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
613 # 'git fetch -t repourl branch; git reset --hard revision'
614 factory.addStep(Git(
615 name = "git",
616 repourl = repo_url,
617 mode = 'full',
618 method = 'fresh',
619 locks = NetLockDl,
620 haltOnFailure = True,
621 ))
622
623 # update remote refs
624 factory.addStep(ShellCommand(
625 name = "fetchrefs",
626 description = "Fetching Git remote refs",
627 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
628 haltOnFailure = True
629 ))
630
631 # switch to tag
632 factory.addStep(ShellCommand(
633 name = "switchtag",
634 description = "Checking out Git tag",
635 command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
636 haltOnFailure = True,
637 doStepIf = IsTaggingRequested
638 ))
639
640 # Verify that Git HEAD points to a tag or branch
641 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
642 factory.addStep(ShellCommand(
643 name = "gitverify",
644 description = "Ensure that Git HEAD is pointing to a branch or tag",
645 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
646 haltOnFailure = True))
647
648 factory.addStep(ShellCommand(
649 name = "rmtmp",
650 description = "Remove tmp folder",
651 command=["rm", "-rf", "tmp/"]))
652
653 # feed
654 factory.addStep(ShellCommand(
655 name = "rmfeedlinks",
656 description = "Remove feed symlinks",
657 command=["rm", "-rf", "package/feeds/"]))
658
659 factory.addStep(StringDownload(
660 name = "ccachecc",
661 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
662 workerdest = "../ccache_cc.sh",
663 mode = 0o755,
664 ))
665
666 factory.addStep(StringDownload(
667 name = "ccachecxx",
668 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
669 workerdest = "../ccache_cxx.sh",
670 mode = 0o755,
671 ))
672
673 # feed
674 factory.addStep(ShellCommand(
675 name = "updatefeeds",
676 description = "Updating feeds",
677 command=["./scripts/feeds", "update"],
678 env = MakeEnv(tryccache=True),
679 haltOnFailure = True,
680 locks = NetLockDl,
681 ))
682
683 # feed
684 factory.addStep(ShellCommand(
685 name = "installfeeds",
686 description = "Installing feeds",
687 command=["./scripts/feeds", "install", "-a"],
688 env = MakeEnv(tryccache=True),
689 haltOnFailure = True
690 ))
691
692 # seed config
693 factory.addStep(StringDownload(
694 name = "dlconfigseed",
695 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
696 workerdest = ".config",
697 mode = 0o644
698 ))
699
700 # configure
701 factory.addStep(ShellCommand(
702 name = "newconfig",
703 description = "Seeding .config",
704 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey)
705 ))
706
707 factory.addStep(ShellCommand(
708 name = "delbin",
709 description = "Removing output directory",
710 command = ["rm", "-rf", "bin/"]
711 ))
712
713 factory.addStep(ShellCommand(
714 name = "defconfig",
715 description = "Populating .config",
716 command = ["make", "defconfig"],
717 env = MakeEnv()
718 ))
719
720 # check arch
721 factory.addStep(ShellCommand(
722 name = "checkarch",
723 description = "Checking architecture",
724 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
725 logEnviron = False,
726 want_stdout = False,
727 want_stderr = False,
728 haltOnFailure = True
729 ))
730
731 # find libc suffix
732 factory.addStep(SetPropertyFromCommand(
733 name = "libc",
734 property = "libc",
735 description = "Finding libc suffix",
736 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
737
738 # install build key
739 factory.addStep(StringDownload(
740 name = "dlkeybuildpub",
741 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
742 workerdest = "key-build.pub",
743 mode = 0o600,
744 doStepIf = IsUsignEnabled,
745 ))
746
747 factory.addStep(StringDownload(
748 name = "dlkeybuild",
749 s = "# fake private key",
750 workerdest = "key-build",
751 mode = 0o600,
752 doStepIf = IsUsignEnabled,
753 ))
754
755 factory.addStep(StringDownload(
756 name = "dlkeybuilducert",
757 s = "# fake certificate",
758 workerdest = "key-build.ucert",
759 mode = 0o600,
760 doStepIf = IsUsignEnabled,
761 ))
762
763 # prepare dl
764 factory.addStep(ShellCommand(
765 name = "dldir",
766 description = "Preparing dl/",
767 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
768 logEnviron = False,
769 want_stdout = False
770 ))
771
772 # prepare tar
773 factory.addStep(ShellCommand(
774 name = "dltar",
775 description = "Building and installing GNU tar",
776 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
777 env = MakeEnv(tryccache=True),
778 haltOnFailure = True
779 ))
780
781 # populate dl
782 factory.addStep(ShellCommand(
783 name = "dlrun",
784 description = "Populating dl/",
785 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
786 env = MakeEnv(),
787 logEnviron = False,
788 locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
789 ))
790
791 factory.addStep(ShellCommand(
792 name = "cleanbase",
793 description = "Cleaning base-files",
794 command=["make", "package/base-files/clean", "V=s"]
795 ))
796
797 # build
798 factory.addStep(ShellCommand(
799 name = "tools",
800 description = "Building and installing tools",
801 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
802 env = MakeEnv(tryccache=True),
803 haltOnFailure = True
804 ))
805
806 factory.addStep(ShellCommand(
807 name = "toolchain",
808 description = "Building and installing toolchain",
809 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
810 env = MakeEnv(),
811 haltOnFailure = True
812 ))
813
814 factory.addStep(ShellCommand(
815 name = "kmods",
816 description = "Building kmods",
817 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
818 env = MakeEnv(),
819 haltOnFailure = True
820 ))
821
822 # find kernel version
823 factory.addStep(SetPropertyFromCommand(
824 name = "kernelversion",
825 property = "kernelversion",
826 description = "Finding the effective Kernel version",
827 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
828 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") }
829 ))
830
831 factory.addStep(ShellCommand(
832 name = "pkgclean",
833 description = "Cleaning up package build",
834 command=["make", "package/cleanup", "V=s"]
835 ))
836
837 factory.addStep(ShellCommand(
838 name = "pkgbuild",
839 description = "Building packages",
840 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
841 env = MakeEnv(),
842 haltOnFailure = True
843 ))
844
845 factory.addStep(ShellCommand(
846 name = "pkginstall",
847 description = "Installing packages",
848 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
849 env = MakeEnv(),
850 haltOnFailure = True
851 ))
852
853 factory.addStep(ShellCommand(
854 name = "pkgindex",
855 description = "Indexing packages",
856 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
857 env = MakeEnv(),
858 haltOnFailure = True
859 ))
860
861 factory.addStep(ShellCommand(
862 name = "images",
863 description = "Building and installing images",
864 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
865 env = MakeEnv(),
866 haltOnFailure = True
867 ))
868
869 factory.addStep(ShellCommand(
870 name = "buildinfo",
871 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
872 command = "make -j1 buildinfo V=s || true",
873 env = MakeEnv(),
874 haltOnFailure = True
875 ))
876
877 factory.addStep(ShellCommand(
878 name = "json_overview_image_info",
879 description = "Generate profiles.json in target folder",
880 command = "make -j1 json_overview_image_info V=s || true",
881 env = MakeEnv(),
882 haltOnFailure = True
883 ))
884
885 factory.addStep(ShellCommand(
886 name = "checksums",
887 description = "Calculating checksums",
888 command=["make", "-j1", "checksum", "V=s"],
889 env = MakeEnv(),
890 haltOnFailure = True
891 ))
892
893 factory.addStep(ShellCommand(
894 name = "kmoddir",
895 description = "Creating kmod directory",
896 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
897 haltOnFailure = True,
898 doStepIf = IsKmodArchiveEnabled,
899 ))
900
901 factory.addStep(ShellCommand(
902 name = "kmodprepare",
903 description = "Preparing kmod archive",
904 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
905 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
906 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
907 haltOnFailure = True,
908 doStepIf = IsKmodArchiveEnabled,
909 ))
910
911 factory.addStep(ShellCommand(
912 name = "kmodindex",
913 description = "Indexing kmod archive",
914 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
915 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
916 env = MakeEnv(),
917 haltOnFailure = True,
918 doStepIf = IsKmodArchiveEnabled,
919 ))
920
921 # sign
922 factory.addStep(MasterShellCommand(
923 name = "signprepare",
924 description = "Preparing temporary signing directory",
925 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
926 haltOnFailure = True,
927 doStepIf = IsSignEnabled,
928
929 ))
930
931 factory.addStep(ShellCommand(
932 name = "signpack",
933 description = "Packing files to sign",
934 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
935 haltOnFailure = True,
936 doStepIf = IsSignEnabled,
937 ))
938
939 factory.addStep(FileUpload(
940 workersrc = "sign.tar.gz",
941 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
942 haltOnFailure = True,
943 doStepIf = IsSignEnabled,
944 ))
945
946 factory.addStep(MasterShellCommand(
947 name = "signfiles",
948 description = "Signing files",
949 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
950 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
951 haltOnFailure = True,
952 doStepIf = IsSignEnabled,
953 ))
954
955 factory.addStep(FileDownload(
956 name = "dlsigntargz",
957 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
958 workerdest = "sign.tar.gz",
959 haltOnFailure = True,
960 doStepIf = IsSignEnabled,
961 ))
962
963 factory.addStep(ShellCommand(
964 name = "signunpack",
965 description = "Unpacking signed files",
966 command = ["tar", "-xzf", "sign.tar.gz"],
967 haltOnFailure = True,
968 doStepIf = IsSignEnabled,
969 ))
970
971 # upload
972 factory.addStep(ShellCommand(
973 name = "dirprepare",
974 description = "Preparing upload directory structure",
975 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
976 haltOnFailure = True
977 ))
978
979 factory.addStep(ShellCommand(
980 name = "linkprepare",
981 description = "Preparing repository symlink",
982 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
983 doStepIf = IsNoMasterBuild,
984 haltOnFailure = True
985 ))
986
987 factory.addStep(ShellCommand(
988 name = "kmoddirprepare",
989 description = "Preparing kmod archive upload directory",
990 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
991 haltOnFailure = True,
992 doStepIf = IsKmodArchiveEnabled,
993 ))
994
995 factory.addStep(ShellCommand(
996 name = "dirupload",
997 description = "Uploading directory structure",
998 command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
999 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1000 haltOnFailure = True,
1001 logEnviron = False,
1002 locks = NetLockUl,
1003 ))
1004
1005 # download remote sha256sums to 'target-sha256sums'
1006 factory.addStep(ShellCommand(
1007 name = "target-sha256sums",
1008 description = "Fetching remote sha256sums for target",
1009 command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1010 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1011 logEnviron = False,
1012 haltOnFailure = False,
1013 flunkOnFailure = False,
1014 warnOnFailure = False,
1015 ))
1016
1017 # build list of files to upload
1018 factory.addStep(FileDownload(
1019 name = "dlsha2rsyncpl",
1020 mastersrc = scripts_dir + '/sha2rsync.pl',
1021 workerdest = "../sha2rsync.pl",
1022 mode = 0o755,
1023 ))
1024
1025 factory.addStep(ShellCommand(
1026 name = "buildlist",
1027 description = "Building list of files to upload",
1028 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1029 haltOnFailure = True,
1030 ))
1031
1032 factory.addStep(FileDownload(
1033 name = "dlrsync.sh",
1034 mastersrc = scripts_dir + '/rsync.sh',
1035 workerdest = "../rsync.sh",
1036 mode = 0o755
1037 ))
1038
1039 # upload new files and update existing ones
1040 factory.addStep(ShellCommand(
1041 name = "targetupload",
1042 description = "Uploading target files",
1043 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1044 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1045 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1046 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1047 haltOnFailure = True,
1048 logEnviron = False,
1049 ))
1050
1051 # delete files which don't exist locally
1052 factory.addStep(ShellCommand(
1053 name = "targetprune",
1054 description = "Pruning target files",
1055 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1056 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1057 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1058 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1059 haltOnFailure = True,
1060 logEnviron = False,
1061 locks = NetLockUl,
1062 ))
1063
1064 factory.addStep(ShellCommand(
1065 name = "kmodupload",
1066 description = "Uploading kmod archive",
1067 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1068 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1069 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1070 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1071 haltOnFailure = True,
1072 logEnviron = False,
1073 locks = NetLockUl,
1074 doStepIf = IsKmodArchiveEnabled,
1075 ))
1076
1077 factory.addStep(ShellCommand(
1078 name = "sourcelist",
1079 description = "Finding source archives to upload",
1080 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1081 haltOnFailure = True
1082 ))
1083
1084 factory.addStep(ShellCommand(
1085 name = "sourceupload",
1086 description = "Uploading source archives",
1087 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
1088 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1089 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1090 haltOnFailure = True,
1091 logEnviron = False,
1092 locks = NetLockUl,
1093 ))
1094
1095 factory.addStep(ShellCommand(
1096 name = "df",
1097 description = "Reporting disk usage",
1098 command=["df", "-h", "."],
1099 env={'LC_ALL': 'C'},
1100 haltOnFailure = False,
1101 flunkOnFailure = False,
1102 warnOnFailure = False,
1103 alwaysRun = True
1104 ))
1105
1106 factory.addStep(ShellCommand(
1107 name = "du",
1108 description = "Reporting estimated file space usage",
1109 command=["du", "-sh", "."],
1110 env={'LC_ALL': 'C'},
1111 haltOnFailure = False,
1112 flunkOnFailure = False,
1113 warnOnFailure = False,
1114 alwaysRun = True
1115 ))
1116
1117 factory.addStep(ShellCommand(
1118 name = "ccachestat",
1119 description = "Reporting ccache stats",
1120 command=["ccache", "-s"],
1121 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1122 want_stderr = False,
1123 haltOnFailure = False,
1124 flunkOnFailure = False,
1125 warnOnFailure = False,
1126 alwaysRun = True,
1127 ))
1128
1129 c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1130
1131 c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1132 force_factory.addStep(steps.Trigger(
1133 name = "trigger_%s" % target,
1134 description = "Triggering %s build" % target,
1135 schedulerNames = [ "trigger_%s" % target ],
1136 set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
1137 doStepIf = IsTargetSelected(target)
1138 ))
1139
1140
1141 ####### STATUS TARGETS
1142
1143 # 'status' is a list of Status Targets. The results of each build will be
1144 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1145 # including web pages, email senders, and IRC bots.
1146
1147 if "status_bind" in inip1:
1148 c['www'] = {
1149 'port': inip1.get("status_bind"),
1150 'plugins': {
1151 'waterfall_view': True,
1152 'console_view': True,
1153 'grid_view': True
1154 }
1155 }
1156
1157 if "status_user" in inip1 and "status_password" in inip1:
1158 c['www']['auth'] = util.UserPasswordAuth([
1159 (inip1.get("status_user"), inip1.get("status_password"))
1160 ])
1161 c['www']['authz'] = util.Authz(
1162 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1163 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1164 )
1165
1166 c['services'] = []
1167 if ini.has_section("irc"):
1168 iniirc = ini['irc']
1169 irc_host = iniirc.get("host", None)
1170 irc_port = iniirc.getint("port", 6667)
1171 irc_chan = iniirc.get("channel", None)
1172 irc_nick = iniirc.get("nickname", None)
1173 irc_pass = iniirc.get("password", None)
1174
1175 if irc_host and irc_nick and irc_chan:
1176 irc = reporters.IRC(irc_host, irc_nick,
1177 port = irc_port,
1178 password = irc_pass,
1179 channels = [ irc_chan ],
1180 notify_events = [ 'exception', 'problem', 'recovery' ]
1181 )
1182
1183 c['services'].append(irc)
1184
1185 c['revlink'] = util.RevlinkMatch([
1186 r'https://git.openwrt.org/openwrt/(.*).git'
1187 ],
1188 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1189
1190 ####### DB URL
1191
1192 c['db'] = {
1193 # This specifies what database buildbot uses to store its state. You can leave
1194 # this at its default for all but the largest installations.
1195 'db_url' : "sqlite:///state.sqlite",
1196 }
1197
1198 c['buildbotNetUsageData'] = None