ci: docker: deploy automatically only on default branch and tags
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import base64
7 import subprocess
8 import configparser
9
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
12
13 from twisted.internet import defer
14 from twisted.python import log
15
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes import filter
19 from buildbot.changes.gitpoller import GitPoller
20 from buildbot.config import BuilderConfig
21 from buildbot.plugins import reporters
22 from buildbot.plugins import schedulers
23 from buildbot.plugins import steps
24 from buildbot.plugins import util
25 from buildbot.process import properties
26 from buildbot.process import results
27 from buildbot.process.factory import BuildFactory
28 from buildbot.process.properties import Interpolate
29 from buildbot.process.properties import Property
30 from buildbot.schedulers.basic import SingleBranchScheduler
31 from buildbot.schedulers.forcesched import BaseParameter
32 from buildbot.schedulers.forcesched import ForceScheduler
33 from buildbot.schedulers.forcesched import ValidationError
34 from buildbot.steps.master import MasterShellCommand
35 from buildbot.steps.shell import SetPropertyFromCommand
36 from buildbot.steps.shell import ShellCommand
37 from buildbot.steps.source.git import Git
38 from buildbot.steps.transfer import FileDownload
39 from buildbot.steps.transfer import FileUpload
40 from buildbot.steps.transfer import StringDownload
41 from buildbot.worker import Worker
42
43
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
47
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
50
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
53
54 # This is the dictionary that the buildmaster pays attention to. We also use
55 # a shorter alias to save typing.
56 c = BuildmasterConfig = {}
57
58 ####### PROJECT IDENTITY
59
60 # the 'title' string will appear at the top of this buildbot
61 # installation's html.WebStatus home page (linked to the
62 # 'titleURL') and is embedded in the title of the waterfall HTML page.
63
64 c['title'] = ini.get("general", "title")
65 c['titleURL'] = ini.get("general", "title_url")
66
67 # the 'buildbotURL' string should point to the location where the buildbot's
68 # internal web server (usually the html.WebStatus page) is visible. This
69 # typically uses the port number set in the Waterfall 'status' entry, but
70 # with an externally-visible host name which the buildbot cannot figure out
71 # without some help.
72
73 c['buildbotURL'] = ini.get("phase1", "buildbot_url")
74
75 ####### BUILDWORKERS
76
77 # The 'workers' list defines the set of recognized buildworkers. Each element is
78 # a Worker object, specifying a unique worker name and password. The same
79 # worker name and password must be configured on the worker.
80
81 worker_port = 9989
82
83 if ini.has_option("phase1", "port"):
84 worker_port = ini.get("phase1", "port")
85
86 c['workers'] = []
87 NetLocks = dict()
88
89 for section in ini.sections():
90 if section.startswith("worker "):
91 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
92 (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
93 sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
94 name = ini.get(section, "name")
95 password = ini.get(section, "password")
96 max_builds = 1
97 if ini.has_option(section, "builds"):
98 max_builds = ini.getint(section, "builds")
99 sl_props['max_builds'] = max_builds
100 if max_builds == 1:
101 sl_props['shared_wd'] = True
102 if ini.has_option(section, "cleanup"):
103 sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
104 if ini.has_option(section, "dl_lock"):
105 lockname = ini.get(section, "dl_lock")
106 sl_props['dl_lock'] = lockname
107 if lockname not in NetLocks:
108 NetLocks[lockname] = locks.MasterLock(lockname)
109 if ini.has_option(section, "ul_lock"):
110 lockname = ini.get(section, "dl_lock")
111 sl_props['ul_lock'] = lockname
112 if lockname not in NetLocks:
113 NetLocks[lockname] = locks.MasterLock(lockname)
114 if ini.has_option(section, "shared_wd"):
115 shared_wd = ini.getboolean(section, "shared_wd")
116 sl_props['shared_wd'] = shared_wd
117 if shared_wd and (max_builds != 1):
118 raise ValueError('max_builds must be 1 with shared workdir!')
119 c['workers'].append(Worker(name, password, max_builds = max_builds, properties = sl_props))
120
121 # 'workerPortnum' defines the TCP port to listen on for connections from workers.
122 # This must match the value configured into the buildworkers (with their
123 # --master option)
124 c['protocols'] = {'pb': {'port': worker_port}}
125
126 # coalesce builds
127 c['collapseRequests'] = True
128
129 # Reduce amount of backlog data
130 c['configurators'] = [util.JanitorConfigurator(
131 logHorizon=timedelta(days=3),
132 hour=6,
133 )]
134
135 @defer.inlineCallbacks
136 def getNewestCompleteTime(bldr):
137 """Returns the complete_at of the latest completed and not SKIPPED
138 build request for this builder, or None if there are no such build
139 requests. We need to filter out SKIPPED requests because we're
140 using collapseRequests=True which is unfortunately marking all
141 previous requests as complete when new buildset is created.
142
143 @returns: datetime instance or None, via Deferred
144 """
145
146 bldrid = yield bldr.getBuilderId()
147 completed = yield bldr.master.data.get(
148 ('builders', bldrid, 'buildrequests'),
149 [
150 resultspec.Filter('complete', 'eq', [True]),
151 resultspec.Filter('results', 'ne', [results.SKIPPED]),
152 ],
153 order=['-complete_at'], limit=1)
154 if not completed:
155 return
156
157 return completed[0]['complete_at']
158
159 @defer.inlineCallbacks
160 def prioritizeBuilders(master, builders):
161 """Returns sorted list of builders by their last timestamp of completed and
162 not skipped build.
163
164 @returns: list of sorted builders
165 """
166
167 def is_building(bldr):
168 return bool(bldr.building) or bool(bldr.old_building)
169
170 def bldr_info(bldr):
171 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
172 d.addCallback(lambda complete_at: (complete_at, bldr))
173 return d
174
175 def bldr_sort(item):
176 (complete_at, bldr) = item
177
178 if not complete_at:
179 date = datetime.min
180 complete_at = date.replace(tzinfo=tzutc())
181
182 if is_building(bldr):
183 date = datetime.max
184 complete_at = date.replace(tzinfo=tzutc())
185
186 return (complete_at, bldr.name)
187
188 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
189 results.sort(key=bldr_sort)
190
191 for r in results:
192 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
193
194 return [r[1] for r in results]
195
196 c['prioritizeBuilders'] = prioritizeBuilders
197
198 ####### CHANGESOURCES
199
200 work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
201 scripts_dir = os.path.abspath("../scripts")
202 tree_expire = 0
203 other_builds = 0
204 cc_version = None
205
206 cc_command = "gcc"
207 cxx_command = "g++"
208
209 config_seed = ""
210
211 git_ssh = False
212 git_ssh_key = None
213
214 if ini.has_option("phase1", "expire"):
215 tree_expire = ini.getint("phase1", "expire")
216
217 if ini.has_option("phase1", "other_builds"):
218 other_builds = ini.getint("phase1", "other_builds")
219
220 if ini.has_option("phase1", "cc_version"):
221 cc_version = ini.get("phase1", "cc_version").split()
222 if len(cc_version) == 1:
223 cc_version = ["eq", cc_version[0]]
224
225 if ini.has_option("general", "git_ssh"):
226 git_ssh = ini.getboolean("general", "git_ssh")
227
228 if ini.has_option("general", "git_ssh_key"):
229 git_ssh_key = ini.get("general", "git_ssh_key")
230 else:
231 git_ssh = False
232
233 if ini.has_option("phase1", "config_seed"):
234 config_seed = ini.get("phase1", "config_seed")
235
236 repo_url = ini.get("repo", "url")
237 repo_branch = "master"
238
239 if ini.has_option("repo", "branch"):
240 repo_branch = ini.get("repo", "branch")
241
242 rsync_bin_url = ini.get("rsync", "binary_url")
243 rsync_bin_key = ini.get("rsync", "binary_password")
244 rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
245
246 if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
247 rsync_bin_defopts += ["--contimeout=20"]
248
249 rsync_src_url = None
250 rsync_src_key = None
251 rsync_src_defopts = ["-v", "-4", "--timeout=120"]
252
253 if ini.has_option("rsync", "source_url"):
254 rsync_src_url = ini.get("rsync", "source_url")
255 rsync_src_key = ini.get("rsync", "source_password")
256
257 if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
258 rsync_src_defopts += ["--contimeout=20"]
259
260 usign_key = None
261 usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
262
263 if ini.has_option("usign", "key"):
264 usign_key = ini.get("usign", "key")
265
266 if ini.has_option("usign", "comment"):
267 usign_comment = ini.get("usign", "comment")
268
269 enable_kmod_archive = False
270 embed_kmod_repository = False
271
272 if ini.has_option("phase1", "kmod_archive"):
273 enable_kmod_archive = ini.getboolean("phase1", "kmod_archive")
274
275 if ini.has_option("phase1", "kmod_repository"):
276 embed_kmod_repository = ini.getboolean("phase1", "kmod_repository")
277
278
279 # find targets
280 targets = [ ]
281
282 if not os.path.isdir(work_dir+'/source.git'):
283 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
284 else:
285 subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
286
287 os.makedirs(work_dir+'/source.git/tmp', exist_ok=True)
288 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
289 stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
290
291 while True:
292 line = findtargets.stdout.readline()
293 if not line:
294 break
295 ta = line.decode().strip().split(' ')
296 targets.append(ta[0])
297
298
299 # the 'change_source' setting tells the buildmaster how it should find out
300 # about source code changes. Here we point to the buildbot clone of pyflakes.
301
302 c['change_source'] = []
303 c['change_source'].append(GitPoller(
304 repo_url,
305 workdir=work_dir+'/work.git', branch=repo_branch,
306 pollinterval=300))
307
308 ####### SCHEDULERS
309
310 # Configure the Schedulers, which decide how to react to incoming changes. In this
311 # case, just kick off a 'basebuild' build
312
313 class TagChoiceParameter(BaseParameter):
314 spec_attributes = ["strict", "choices"]
315 type = "list"
316 strict = True
317
318 def __init__(self, name, label=None, **kw):
319 super().__init__(name, label, **kw)
320 self._choice_list = []
321
322 @property
323 def choices(self):
324 taglist = []
325 basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch)
326
327 if basever:
328 findtags = subprocess.Popen(
329 ['git', 'ls-remote', '--tags', repo_url],
330 stdout = subprocess.PIPE)
331
332 while True:
333 line = findtags.stdout.readline()
334
335 if not line:
336 break
337
338 tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
339
340 if tagver and tagver[1].find(basever[1]) == 0:
341 taglist.append(tagver[1])
342
343 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
344 taglist.insert(0, '')
345
346 self._choice_list = taglist
347
348 return self._choice_list
349
350 def parse_from_arg(self, s):
351 if self.strict and s not in self._choice_list:
352 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
353 return s
354
355 c['schedulers'] = []
356 c['schedulers'].append(SingleBranchScheduler(
357 name = "all",
358 change_filter = filter.ChangeFilter(branch=repo_branch),
359 treeStableTimer = 60,
360 builderNames = targets))
361
362 c['schedulers'].append(ForceScheduler(
363 name = "force",
364 buttonName = "Force builds",
365 label = "Force build details",
366 builderNames = [ "00_force_build" ],
367
368 codebases = [
369 util.CodebaseParameter(
370 "",
371 label = "Repository",
372 branch = util.FixedParameter(name = "branch", default = ""),
373 revision = util.FixedParameter(name = "revision", default = ""),
374 repository = util.FixedParameter(name = "repository", default = ""),
375 project = util.FixedParameter(name = "project", default = "")
376 )
377 ],
378
379 reason = util.StringParameter(
380 name = "reason",
381 label = "Reason",
382 default = "Trigger build",
383 required = True,
384 size = 80
385 ),
386
387 properties = [
388 util.NestedParameter(
389 name="options",
390 label="Build Options",
391 layout="vertical",
392 fields=[
393 util.ChoiceStringParameter(
394 name = "target",
395 label = "Build target",
396 default = "all",
397 choices = [ "all" ] + targets
398 ),
399 TagChoiceParameter(
400 name = "tag",
401 label = "Build tag",
402 default = ""
403 )
404 ]
405 )
406 ]
407 ))
408
409 ####### BUILDERS
410
411 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
412 # what steps, and which workers can execute them. Note that any particular build will
413 # only take place on one worker.
414
415 CleanTargetMap = [
416 [ "tools", "tools/clean" ],
417 [ "chain", "toolchain/clean" ],
418 [ "linux", "target/linux/clean" ],
419 [ "dir", "dirclean" ],
420 [ "dist", "distclean" ]
421 ]
422
423 def IsMakeCleanRequested(pattern):
424 def CheckCleanProperty(step):
425 val = step.getProperty("clean")
426 if val and re.match(pattern, val):
427 return True
428 else:
429 return False
430
431 return CheckCleanProperty
432
433 def IsSharedWorkdir(step):
434 return bool(step.getProperty("shared_wd"))
435
436 def IsCleanupRequested(step):
437 if IsSharedWorkdir(step):
438 return False
439 do_cleanup = step.getProperty("do_cleanup")
440 if do_cleanup:
441 return True
442 else:
443 return False
444
445 def IsExpireRequested(step):
446 if IsSharedWorkdir(step):
447 return False
448 else:
449 return not IsCleanupRequested(step)
450
451 def IsGitFreshRequested(step):
452 do_cleanup = step.getProperty("do_cleanup")
453 if do_cleanup:
454 return True
455 else:
456 return False
457
458 def IsGitCleanRequested(step):
459 return not IsGitFreshRequested(step)
460
461 def IsTaggingRequested(step):
462 val = step.getProperty("tag")
463 if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
464 return True
465 else:
466 return False
467
468 def IsNoTaggingRequested(step):
469 return not IsTaggingRequested(step)
470
471 def IsNoMasterBuild(step):
472 return repo_branch != "master"
473
474 def GetBaseVersion():
475 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
476 return repo_branch.split('-')[1]
477 else:
478 return "master"
479
480 @properties.renderer
481 def GetVersionPrefix(props):
482 basever = GetBaseVersion()
483 if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
484 return "%s/" % props["tag"]
485 elif basever != "master":
486 return "%s-SNAPSHOT/" % basever
487 else:
488 return ""
489
490 @properties.renderer
491 def GetNumJobs(props):
492 if props.hasProperty("max_builds") and props.hasProperty("nproc"):
493 return str(int(int(props["nproc"]) / (props["max_builds"] + other_builds)))
494 else:
495 return "1"
496
497 @properties.renderer
498 def GetCC(props):
499 if props.hasProperty("cc_command"):
500 return props["cc_command"]
501 else:
502 return "gcc"
503
504 @properties.renderer
505 def GetCXX(props):
506 if props.hasProperty("cxx_command"):
507 return props["cxx_command"]
508 else:
509 return "g++"
510
511 @properties.renderer
512 def GetCwd(props):
513 if props.hasProperty("builddir"):
514 return props["builddir"]
515 elif props.hasProperty("workdir"):
516 return props["workdir"]
517 else:
518 return "/"
519
520 @properties.renderer
521 def GetCCache(props):
522 if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
523 return props["ccache_command"]
524 else:
525 return ""
526
527 def GetNextBuild(builder, requests):
528 for r in requests:
529 if r.properties and r.properties.hasProperty("tag"):
530 return r
531
532 r = requests[0]
533 log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
534 return r
535
536 def MakeEnv(overrides=None, tryccache=False):
537 env = {
538 'CCC': Interpolate("%(kw:cc)s", cc=GetCC),
539 'CCXX': Interpolate("%(kw:cxx)s", cxx=GetCXX),
540 }
541 if tryccache:
542 env['CC'] = Interpolate("%(kw:cwd)s/ccache_cc.sh", cwd=GetCwd)
543 env['CXX'] = Interpolate("%(kw:cwd)s/ccache_cxx.sh", cwd=GetCwd)
544 env['CCACHE'] = Interpolate("%(kw:ccache)s", ccache=GetCCache)
545 else:
546 env['CC'] = env['CCC']
547 env['CXX'] = env['CCXX']
548 env['CCACHE'] = ''
549 if overrides is not None:
550 env.update(overrides)
551 return env
552
553 @properties.renderer
554 def NetLockDl(props):
555 lock = None
556 if props.hasProperty("dl_lock"):
557 lock = NetLocks[props["dl_lock"]]
558 if lock is not None:
559 return [lock.access('exclusive')]
560 else:
561 return []
562
563 @properties.renderer
564 def NetLockUl(props):
565 lock = None
566 if props.hasProperty("ul_lock"):
567 lock = NetLocks[props["ul_lock"]]
568 if lock is not None:
569 return [lock.access('exclusive')]
570 else:
571 return []
572
573 @util.renderer
574 def TagPropertyValue(props):
575 if props.hasProperty("options"):
576 options = props.getProperty("options")
577 if type(options) is dict:
578 return options.get("tag")
579 return None
580
581 def IsTargetSelected(target):
582 def CheckTargetProperty(step):
583 try:
584 options = step.getProperty("options")
585 if type(options) is dict:
586 selected_target = options.get("target", "all")
587 if selected_target != "all" and selected_target != target:
588 return False
589 except KeyError:
590 pass
591
592 return True
593
594 return CheckTargetProperty
595
596 def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
597 try:
598 seckey = base64.b64decode(seckey)
599 except:
600 return None
601
602 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
603 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
604
605
606 c['builders'] = []
607
608 dlLock = locks.WorkerLock("worker_dl")
609
610 checkBuiltin = re.sub('[\t\n ]+', ' ', """
611 checkBuiltin() {
612 local symbol op path file;
613 for file in $CHANGED_FILES; do
614 case "$file" in
615 package/*/*) : ;;
616 *) return 0 ;;
617 esac;
618 done;
619 while read symbol op path; do
620 case "$symbol" in package-*)
621 symbol="${symbol##*(}";
622 symbol="${symbol%)}";
623 for file in $CHANGED_FILES; do
624 case "$file" in "package/$path/"*)
625 grep -qsx "$symbol=y" .config && return 0
626 ;; esac;
627 done;
628 esac;
629 done < tmp/.packagedeps;
630 return 1;
631 }
632 """).strip()
633
634
635 class IfBuiltinShellCommand(ShellCommand):
636 def _quote(self, str):
637 if re.search("[^a-zA-Z0-9/_.-]", str):
638 return "'%s'" %(re.sub("'", "'\"'\"'", str))
639 return str
640
641 def setCommand(self, command):
642 if not isinstance(command, (str, unicode)):
643 command = ' '.join(map(self._quote, command))
644 self.command = [
645 '/bin/sh', '-c',
646 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
647 ]
648
649 def setupEnvironment(self, cmd):
650 workerEnv = self.workerEnvironment
651 if workerEnv is None:
652 workerEnv = { }
653 changedFiles = { }
654 for request in self.build.requests:
655 for source in request.sources:
656 for change in source.changes:
657 for file in change.files:
658 changedFiles[file] = True
659 fullSlaveEnv = workerEnv.copy()
660 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
661 cmd.args['env'] = fullSlaveEnv
662
663 workerNames = [ ]
664
665 for worker in c['workers']:
666 workerNames.append(worker.workername)
667
668 force_factory = BuildFactory()
669
670 c['builders'].append(BuilderConfig(
671 name = "00_force_build",
672 workernames = workerNames,
673 factory = force_factory))
674
675 for target in targets:
676 ts = target.split('/')
677
678 factory = BuildFactory()
679
680 # setup shared work directory if required
681 factory.addStep(ShellCommand(
682 name = "sharedwd",
683 description = "Setting up shared work directory",
684 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
685 workdir = ".",
686 haltOnFailure = True,
687 doStepIf = IsSharedWorkdir))
688
689 # find number of cores
690 factory.addStep(SetPropertyFromCommand(
691 name = "nproc",
692 property = "nproc",
693 description = "Finding number of CPUs",
694 command = ["nproc"]))
695
696 # find gcc and g++ compilers
697 factory.addStep(FileDownload(
698 name = "dlfindbinpl",
699 mastersrc = scripts_dir + '/findbin.pl',
700 workerdest = "../findbin.pl",
701 mode = 0o755))
702
703 factory.addStep(SetPropertyFromCommand(
704 name = "gcc",
705 property = "cc_command",
706 description = "Finding gcc command",
707 command = [
708 "../findbin.pl", "gcc",
709 cc_version[0] if cc_version is not None else '',
710 cc_version[1] if cc_version is not None else ''
711 ],
712 haltOnFailure = True))
713
714 factory.addStep(SetPropertyFromCommand(
715 name = "g++",
716 property = "cxx_command",
717 description = "Finding g++ command",
718 command = [
719 "../findbin.pl", "g++",
720 cc_version[0] if cc_version is not None else '',
721 cc_version[1] if cc_version is not None else ''
722 ],
723 haltOnFailure = True))
724
725 # see if ccache is available
726 factory.addStep(SetPropertyFromCommand(
727 property = "ccache_command",
728 command = ["which", "ccache"],
729 description = "Testing for ccache command",
730 haltOnFailure = False,
731 flunkOnFailure = False,
732 warnOnFailure = False,
733 ))
734
735 # expire tree if needed
736 if tree_expire > 0:
737 factory.addStep(FileDownload(
738 name = "dlexpiresh",
739 doStepIf = IsExpireRequested,
740 mastersrc = scripts_dir + '/expire.sh',
741 workerdest = "../expire.sh",
742 mode = 0o755))
743
744 factory.addStep(ShellCommand(
745 name = "expire",
746 description = "Checking for build tree expiry",
747 command = ["./expire.sh", str(tree_expire)],
748 workdir = ".",
749 haltOnFailure = True,
750 doStepIf = IsExpireRequested,
751 timeout = 2400))
752
753 # cleanup.sh if needed
754 factory.addStep(FileDownload(
755 name = "dlcleanupsh",
756 mastersrc = scripts_dir + '/cleanup.sh',
757 workerdest = "../cleanup.sh",
758 mode = 0o755,
759 doStepIf = IsCleanupRequested))
760
761 factory.addStep(ShellCommand(
762 name = "cleanold",
763 description = "Cleaning previous builds",
764 command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "full"],
765 workdir = ".",
766 haltOnFailure = True,
767 doStepIf = IsCleanupRequested,
768 timeout = 2400))
769
770 factory.addStep(ShellCommand(
771 name = "cleanup",
772 description = "Cleaning work area",
773 command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "single"],
774 workdir = ".",
775 haltOnFailure = True,
776 doStepIf = IsCleanupRequested,
777 timeout = 2400))
778
779 # user-requested clean targets
780 for tuple in CleanTargetMap:
781 factory.addStep(ShellCommand(
782 name = tuple[1],
783 description = 'User-requested "make %s"' % tuple[1],
784 command = ["make", tuple[1], "V=s"],
785 env = MakeEnv(),
786 doStepIf = IsMakeCleanRequested(tuple[0])
787 ))
788
789 # Workaround bug when switching from a checked out tag back to a branch
790 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
791 factory.addStep(ShellCommand(
792 name = "gitcheckout",
793 description = "Ensure that Git HEAD is sane",
794 command = "if [ -d .git ]; then git checkout -f %s && git branch --set-upstream-to origin/%s || rm -fr .git; else exit 0; fi" %(repo_branch, repo_branch),
795 haltOnFailure = True))
796
797 # check out the source
798 # Git() runs:
799 # if repo doesn't exist: 'git clone repourl'
800 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
801 # 'git fetch -t repourl branch; git reset --hard revision'
802 # Git() parameters can't take a renderer until buildbot 0.8.10, so we have to split the fresh and clean cases
803 # if buildbot is updated, one can use: method = Interpolate('%(prop:do_cleanup:#?|fresh|clean)s')
804 factory.addStep(Git(
805 name = "gitclean",
806 repourl = repo_url,
807 branch = repo_branch,
808 mode = 'full',
809 method = 'clean',
810 haltOnFailure = True,
811 doStepIf = IsGitCleanRequested,
812 ))
813
814 factory.addStep(Git(
815 name = "gitfresh",
816 repourl = repo_url,
817 branch = repo_branch,
818 mode = 'full',
819 method = 'fresh',
820 haltOnFailure = True,
821 doStepIf = IsGitFreshRequested,
822 ))
823
824 # update remote refs
825 factory.addStep(ShellCommand(
826 name = "fetchrefs",
827 description = "Fetching Git remote refs",
828 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
829 haltOnFailure = True
830 ))
831
832 # switch to tag
833 factory.addStep(ShellCommand(
834 name = "switchtag",
835 description = "Checking out Git tag",
836 command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
837 haltOnFailure = True,
838 doStepIf = IsTaggingRequested
839 ))
840
841 # Verify that Git HEAD points to a tag or branch
842 # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
843 factory.addStep(ShellCommand(
844 name = "gitverify",
845 description = "Ensure that Git HEAD is pointing to a branch or tag",
846 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
847 haltOnFailure = True))
848
849 factory.addStep(ShellCommand(
850 name = "rmtmp",
851 description = "Remove tmp folder",
852 command=["rm", "-rf", "tmp/"]))
853
854 # feed
855 # factory.addStep(ShellCommand(
856 # name = "feedsconf",
857 # description = "Copy the feeds.conf",
858 # command='''cp ~/feeds.conf ./feeds.conf''' ))
859
860 # feed
861 factory.addStep(ShellCommand(
862 name = "rmfeedlinks",
863 description = "Remove feed symlinks",
864 command=["rm", "-rf", "package/feeds/"]))
865
866 factory.addStep(StringDownload(
867 name = "ccachecc",
868 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
869 workerdest = "../ccache_cc.sh",
870 mode = 0o755,
871 ))
872
873 factory.addStep(StringDownload(
874 name = "ccachecxx",
875 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
876 workerdest = "../ccache_cxx.sh",
877 mode = 0o755,
878 ))
879
880 # Git SSH
881 if git_ssh:
882 factory.addStep(StringDownload(
883 name = "dlgitclonekey",
884 s = git_ssh_key,
885 workerdest = "../git-clone.key",
886 mode = 0o600,
887 ))
888
889 factory.addStep(ShellCommand(
890 name = "patchfeedsconf",
891 description = "Patching feeds.conf",
892 command="sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf",
893 haltOnFailure = True
894 ))
895
896 # feed
897 factory.addStep(ShellCommand(
898 name = "updatefeeds",
899 description = "Updating feeds",
900 command=["./scripts/feeds", "update"],
901 env = MakeEnv(tryccache=True, overrides={'GIT_SSH_COMMAND': Interpolate("ssh -o IdentitiesOnly=yes -o IdentityFile=%(kw:cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {}),
902 haltOnFailure = True
903 ))
904
905 # Git SSH
906 if git_ssh:
907 factory.addStep(ShellCommand(
908 name = "rmfeedsconf",
909 description = "Removing feeds.conf",
910 command=["rm", "feeds.conf"],
911 haltOnFailure = True
912 ))
913
914 # feed
915 factory.addStep(ShellCommand(
916 name = "installfeeds",
917 description = "Installing feeds",
918 command=["./scripts/feeds", "install", "-a"],
919 env = MakeEnv(tryccache=True),
920 haltOnFailure = True
921 ))
922
923 # seed config
924 if config_seed is not None:
925 factory.addStep(StringDownload(
926 name = "dlconfigseed",
927 s = config_seed + '\n',
928 workerdest = ".config",
929 mode = 0o644
930 ))
931
932 # configure
933 factory.addStep(ShellCommand(
934 name = "newconfig",
935 description = "Seeding .config",
936 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\nCONFIG_SIGNED_PACKAGES=%s\\n' >> .config" %(ts[0], ts[0], ts[1], 'y' if usign_key is not None else 'n')
937 ))
938
939 factory.addStep(ShellCommand(
940 name = "delbin",
941 description = "Removing output directory",
942 command = ["rm", "-rf", "bin/"]
943 ))
944
945 factory.addStep(ShellCommand(
946 name = "defconfig",
947 description = "Populating .config",
948 command = ["make", "defconfig"],
949 env = MakeEnv()
950 ))
951
952 # check arch
953 factory.addStep(ShellCommand(
954 name = "checkarch",
955 description = "Checking architecture",
956 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
957 logEnviron = False,
958 want_stdout = False,
959 want_stderr = False,
960 haltOnFailure = True
961 ))
962
963 # find libc suffix
964 factory.addStep(SetPropertyFromCommand(
965 name = "libc",
966 property = "libc",
967 description = "Finding libc suffix",
968 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
969
970 # install build key
971 if usign_key is not None:
972 factory.addStep(StringDownload(
973 name = "dlkeybuildpub",
974 s = UsignSec2Pub(usign_key, usign_comment),
975 workerdest = "key-build.pub",
976 mode = 0o600,
977 ))
978
979 factory.addStep(StringDownload(
980 name = "dlkeybuild",
981 s = "# fake private key",
982 workerdest = "key-build",
983 mode = 0o600,
984 ))
985
986 factory.addStep(StringDownload(
987 name = "dlkeybuilducert",
988 s = "# fake certificate",
989 workerdest = "key-build.ucert",
990 mode = 0o600,
991 ))
992
993 # prepare dl
994 factory.addStep(ShellCommand(
995 name = "dldir",
996 description = "Preparing dl/",
997 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
998 logEnviron = False,
999 want_stdout = False
1000 ))
1001
1002 # prepare tar
1003 factory.addStep(ShellCommand(
1004 name = "dltar",
1005 description = "Building and installing GNU tar",
1006 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
1007 env = MakeEnv(tryccache=True),
1008 haltOnFailure = True
1009 ))
1010
1011 # populate dl
1012 factory.addStep(ShellCommand(
1013 name = "dlrun",
1014 description = "Populating dl/",
1015 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
1016 env = MakeEnv(),
1017 logEnviron = False,
1018 locks = [dlLock.access('exclusive')],
1019 ))
1020
1021 factory.addStep(ShellCommand(
1022 name = "cleanbase",
1023 description = "Cleaning base-files",
1024 command=["make", "package/base-files/clean", "V=s"]
1025 ))
1026
1027 # build
1028 factory.addStep(ShellCommand(
1029 name = "tools",
1030 description = "Building and installing tools",
1031 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
1032 env = MakeEnv(tryccache=True),
1033 haltOnFailure = True
1034 ))
1035
1036 factory.addStep(ShellCommand(
1037 name = "toolchain",
1038 description = "Building and installing toolchain",
1039 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
1040 env = MakeEnv(),
1041 haltOnFailure = True
1042 ))
1043
1044 factory.addStep(ShellCommand(
1045 name = "kmods",
1046 description = "Building kmods",
1047 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
1048 env = MakeEnv(),
1049 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
1050 haltOnFailure = True
1051 ))
1052
1053 # find kernel version
1054 factory.addStep(SetPropertyFromCommand(
1055 name = "kernelversion",
1056 property = "kernelversion",
1057 description = "Finding the effective Kernel version",
1058 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
1059 env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
1060 ))
1061
1062 factory.addStep(ShellCommand(
1063 name = "pkgclean",
1064 description = "Cleaning up package build",
1065 command=["make", "package/cleanup", "V=s"]
1066 ))
1067
1068 factory.addStep(ShellCommand(
1069 name = "pkgbuild",
1070 description = "Building packages",
1071 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
1072 env = MakeEnv(),
1073 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
1074 haltOnFailure = True
1075 ))
1076
1077 # factory.addStep(IfBuiltinShellCommand(
1078 factory.addStep(ShellCommand(
1079 name = "pkginstall",
1080 description = "Installing packages",
1081 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
1082 env = MakeEnv(),
1083 haltOnFailure = True
1084 ))
1085
1086 factory.addStep(ShellCommand(
1087 name = "pkgindex",
1088 description = "Indexing packages",
1089 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
1090 env = MakeEnv(),
1091 haltOnFailure = True
1092 ))
1093
1094 if enable_kmod_archive and embed_kmod_repository:
1095 # embed kmod repository. Must happen before 'images'
1096
1097 # find rootfs staging directory
1098 factory.addStep(SetPropertyFromCommand(
1099 name = "stageroot",
1100 property = "stageroot",
1101 description = "Finding the rootfs staging directory",
1102 command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
1103 env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) },
1104 want_stderr = False
1105 ))
1106
1107 factory.addStep(ShellCommand(
1108 name = "filesdir",
1109 description = "Creating file overlay directory",
1110 command=["mkdir", "-p", "files/etc/opkg"],
1111 haltOnFailure = True
1112 ))
1113
1114 factory.addStep(ShellCommand(
1115 name = "kmodconfig",
1116 description = "Embedding kmod repository configuration",
1117 command=Interpolate("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(prop:kernelversion)s#' " +
1118 "%(prop:stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
1119 haltOnFailure = True
1120 ))
1121
1122 #factory.addStep(IfBuiltinShellCommand(
1123 factory.addStep(ShellCommand(
1124 name = "images",
1125 description = "Building and installing images",
1126 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
1127 env = MakeEnv(),
1128 haltOnFailure = True
1129 ))
1130
1131 factory.addStep(ShellCommand(
1132 name = "buildinfo",
1133 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
1134 command = "make -j1 buildinfo V=s || true",
1135 env = MakeEnv(),
1136 haltOnFailure = True
1137 ))
1138
1139 factory.addStep(ShellCommand(
1140 name = "json_overview_image_info",
1141 description = "Generate profiles.json in target folder",
1142 command = "make -j1 json_overview_image_info V=s || true",
1143 env = MakeEnv(),
1144 haltOnFailure = True
1145 ))
1146
1147 factory.addStep(ShellCommand(
1148 name = "checksums",
1149 description = "Calculating checksums",
1150 command=["make", "-j1", "checksum", "V=s"],
1151 env = MakeEnv(),
1152 haltOnFailure = True
1153 ))
1154
1155 if enable_kmod_archive:
1156 factory.addStep(ShellCommand(
1157 name = "kmoddir",
1158 description = "Creating kmod directory",
1159 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
1160 haltOnFailure = True
1161 ))
1162
1163 factory.addStep(ShellCommand(
1164 name = "kmodprepare",
1165 description = "Preparing kmod archive",
1166 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
1167 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
1168 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1169 haltOnFailure = True
1170 ))
1171
1172 factory.addStep(ShellCommand(
1173 name = "kmodindex",
1174 description = "Indexing kmod archive",
1175 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
1176 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1177 env = MakeEnv(),
1178 haltOnFailure = True
1179 ))
1180
1181 # sign
1182 if ini.has_option("gpg", "key") or usign_key is not None:
1183 factory.addStep(MasterShellCommand(
1184 name = "signprepare",
1185 description = "Preparing temporary signing directory",
1186 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1187 haltOnFailure = True
1188 ))
1189
1190 factory.addStep(ShellCommand(
1191 name = "signpack",
1192 description = "Packing files to sign",
1193 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1194 haltOnFailure = True
1195 ))
1196
1197 factory.addStep(FileUpload(
1198 workersrc = "sign.tar.gz",
1199 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1200 haltOnFailure = True
1201 ))
1202
1203 factory.addStep(MasterShellCommand(
1204 name = "signfiles",
1205 description = "Signing files",
1206 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
1207 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1208 haltOnFailure = True
1209 ))
1210
1211 factory.addStep(FileDownload(
1212 name = "dlsigntargz",
1213 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1214 workerdest = "sign.tar.gz",
1215 haltOnFailure = True
1216 ))
1217
1218 factory.addStep(ShellCommand(
1219 name = "signunpack",
1220 description = "Unpacking signed files",
1221 command = ["tar", "-xzf", "sign.tar.gz"],
1222 haltOnFailure = True
1223 ))
1224
1225 # upload
1226 factory.addStep(ShellCommand(
1227 name = "dirprepare",
1228 description = "Preparing upload directory structure",
1229 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1230 haltOnFailure = True
1231 ))
1232
1233 factory.addStep(ShellCommand(
1234 name = "linkprepare",
1235 description = "Preparing repository symlink",
1236 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=GetBaseVersion()), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1237 doStepIf = IsNoMasterBuild,
1238 haltOnFailure = True
1239 ))
1240
1241 if enable_kmod_archive:
1242 factory.addStep(ShellCommand(
1243 name = "kmoddirprepare",
1244 description = "Preparing kmod archive upload directory",
1245 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1246 haltOnFailure = True
1247 ))
1248
1249 factory.addStep(ShellCommand(
1250 name = "dirupload",
1251 description = "Uploading directory structure",
1252 command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
1253 env={'RSYNC_PASSWORD': rsync_bin_key},
1254 haltOnFailure = True,
1255 logEnviron = False,
1256 ))
1257
1258 # download remote sha256sums to 'target-sha256sums'
1259 factory.addStep(ShellCommand(
1260 name = "target-sha256sums",
1261 description = "Fetching remote sha256sums for target",
1262 command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1263 env={'RSYNC_PASSWORD': rsync_bin_key},
1264 logEnviron = False,
1265 haltOnFailure = False,
1266 flunkOnFailure = False,
1267 warnOnFailure = False,
1268 ))
1269
1270 # build list of files to upload
1271 factory.addStep(FileDownload(
1272 name = "dlsha2rsyncpl",
1273 mastersrc = scripts_dir + '/sha2rsync.pl',
1274 workerdest = "../sha2rsync.pl",
1275 mode = 0o755,
1276 ))
1277
1278 factory.addStep(ShellCommand(
1279 name = "buildlist",
1280 description = "Building list of files to upload",
1281 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1282 haltOnFailure = True,
1283 ))
1284
1285 factory.addStep(FileDownload(
1286 name = "dlrsync.sh",
1287 mastersrc = scripts_dir + '/rsync.sh',
1288 workerdest = "../rsync.sh",
1289 mode = 0o755
1290 ))
1291
1292 # upload new files and update existing ones
1293 factory.addStep(ShellCommand(
1294 name = "targetupload",
1295 description = "Uploading target files",
1296 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1297 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1298 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1299 env={'RSYNC_PASSWORD': rsync_bin_key},
1300 haltOnFailure = True,
1301 logEnviron = False,
1302 ))
1303
1304 # delete files which don't exist locally
1305 factory.addStep(ShellCommand(
1306 name = "targetprune",
1307 description = "Pruning target files",
1308 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1309 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1310 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1311 env={'RSYNC_PASSWORD': rsync_bin_key},
1312 haltOnFailure = True,
1313 logEnviron = False,
1314 ))
1315
1316 if enable_kmod_archive:
1317 factory.addStep(ShellCommand(
1318 name = "kmodupload",
1319 description = "Uploading kmod archive",
1320 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
1321 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1322 Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1323 env={'RSYNC_PASSWORD': rsync_bin_key},
1324 haltOnFailure = True,
1325 logEnviron = False,
1326 ))
1327
1328 if rsync_src_url is not None:
1329 factory.addStep(ShellCommand(
1330 name = "sourcelist",
1331 description = "Finding source archives to upload",
1332 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1333 haltOnFailure = True
1334 ))
1335
1336 factory.addStep(ShellCommand(
1337 name = "sourceupload",
1338 description = "Uploading source archives",
1339 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
1340 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
1341 env={'RSYNC_PASSWORD': rsync_src_key},
1342 haltOnFailure = True,
1343 logEnviron = False,
1344 ))
1345
1346 if False:
1347 factory.addStep(ShellCommand(
1348 name = "packageupload",
1349 description = "Uploading package files",
1350 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a"] + rsync_bin_defopts + ["bin/packages/", "%s/packages/" %(rsync_bin_url)],
1351 env={'RSYNC_PASSWORD': rsync_bin_key},
1352 haltOnFailure = False,
1353 flunkOnFailure = False,
1354 warnOnFailure = True,
1355 logEnviron = False,
1356 ))
1357
1358 # logs
1359 if False:
1360 factory.addStep(ShellCommand(
1361 name = "upload",
1362 description = "Uploading logs",
1363 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az"] + rsync_bin_defopts + ["logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
1364 env={'RSYNC_PASSWORD': rsync_bin_key},
1365 haltOnFailure = False,
1366 flunkOnFailure = False,
1367 warnOnFailure = True,
1368 alwaysRun = True,
1369 logEnviron = False,
1370 ))
1371
1372 factory.addStep(ShellCommand(
1373 name = "df",
1374 description = "Reporting disk usage",
1375 command=["df", "-h", "."],
1376 env={'LC_ALL': 'C'},
1377 haltOnFailure = False,
1378 flunkOnFailure = False,
1379 warnOnFailure = False,
1380 alwaysRun = True
1381 ))
1382
1383 factory.addStep(ShellCommand(
1384 name = "du",
1385 description = "Reporting estimated file space usage",
1386 command=["du", "-sh", "."],
1387 env={'LC_ALL': 'C'},
1388 haltOnFailure = False,
1389 flunkOnFailure = False,
1390 warnOnFailure = False,
1391 alwaysRun = True
1392 ))
1393
1394 factory.addStep(ShellCommand(
1395 name = "ccachestat",
1396 description = "Reporting ccache stats",
1397 command=["ccache", "-s"],
1398 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1399 want_stderr = False,
1400 haltOnFailure = False,
1401 flunkOnFailure = False,
1402 warnOnFailure = False,
1403 alwaysRun = True,
1404 ))
1405
1406 c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1407
1408 c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1409 force_factory.addStep(steps.Trigger(
1410 name = "trigger_%s" % target,
1411 description = "Triggering %s build" % target,
1412 schedulerNames = [ "trigger_%s" % target ],
1413 set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
1414 doStepIf = IsTargetSelected(target)
1415 ))
1416
1417
1418 ####### STATUS TARGETS
1419
1420 # 'status' is a list of Status Targets. The results of each build will be
1421 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1422 # including web pages, email senders, and IRC bots.
1423
1424 if ini.has_option("phase1", "status_bind"):
1425 c['www'] = {
1426 'port': ini.get("phase1", "status_bind"),
1427 'plugins': {
1428 'waterfall_view': True,
1429 'console_view': True,
1430 'grid_view': True
1431 }
1432 }
1433
1434 if ini.has_option("phase1", "status_user") and ini.has_option("phase1", "status_password"):
1435 c['www']['auth'] = util.UserPasswordAuth([
1436 (ini.get("phase1", "status_user"), ini.get("phase1", "status_password"))
1437 ])
1438 c['www']['authz'] = util.Authz(
1439 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1440 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[ini.get("phase1", "status_user")]) ]
1441 )
1442
1443 c['services'] = []
1444 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
1445 irc_host = ini.get("irc", "host")
1446 irc_port = 6667
1447 irc_chan = ini.get("irc", "channel")
1448 irc_nick = ini.get("irc", "nickname")
1449 irc_pass = None
1450
1451 if ini.has_option("irc", "port"):
1452 irc_port = ini.getint("irc", "port")
1453
1454 if ini.has_option("irc", "password"):
1455 irc_pass = ini.get("irc", "password")
1456
1457 irc = reporters.IRC(irc_host, irc_nick,
1458 port = irc_port,
1459 password = irc_pass,
1460 channels = [ irc_chan ],
1461 notify_events = [ 'exception', 'problem', 'recovery' ]
1462 )
1463
1464 c['services'].append(irc)
1465
1466 c['revlink'] = util.RevlinkMatch([
1467 r'https://git.openwrt.org/openwrt/(.*).git'
1468 ],
1469 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1470
1471 ####### DB URL
1472
1473 c['db'] = {
1474 # This specifies what database buildbot uses to store its state. You can leave
1475 # this at its default for all but the largest installations.
1476 'db_url' : "sqlite:///state.sqlite",
1477 }
1478
1479 c['buildbotNetUsageData'] = None