phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41
  42
  43 if not os.path.exists("twistd.pid"):
  44         with open("twistd.pid", "w") as pidfile:
  45                 pidfile.write("{}".format(os.getpid()))
  46
  47 # This is a sample buildmaster config file. It must be installed as
  48 # 'master.cfg' in your buildmaster's base directory.
  49
  50 ini = configparser.ConfigParser()
  51 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  52
  53 if "general" not in ini or "phase1" not in ini:
  54         raise ValueError("Fix your configuration")
  55
  56 inip1 = ini['phase1']
  57
  58 # Globals
  59 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
  60 scripts_dir = os.path.abspath("../scripts")
  61
  62 repo_url = ini['repo'].get("url")
  63
  64 rsync_defopts = ["-v", "-4", "--timeout=120"]
  65
  66 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  67 #       rsync_bin_defopts += ["--contimeout=20"]
  68
  69 branches = {}
  70
  71 def ini_parse_branch(section):
  72         b = {}
  73         name = section.get("name")
  74
  75         if not name:
  76                 raise ValueError("missing 'name' in " + repr(section))
  77         if name in branches:
  78                 raise ValueError("duplicate branch name in " + repr(section))
  79
  80         b["name"] = name
  81         b["bin_url"] = section.get("binary_url")
  82         b["bin_key"] = section.get("binary_password")
  83
  84         b["src_url"] = section.get("source_url")
  85         b["src_key"] = section.get("source_password")
  86
  87         b["gpg_key"] = section.get("gpg_key")
  88
  89         b["usign_key"] = section.get("usign_key")
  90         usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  91         b["usign_comment"] = section.get("usign_comment", usign_comment)
  92
  93         b["config_seed"] = section.get("config_seed")
  94
  95         b["kmod_archive"] = section.getboolean("kmod_archive", False)
  96
  97         branches[name] = b
  98         log.msg("Configured branch: {}".format(name))
  99
 100 # PB port can be either a numeric port or a connection string
 101 pb_port = inip1.get("port") or 9989
 102
 103 # This is the dictionary that the buildmaster pays attention to. We also use
 104 # a shorter alias to save typing.
 105 c = BuildmasterConfig = {}
 106
 107 ####### PROJECT IDENTITY
 108
 109 # the 'title' string will appear at the top of this buildbot
 110 # installation's html.WebStatus home page (linked to the
 111 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 112
 113 c['title'] = ini['general'].get("title")
 114 c['titleURL'] = ini['general'].get("title_url")
 115
 116 # the 'buildbotURL' string should point to the location where the buildbot's
 117 # internal web server (usually the html.WebStatus page) is visible. This
 118 # typically uses the port number set in the Waterfall 'status' entry, but
 119 # with an externally-visible host name which the buildbot cannot figure out
 120 # without some help.
 121
 122 c['buildbotURL'] = inip1.get("buildbot_url")
 123
 124 ####### BUILDWORKERS
 125
 126 # The 'workers' list defines the set of recognized buildworkers. Each element is
 127 # a Worker object, specifying a unique worker name and password.  The same
 128 # worker name and password must be configured on the worker.
 129
 130 c['workers'] = []
 131 NetLocks = dict()
 132
 133 for section in ini.sections():
 134         if section.startswith("branch "):
 135                 ini_parse_branch(ini[section])
 136
 137         if section.startswith("worker "):
 138                 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
 139                    (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
 140                         sl_props = { 'dl_lock':None, 'ul_lock':None }
 141                         name = ini.get(section, "name")
 142                         password = ini.get(section, "password")
 143                         if ini.has_option(section, "dl_lock"):
 144                                 lockname = ini.get(section, "dl_lock")
 145                                 sl_props['dl_lock'] = lockname
 146                                 if lockname not in NetLocks:
 147                                         NetLocks[lockname] = locks.MasterLock(lockname)
 148                         if ini.has_option(section, "ul_lock"):
 149                                 lockname = ini.get(section, "ul_lock")
 150                                 sl_props['ul_lock'] = lockname
 151                                 if lockname not in NetLocks:
 152                                         NetLocks[lockname] = locks.MasterLock(lockname)
 153                         c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
 154
 155 c['protocols'] = {'pb': {'port': pb_port}}
 156
 157 # coalesce builds
 158 c['collapseRequests'] = True
 159
 160 # Reduce amount of backlog data
 161 c['configurators'] = [util.JanitorConfigurator(
 162         logHorizon=timedelta(days=3),
 163         hour=6,
 164 )]
 165
 166 @defer.inlineCallbacks
 167 def getNewestCompleteTime(bldr):
 168         """Returns the complete_at of the latest completed and not SKIPPED
 169         build request for this builder, or None if there are no such build
 170         requests. We need to filter out SKIPPED requests because we're
 171         using collapseRequests=True which is unfortunately marking all
 172         previous requests as complete when new buildset is created.
 173
 174         @returns: datetime instance or None, via Deferred
 175         """
 176
 177         bldrid = yield bldr.getBuilderId()
 178         completed = yield bldr.master.data.get(
 179                         ('builders', bldrid, 'buildrequests'),
 180                         [
 181                                 resultspec.Filter('complete', 'eq', [True]),
 182                                 resultspec.Filter('results', 'ne', [results.SKIPPED]),
 183                         ],
 184                         order=['-complete_at'], limit=1)
 185         if not completed:
 186                 return
 187
 188         complete_at = completed[0]['complete_at']
 189
 190         last_build = yield bldr.master.data.get(
 191                         ('builds', ),
 192                         [
 193                                 resultspec.Filter('builderid', 'eq', [bldrid]),
 194                         ],
 195                         order=['-started_at'], limit=1)
 196
 197         if last_build and last_build[0]:
 198                 last_complete_at = last_build[0]['complete_at']
 199                 if last_complete_at and (last_complete_at > complete_at):
 200                         return last_complete_at
 201
 202         return complete_at
 203
 204 @defer.inlineCallbacks
 205 def prioritizeBuilders(master, builders):
 206         """Returns sorted list of builders by their last timestamp of completed and
 207         not skipped build.
 208
 209         @returns: list of sorted builders
 210         """
 211
 212         def is_building(bldr):
 213                 return bool(bldr.building) or bool(bldr.old_building)
 214
 215         def bldr_info(bldr):
 216                 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
 217                 d.addCallback(lambda complete_at: (complete_at, bldr))
 218                 return d
 219
 220         def bldr_sort(item):
 221                 (complete_at, bldr) = item
 222
 223                 if not complete_at:
 224                         date = datetime.min
 225                         complete_at = date.replace(tzinfo=tzutc())
 226
 227                 if is_building(bldr):
 228                         date = datetime.max
 229                         complete_at = date.replace(tzinfo=tzutc())
 230
 231                 return (complete_at, bldr.name)
 232
 233         results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 234         results.sort(key=bldr_sort)
 235
 236         for r in results:
 237                 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 238
 239         return [r[1] for r in results]
 240
 241 c['prioritizeBuilders'] = prioritizeBuilders
 242
 243 ####### CHANGESOURCES
 244
 245 branchNames = [branches[b]["name"] for b in branches]
 246
 247 # find targets
 248 targets = set()
 249
 250 def populateTargets():
 251         log.msg("Populating targets, this will take time")
 252         sourcegit = work_dir + '/source.git'
 253         for branch in branchNames:
 254                 if os.path.isdir(sourcegit):
 255                         subprocess.call(["rm", "-rf", sourcegit])
 256
 257                 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
 258
 259                 os.makedirs(sourcegit + '/tmp', exist_ok=True)
 260                 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
 261                         stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
 262
 263                 while True:
 264                         line = findtargets.stdout.readline()
 265                         if not line:
 266                                 break
 267                         ta = line.decode().strip().split(' ')
 268                         targets.add(ta[0])
 269
 270                 subprocess.call(["rm", "-rf", sourcegit])
 271
 272 populateTargets()
 273
 274 # the 'change_source' setting tells the buildmaster how it should find out
 275 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 276
 277 c['change_source'] = []
 278 c['change_source'].append(GitPoller(
 279         repo_url,
 280         workdir=work_dir+'/work.git', branches=branchNames,
 281         pollAtLaunch=True, pollinterval=300))
 282
 283 ####### SCHEDULERS
 284
 285 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 286 # case, just kick off a 'basebuild' build
 287
 288 class TagChoiceParameter(BaseParameter):
 289         spec_attributes = ["strict", "choices"]
 290         type = "list"
 291         strict = True
 292
 293         def __init__(self, name, label=None, **kw):
 294                 super().__init__(name, label, **kw)
 295                 self._choice_list = []
 296
 297         @property
 298         def choices(self):
 299                 taglist = []
 300                 branchvers = []
 301
 302                 for b in branchNames:
 303                         basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
 304                         if basever:
 305                                 branchvers.append(basever[1])
 306
 307                 alltags = subprocess.Popen(
 308                         ['git', 'ls-remote', '--tags', repo_url],
 309                         stdout = subprocess.PIPE)
 310
 311                 while True:
 312                         line = alltags.stdout.readline()
 313
 314                         if not line:
 315                                 break
 316
 317                         tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
 318
 319                         # only list tags matching configured branches
 320                         if tagver and any(tagver[1].startswith(b) for b in branchvers):
 321                                 taglist.append(tagver[1])
 322
 323                 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
 324                 taglist.insert(0, '')
 325
 326                 self._choice_list = taglist
 327
 328                 return self._choice_list
 329
 330         def parse_from_arg(self, s):
 331                 if self.strict and s not in self._choice_list:
 332                         raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
 333                 return s
 334
 335 c['schedulers'] = []
 336 c['schedulers'].append(AnyBranchScheduler(
 337         name            = "all",
 338         change_filter   = util.ChangeFilter(branch=branchNames),
 339         treeStableTimer = 15*60,
 340         builderNames    = list(targets)))
 341
 342 c['schedulers'].append(ForceScheduler(
 343         name         = "force",
 344         buttonName   = "Force builds",
 345         label        = "Force build details",
 346         builderNames = [ "00_force_build" ],
 347
 348         codebases = [
 349                 util.CodebaseParameter(
 350                         "",
 351                         label      = "Repository",
 352                         branch     = util.FixedParameter(name = "branch",     default = ""),
 353                         revision   = util.FixedParameter(name = "revision",   default = ""),
 354                         repository = util.FixedParameter(name = "repository", default = ""),
 355                         project    = util.FixedParameter(name = "project",    default = "")
 356                 )
 357         ],
 358
 359         reason = util.StringParameter(
 360                 name     = "reason",
 361                 label    = "Reason",
 362                 default  = "Trigger build",
 363                 required = True,
 364                 size     = 80
 365         ),
 366
 367         properties = [
 368                 util.NestedParameter(
 369                         name="options",
 370                         label="Build Options",
 371                         layout="vertical",
 372                         fields=[
 373                                 util.ChoiceStringParameter(
 374                                         name    = "target",
 375                                         label   = "Build target",
 376                                         default = "all",
 377                                         choices = [ "all" ] + list(targets)
 378                                 ),
 379                                 TagChoiceParameter(
 380                                         name    = "tag",
 381                                         label   = "Build tag",
 382                                         default = ""
 383                                 )
 384                         ]
 385                 )
 386         ]
 387 ))
 388
 389 ####### BUILDERS
 390
 391 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 392 # what steps, and which workers can execute them.  Note that any particular build will
 393 # only take place on one worker.
 394
 395 def IsTaggingRequested(step):
 396         tag = step.getProperty("tag")
 397         return tag and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", tag)
 398
 399 def IsNoMasterBuild(step):
 400         return step.getProperty("branch") != "master"
 401
 402 def IsUsignEnabled(step):
 403         branch = step.getProperty("branch")
 404         return branch and branches[branch].get("usign_key")
 405
 406 def IsSignEnabled(step):
 407         branch = step.getProperty("branch")
 408         return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 409
 410 def IsKmodArchiveEnabled(step):
 411         branch = step.getProperty("branch")
 412         return branch and branches[branch].get("kmod_archive")
 413
 414 def GetBaseVersion(branch):
 415         if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 416                 return branch.split('-')[1]
 417         else:
 418                 return "master"
 419
 420 @properties.renderer
 421 def GetVersionPrefix(props):
 422         branch = props.getProperty("branch")
 423         basever = GetBaseVersion(branch)
 424         if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 425                 return "%s/" % props["tag"]
 426         elif basever != "master":
 427                 return "%s-SNAPSHOT/" % basever
 428         else:
 429                 return ""
 430
 431 @util.renderer
 432 def GetConfigSeed(props):
 433         branch = props.getProperty("branch")
 434         return branch and branches[branch].get("config_seed") or ""
 435
 436 @util.renderer
 437 def GetRsyncParams(props, srcorbin, urlorkey):
 438         # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 439         branch = props.getProperty("branch")
 440         opt = srcorbin + "_" + urlorkey
 441         return branch and branches[branch].get(opt)
 442
 443 @util.renderer
 444 def GetUsignKey(props):
 445         branch = props.getProperty("branch")
 446         return branch and branches[branch].get("usign_key")
 447
 448 def GetNextBuild(builder, requests):
 449         for r in requests:
 450                 if r.properties:
 451                         # order tagged build first
 452                         if r.properties.hasProperty("tag"):
 453                                 return r
 454                         # then order by branch order
 455                         pbranch = r.properties.getProperty("branch")
 456                         for name in branchNames:
 457                                 if pbranch == name:
 458                                         return r
 459
 460         r = requests[0]
 461         log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 462         return r
 463
 464 def MakeEnv(overrides=None, tryccache=False):
 465         env = {
 466                 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
 467                 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
 468         }
 469         if tryccache:
 470                 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 471                 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 472                 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
 473         else:
 474                 env['CC'] = env['CCC']
 475                 env['CXX'] = env['CCXX']
 476                 env['CCACHE'] = ''
 477         if overrides is not None:
 478                 env.update(overrides)
 479         return env
 480
 481 @properties.renderer
 482 def NetLockDl(props, extralock=None):
 483         lock = None
 484         locks = []
 485         if props.hasProperty("dl_lock"):
 486                 lock = NetLocks[props["dl_lock"]]
 487         if lock is not None:
 488                 locks.append(lock.access('exclusive'))
 489         if extralock is not None:
 490                 locks.append(extralock)
 491         return locks
 492
 493 @properties.renderer
 494 def NetLockUl(props):
 495         lock = None
 496         if props.hasProperty("ul_lock"):
 497                 lock = NetLocks[props["ul_lock"]]
 498         if lock is not None:
 499                 return [lock.access('exclusive')]
 500         else:
 501                 return []
 502
 503 @util.renderer
 504 def TagPropertyValue(props):
 505         if props.hasProperty("options"):
 506                 options = props.getProperty("options")
 507                 if type(options) is dict:
 508                         return options.get("tag")
 509         return None
 510
 511 def IsTargetSelected(target):
 512         def CheckTargetProperty(step):
 513                 try:
 514                         options = step.getProperty("options")
 515                         if type(options) is dict:
 516                                 selected_target = options.get("target", "all")
 517                                 if selected_target != "all" and selected_target != target:
 518                                         return False
 519                 except KeyError:
 520                         pass
 521
 522                 return True
 523
 524         return CheckTargetProperty
 525
 526 @util.renderer
 527 def UsignSec2Pub(props):
 528         branch = props.getProperty("branch")
 529         try:
 530                 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
 531                 seckey = branches[branch].get("usign_key")
 532                 seckey = base64.b64decode(seckey)
 533         except:
 534                 return None
 535
 536         return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
 537                 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
 538
 539
 540 c['builders'] = []
 541
 542 dlLock = locks.WorkerLock("worker_dl")
 543
 544 workerNames = [ ]
 545
 546 for worker in c['workers']:
 547         workerNames.append(worker.workername)
 548
 549 force_factory = BuildFactory()
 550
 551 c['builders'].append(BuilderConfig(
 552         name        = "00_force_build",
 553         workernames = workerNames,
 554         factory     = force_factory))
 555
 556 for target in targets:
 557         ts = target.split('/')
 558
 559         factory = BuildFactory()
 560
 561         # setup shared work directory if required
 562         factory.addStep(ShellCommand(
 563                 name = "sharedwd",
 564                 descriptionDone = "Shared work directory set up",
 565                 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 566                 workdir = ".",
 567                 haltOnFailure = True,
 568         ))
 569
 570         # find number of cores
 571         factory.addStep(SetPropertyFromCommand(
 572                 name = "nproc",
 573                 property = "nproc",
 574                 description = "Finding number of CPUs",
 575                 command = ["nproc"],
 576         ))
 577
 578         # find gcc and g++ compilers
 579         factory.addStep(FileDownload(
 580                 name = "dlfindbinpl",
 581                 mastersrc = scripts_dir + '/findbin.pl',
 582                 workerdest = "../findbin.pl",
 583                 mode = 0o755,
 584         ))
 585
 586         factory.addStep(SetPropertyFromCommand(
 587                 name = "gcc",
 588                 property = "cc_command",
 589                 description = "Finding gcc command",
 590                 command = ["../findbin.pl", "gcc", "", ""],
 591                 haltOnFailure = True,
 592         ))
 593
 594         factory.addStep(SetPropertyFromCommand(
 595                 name = "g++",
 596                 property = "cxx_command",
 597                 description = "Finding g++ command",
 598                 command = ["../findbin.pl", "g++", "", ""],
 599                 haltOnFailure = True,
 600         ))
 601
 602         # see if ccache is available
 603         factory.addStep(SetPropertyFromCommand(
 604                 name = "ccache",
 605                 property = "ccache_command",
 606                 description = "Testing for ccache command",
 607                 command = ["which", "ccache"],
 608                 haltOnFailure = False,
 609                 flunkOnFailure = False,
 610                 warnOnFailure = False,
 611                 hideStepIf = lambda r, s: r==results.FAILURE,
 612         ))
 613
 614         # check out the source
 615         # Git() runs:
 616           # if repo doesn't exist: 'git clone repourl'
 617           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 618           # git cat-file -e <commit>
 619           # git checkout -f <commit>
 620           # git checkout -B <branch>
 621           # git rev-parse HEAD
 622         factory.addStep(Git(
 623                 name = "git",
 624                 repourl = repo_url,
 625                 mode = 'full',
 626                 method = 'fresh',
 627                 locks = NetLockDl,
 628                 haltOnFailure = True,
 629         ))
 630
 631         # update remote refs
 632         factory.addStep(ShellCommand(
 633                 name = "fetchrefs",
 634                 description = "Fetching Git remote refs",
 635                 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
 636                 haltOnFailure = True,
 637         ))
 638
 639         # switch to tag
 640         factory.addStep(ShellCommand(
 641                 name = "switchtag",
 642                 description = "Checking out Git tag",
 643                 command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
 644                 haltOnFailure = True,
 645                 doStepIf = IsTaggingRequested
 646         ))
 647
 648         # Verify that Git HEAD points to a tag or branch
 649         # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 650         factory.addStep(ShellCommand(
 651                 name = "gitverify",
 652                 description = "Ensure that Git HEAD is pointing to a branch or tag",
 653                 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
 654                 haltOnFailure = True,
 655         ))
 656
 657         factory.addStep(ShellCommand(
 658                 name = "rmtmp",
 659                 description = "Remove tmp folder",
 660                 command=["rm", "-rf", "tmp/"],
 661         ))
 662
 663         # feed
 664         factory.addStep(ShellCommand(
 665                 name = "rmfeedlinks",
 666                 description = "Remove feed symlinks",
 667                 command=["rm", "-rf", "package/feeds/"],
 668         ))
 669
 670         factory.addStep(StringDownload(
 671                 name = "ccachecc",
 672                 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 673                 workerdest = "../ccache_cc.sh",
 674                 mode = 0o755,
 675         ))
 676
 677         factory.addStep(StringDownload(
 678                 name = "ccachecxx",
 679                 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 680                 workerdest = "../ccache_cxx.sh",
 681                 mode = 0o755,
 682         ))
 683
 684         # feed
 685         factory.addStep(ShellCommand(
 686                 name = "updatefeeds",
 687                 description = "Updating feeds",
 688                 command=["./scripts/feeds", "update"],
 689                 env = MakeEnv(tryccache=True),
 690                 haltOnFailure = True,
 691                 locks = NetLockDl,
 692         ))
 693
 694         # feed
 695         factory.addStep(ShellCommand(
 696                 name = "installfeeds",
 697                 description = "Installing feeds",
 698                 command=["./scripts/feeds", "install", "-a"],
 699                 env = MakeEnv(tryccache=True),
 700                 haltOnFailure = True,
 701         ))
 702
 703         # seed config
 704         factory.addStep(StringDownload(
 705                 name = "dlconfigseed",
 706                 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 707                 workerdest = ".config",
 708                 mode = 0o644,
 709         ))
 710
 711         # configure
 712         factory.addStep(ShellCommand(
 713                 name = "newconfig",
 714                 descriptionDone = ".config seeded",
 715                 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
 716         ))
 717
 718         factory.addStep(ShellCommand(
 719                 name = "delbin",
 720                 description = "Removing output directory",
 721                 command = ["rm", "-rf", "bin/"],
 722         ))
 723
 724         factory.addStep(ShellCommand(
 725                 name = "defconfig",
 726                 description = "Populating .config",
 727                 command = ["make", "defconfig"],
 728                 env = MakeEnv(),
 729         ))
 730
 731         # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 732         factory.addStep(ShellCommand(
 733                 name = "checkarch",
 734                 description = "Checking architecture",
 735                 descriptionDone = "Architecture validated",
 736                 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
 737                 logEnviron = False,
 738                 want_stdout = False,
 739                 want_stderr = False,
 740                 haltOnFailure = True,
 741                 flunkOnFailure = False, # this is not a build FAILURE
 742         ))
 743
 744         # find libc suffix
 745         factory.addStep(SetPropertyFromCommand(
 746                 name = "libc",
 747                 property = "libc",
 748                 description = "Finding libc suffix",
 749                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
 750         ))
 751
 752         # install build key
 753         factory.addStep(StringDownload(
 754                 name = "dlkeybuildpub",
 755                 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
 756                 workerdest = "key-build.pub",
 757                 mode = 0o600,
 758                 doStepIf = IsUsignEnabled,
 759         ))
 760
 761         factory.addStep(StringDownload(
 762                 name = "dlkeybuild",
 763                 s = "# fake private key",
 764                 workerdest = "key-build",
 765                 mode = 0o600,
 766                 doStepIf = IsUsignEnabled,
 767         ))
 768
 769         factory.addStep(StringDownload(
 770                 name = "dlkeybuilducert",
 771                 s = "# fake certificate",
 772                 workerdest = "key-build.ucert",
 773                 mode = 0o600,
 774                 doStepIf = IsUsignEnabled,
 775         ))
 776
 777         # prepare dl
 778         factory.addStep(ShellCommand(
 779                 name = "dldir",
 780                 description = "Preparing dl/",
 781                 descriptionDone = "dl/ prepared",
 782                 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
 783                 workdir = Property("builddir"),
 784                 logEnviron = False,
 785                 want_stdout = False,
 786         ))
 787
 788         # prepare tar
 789         factory.addStep(ShellCommand(
 790                 name = "dltar",
 791                 description = "Building and installing GNU tar",
 792                 descriptionDone = "GNU tar built and installed",
 793                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
 794                 env = MakeEnv(tryccache=True),
 795                 haltOnFailure = True,
 796         ))
 797
 798         # populate dl
 799         factory.addStep(ShellCommand(
 800                 name = "dlrun",
 801                 description = "Populating dl/",
 802                 descriptionDone = "dl/ populated",
 803                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
 804                 env = MakeEnv(),
 805                 logEnviron = False,
 806                 locks = NetLockDl.withArgs(dlLock.access('exclusive')),
 807         ))
 808
 809         factory.addStep(ShellCommand(
 810                 name = "cleanbase",
 811                 description = "Cleaning base-files",
 812                 command=["make", "package/base-files/clean", "V=s"],
 813         ))
 814
 815         # build
 816         factory.addStep(ShellCommand(
 817                 name = "tools",
 818                 description = "Building and installing tools",
 819                 descriptionDone = "Tools built and installed",
 820                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
 821                 env = MakeEnv(tryccache=True),
 822                 haltOnFailure = True,
 823         ))
 824
 825         factory.addStep(ShellCommand(
 826                 name = "toolchain",
 827                 description = "Building and installing toolchain",
 828                 descriptionDone = "Toolchain built and installed",
 829                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
 830                 env = MakeEnv(),
 831                 haltOnFailure = True,
 832         ))
 833
 834         factory.addStep(ShellCommand(
 835                 name = "kmods",
 836                 description = "Building kmods",
 837                 descriptionDone = "Kmods built",
 838                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 839                 env = MakeEnv(),
 840                 haltOnFailure = True,
 841         ))
 842
 843         # find kernel version
 844         factory.addStep(SetPropertyFromCommand(
 845                 name = "kernelversion",
 846                 property = "kernelversion",
 847                 description = "Finding the effective Kernel version",
 848                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 849                 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
 850         ))
 851
 852         factory.addStep(ShellCommand(
 853                 name = "pkgclean",
 854                 description = "Cleaning up package build",
 855                 descriptionDone = "Package build cleaned up",
 856                 command=["make", "package/cleanup", "V=s"],
 857         ))
 858
 859         factory.addStep(ShellCommand(
 860                 name = "pkgbuild",
 861                 description = "Building packages",
 862                 descriptionDone = "Packages built",
 863                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 864                 env = MakeEnv(),
 865                 haltOnFailure = True,
 866         ))
 867
 868         factory.addStep(ShellCommand(
 869                 name = "pkginstall",
 870                 description = "Installing packages",
 871                 descriptionDone = "Packages installed",
 872                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
 873                 env = MakeEnv(),
 874                 haltOnFailure = True,
 875         ))
 876
 877         factory.addStep(ShellCommand(
 878                 name = "pkgindex",
 879                 description = "Indexing packages",
 880                 descriptionDone = "Packages indexed",
 881                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
 882                 env = MakeEnv(),
 883                 haltOnFailure = True,
 884         ))
 885
 886         factory.addStep(ShellCommand(
 887                 name = "images",
 888                 description = "Building and installing images",
 889                 descriptionDone = "Images built and installed",
 890                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
 891                 env = MakeEnv(),
 892                 haltOnFailure = True,
 893         ))
 894
 895         factory.addStep(ShellCommand(
 896                 name = "buildinfo",
 897                 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
 898                 command = "make -j1 buildinfo V=s || true",
 899                 env = MakeEnv(),
 900                 haltOnFailure = True,
 901         ))
 902
 903         factory.addStep(ShellCommand(
 904                 name = "json_overview_image_info",
 905                 description = "Generating profiles.json in target folder",
 906                 command = "make -j1 json_overview_image_info V=s || true",
 907                 env = MakeEnv(),
 908                 haltOnFailure = True,
 909         ))
 910
 911         factory.addStep(ShellCommand(
 912                 name = "checksums",
 913                 description = "Calculating checksums",
 914                 descriptionDone = "Checksums calculated",
 915                 command=["make", "-j1", "checksum", "V=s"],
 916                 env = MakeEnv(),
 917                 haltOnFailure = True,
 918         ))
 919
 920         factory.addStep(ShellCommand(
 921                 name = "kmoddir",
 922                 descriptionDone = "Kmod directory created",
 923                 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
 924                 haltOnFailure = True,
 925                 doStepIf = IsKmodArchiveEnabled,
 926         ))
 927
 928         factory.addStep(ShellCommand(
 929                 name = "kmodprepare",
 930                 description = "Preparing kmod archive",
 931                 descriptionDone = "Kmod archive prepared",
 932                 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 933                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
 934                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 935                 haltOnFailure = True,
 936                 doStepIf = IsKmodArchiveEnabled,
 937         ))
 938
 939         factory.addStep(ShellCommand(
 940                 name = "kmodindex",
 941                 description = "Indexing kmod archive",
 942                 descriptionDone = "Kmod archive indexed",
 943                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
 944                         Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 945                 env = MakeEnv(),
 946                 haltOnFailure = True,
 947                 doStepIf = IsKmodArchiveEnabled,
 948         ))
 949
 950         # sign
 951         factory.addStep(MasterShellCommand(
 952                 name = "signprepare",
 953                 descriptionDone = "Temporary signing directory prepared",
 954                 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
 955                 haltOnFailure = True,
 956                 doStepIf = IsSignEnabled,
 957
 958         ))
 959
 960         factory.addStep(ShellCommand(
 961                 name = "signpack",
 962                 description = "Packing files to sign",
 963                 descriptionDone = "Files to sign packed",
 964                 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
 965                 haltOnFailure = True,
 966                 doStepIf = IsSignEnabled,
 967         ))
 968
 969         factory.addStep(FileUpload(
 970                 workersrc = "sign.tar.gz",
 971                 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
 972                 haltOnFailure = True,
 973                 doStepIf = IsSignEnabled,
 974         ))
 975
 976         factory.addStep(MasterShellCommand(
 977                 name = "signfiles",
 978                 description = "Signing files",
 979                 descriptionDone = "Files signed",
 980                 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
 981                 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
 982                 haltOnFailure = True,
 983                 doStepIf = IsSignEnabled,
 984         ))
 985
 986         factory.addStep(FileDownload(
 987                 name = "dlsigntargz",
 988                 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
 989                 workerdest = "sign.tar.gz",
 990                 haltOnFailure = True,
 991                 doStepIf = IsSignEnabled,
 992         ))
 993
 994         factory.addStep(ShellCommand(
 995                 name = "signunpack",
 996                 description = "Unpacking signed files",
 997                 descriptionDone = "Signed files unpacked",
 998                 command = ["tar", "-xzf", "sign.tar.gz"],
 999                 haltOnFailure = True,
1000                 doStepIf = IsSignEnabled,
1001         ))
1002
1003         # upload
1004         factory.addStep(ShellCommand(
1005                 name = "dirprepare",
1006                 descriptionDone = "Upload directory structure prepared",
1007                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1008                 haltOnFailure = True,
1009         ))
1010
1011         factory.addStep(ShellCommand(
1012                 name = "linkprepare",
1013                 descriptionDone = "Repository symlink prepared",
1014                 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1015                 doStepIf = IsNoMasterBuild,
1016                 haltOnFailure = True,
1017         ))
1018
1019         factory.addStep(ShellCommand(
1020                 name = "kmoddirprepare",
1021                 descriptionDone = "Kmod archive upload directory prepared",
1022                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1023                 haltOnFailure = True,
1024                 doStepIf = IsKmodArchiveEnabled,
1025         ))
1026
1027         factory.addStep(ShellCommand(
1028                 name = "dirupload",
1029                 description = "Uploading directory structure",
1030                 descriptionDone = "Directory structure uploaded",
1031                 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1032                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1033                 haltOnFailure = True,
1034                 logEnviron = False,
1035                 locks = NetLockUl,
1036                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1037         ))
1038
1039         # download remote sha256sums to 'target-sha256sums'
1040         factory.addStep(ShellCommand(
1041                 name = "target-sha256sums",
1042                 description = "Fetching remote sha256sums for target",
1043                 descriptionDone = "Remote sha256sums for target fetched",
1044                 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1045                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1046                 logEnviron = False,
1047                 haltOnFailure = False,
1048                 flunkOnFailure = False,
1049                 warnOnFailure = False,
1050                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1051         ))
1052
1053         # build list of files to upload
1054         factory.addStep(FileDownload(
1055                 name = "dlsha2rsyncpl",
1056                 mastersrc = scripts_dir + '/sha2rsync.pl',
1057                 workerdest = "../sha2rsync.pl",
1058                 mode = 0o755,
1059         ))
1060
1061         factory.addStep(ShellCommand(
1062                 name = "buildlist",
1063                 description = "Building list of files to upload",
1064                 descriptionDone = "List of files to upload built",
1065                 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1066                 haltOnFailure = True,
1067         ))
1068
1069         factory.addStep(FileDownload(
1070                 name = "dlrsync.sh",
1071                 mastersrc = scripts_dir + '/rsync.sh',
1072                 workerdest = "../rsync.sh",
1073                 mode = 0o755,
1074         ))
1075
1076         # upload new files and update existing ones
1077         factory.addStep(ShellCommand(
1078                 name = "targetupload",
1079                 description = "Uploading target files",
1080                 descriptionDone = "Target files uploaded",
1081                 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1082                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1083                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1084                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1085                 haltOnFailure = True,
1086                 logEnviron = False,
1087                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1088         ))
1089
1090         # delete files which don't exist locally
1091         factory.addStep(ShellCommand(
1092                 name = "targetprune",
1093                 description = "Pruning target files",
1094                 descriptionDone = "Target files pruned",
1095                 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1096                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1097                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1098                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1099                 haltOnFailure = True,
1100                 logEnviron = False,
1101                 locks = NetLockUl,
1102                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1103         ))
1104
1105         factory.addStep(ShellCommand(
1106                 name = "kmodupload",
1107                 description = "Uploading kmod archive",
1108                 descriptionDone = "Kmod archive uploaded",
1109                 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1110                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1111                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1112                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1113                 haltOnFailure = True,
1114                 logEnviron = False,
1115                 locks = NetLockUl,
1116                 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1117         ))
1118
1119         factory.addStep(ShellCommand(
1120                 name = "sourcelist",
1121                 description = "Finding source archives to upload",
1122                 descriptionDone = "Source archives to upload found",
1123                 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1124                 haltOnFailure = True,
1125         ))
1126
1127         factory.addStep(ShellCommand(
1128                 name = "sourceupload",
1129                 description = "Uploading source archives",
1130                 descriptionDone = "Source archives uploaded",
1131                 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1132                         [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1133                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1134                 haltOnFailure = True,
1135                 logEnviron = False,
1136                 locks = NetLockUl,
1137                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1138         ))
1139
1140         factory.addStep(ShellCommand(
1141                 name = "df",
1142                 description = "Reporting disk usage",
1143                 command=["df", "-h", "."],
1144                 env={'LC_ALL': 'C'},
1145                 logEnviron = False,
1146                 haltOnFailure = False,
1147                 flunkOnFailure = False,
1148                 warnOnFailure = False,
1149                 alwaysRun = True,
1150         ))
1151
1152         factory.addStep(ShellCommand(
1153                 name = "du",
1154                 description = "Reporting estimated file space usage",
1155                 command=["du", "-sh", "."],
1156                 env={'LC_ALL': 'C'},
1157                 logEnviron = False,
1158                 haltOnFailure = False,
1159                 flunkOnFailure = False,
1160                 warnOnFailure = False,
1161                 alwaysRun = True,
1162         ))
1163
1164         factory.addStep(ShellCommand(
1165                 name = "ccachestat",
1166                 description = "Reporting ccache stats",
1167                 command=["ccache", "-s"],
1168                 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1169                 logEnviron = False,
1170                 want_stderr = False,
1171                 haltOnFailure = False,
1172                 flunkOnFailure = False,
1173                 warnOnFailure = False,
1174                 hideStepIf = lambda r, s: r==results.FAILURE,
1175         ))
1176
1177         c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1178
1179         c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1180         force_factory.addStep(steps.Trigger(
1181                 name = "trigger_%s" % target,
1182                 description = "Triggering %s build" % target,
1183                 schedulerNames = [ "trigger_%s" % target ],
1184                 set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
1185                 doStepIf = IsTargetSelected(target),
1186         ))
1187
1188
1189 ####### STATUS TARGETS
1190
1191 # 'status' is a list of Status Targets. The results of each build will be
1192 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1193 # including web pages, email senders, and IRC bots.
1194
1195 if "status_bind" in inip1:
1196         c['www'] = {
1197                 'port': inip1.get("status_bind"),
1198                 'plugins': {
1199                         'waterfall_view': True,
1200                         'console_view': True,
1201                         'grid_view': True
1202                 }
1203         }
1204
1205         if "status_user" in inip1 and "status_password" in inip1:
1206                 c['www']['auth'] = util.UserPasswordAuth([
1207                         (inip1.get("status_user"), inip1.get("status_password"))
1208                 ])
1209                 c['www']['authz'] = util.Authz(
1210                         allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1211                         roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1212                 )
1213
1214 c['services'] = []
1215 if ini.has_section("irc"):
1216         iniirc = ini['irc']
1217         irc_host = iniirc.get("host", None)
1218         irc_port = iniirc.getint("port", 6667)
1219         irc_chan = iniirc.get("channel", None)
1220         irc_nick = iniirc.get("nickname", None)
1221         irc_pass = iniirc.get("password", None)
1222
1223         if irc_host and irc_nick and irc_chan:
1224                 irc = reporters.IRC(irc_host, irc_nick,
1225                         port = irc_port,
1226                         password = irc_pass,
1227                         channels = [ irc_chan ],
1228                         notify_events = [ 'exception', 'problem', 'recovery' ]
1229                 )
1230
1231                 c['services'].append(irc)
1232
1233 c['revlink'] = util.RevlinkMatch([
1234         r'https://git.openwrt.org/openwrt/(.*).git'
1235         ],
1236         r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1237
1238 ####### DB URL
1239
1240 c['db'] = {
1241         # This specifies what database buildbot uses to store its state.  You can leave
1242         # this at its default for all but the largest installations.
1243         'db_url' : "sqlite:///state.sqlite",
1244 }
1245
1246 c['buildbotNetUsageData'] = None