phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41
  42
  43 if not os.path.exists("twistd.pid"):
  44         with open("twistd.pid", "w") as pidfile:
  45                 pidfile.write("{}".format(os.getpid()))
  46
  47 # This is a sample buildmaster config file. It must be installed as
  48 # 'master.cfg' in your buildmaster's base directory.
  49
  50 ini = configparser.ConfigParser()
  51 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  52
  53 if "general" not in ini or "phase1" not in ini:
  54         raise ValueError("Fix your configuration")
  55
  56 inip1 = ini['phase1']
  57
  58 # Globals
  59 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
  60 scripts_dir = os.path.abspath("../scripts")
  61
  62 repo_url = ini['repo'].get("url")
  63
  64 rsync_defopts = ["-v", "-4", "--timeout=120"]
  65
  66 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  67 #       rsync_bin_defopts += ["--contimeout=20"]
  68
  69 branches = {}
  70
  71 def ini_parse_branch(section):
  72         b = {}
  73         name = section.get("name")
  74
  75         if not name:
  76                 raise ValueError("missing 'name' in " + repr(section))
  77         if name in branches:
  78                 raise ValueError("duplicate branch name in " + repr(section))
  79
  80         b["name"] = name
  81         b["bin_url"] = section.get("binary_url")
  82         b["bin_key"] = section.get("binary_password")
  83
  84         b["src_url"] = section.get("source_url")
  85         b["src_key"] = section.get("source_password")
  86
  87         b["gpg_key"] = section.get("gpg_key")
  88
  89         b["usign_key"] = section.get("usign_key")
  90         usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  91         b["usign_comment"] = section.get("usign_comment", usign_comment)
  92
  93         b["config_seed"] = section.get("config_seed")
  94
  95         b["kmod_archive"] = section.getboolean("kmod_archive", False)
  96
  97         branches[name] = b
  98         log.msg("Configured branch: {}".format(name))
  99
 100 # PB port can be either a numeric port or a connection string
 101 pb_port = inip1.get("port") or 9989
 102
 103 # This is the dictionary that the buildmaster pays attention to. We also use
 104 # a shorter alias to save typing.
 105 c = BuildmasterConfig = {}
 106
 107 ####### PROJECT IDENTITY
 108
 109 # the 'title' string will appear at the top of this buildbot
 110 # installation's html.WebStatus home page (linked to the
 111 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 112
 113 c['title'] = ini['general'].get("title")
 114 c['titleURL'] = ini['general'].get("title_url")
 115
 116 # the 'buildbotURL' string should point to the location where the buildbot's
 117 # internal web server (usually the html.WebStatus page) is visible. This
 118 # typically uses the port number set in the Waterfall 'status' entry, but
 119 # with an externally-visible host name which the buildbot cannot figure out
 120 # without some help.
 121
 122 c['buildbotURL'] = inip1.get("buildbot_url")
 123
 124 ####### BUILDWORKERS
 125
 126 # The 'workers' list defines the set of recognized buildworkers. Each element is
 127 # a Worker object, specifying a unique worker name and password.  The same
 128 # worker name and password must be configured on the worker.
 129
 130 c['workers'] = []
 131 NetLocks = dict()
 132
 133 for section in ini.sections():
 134         if section.startswith("branch "):
 135                 ini_parse_branch(ini[section])
 136
 137         if section.startswith("worker "):
 138                 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
 139                    (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
 140                         sl_props = { 'dl_lock':None, 'ul_lock':None }
 141                         name = ini.get(section, "name")
 142                         password = ini.get(section, "password")
 143                         if ini.has_option(section, "dl_lock"):
 144                                 lockname = ini.get(section, "dl_lock")
 145                                 sl_props['dl_lock'] = lockname
 146                                 if lockname not in NetLocks:
 147                                         NetLocks[lockname] = locks.MasterLock(lockname)
 148                         if ini.has_option(section, "ul_lock"):
 149                                 lockname = ini.get(section, "ul_lock")
 150                                 sl_props['ul_lock'] = lockname
 151                                 if lockname not in NetLocks:
 152                                         NetLocks[lockname] = locks.MasterLock(lockname)
 153                         c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
 154
 155 c['protocols'] = {'pb': {'port': pb_port}}
 156
 157 # coalesce builds
 158 c['collapseRequests'] = True
 159
 160 # Reduce amount of backlog data
 161 c['configurators'] = [util.JanitorConfigurator(
 162         logHorizon=timedelta(days=3),
 163         hour=6,
 164 )]
 165
 166 @defer.inlineCallbacks
 167 def getNewestCompleteTime(bldr):
 168         """Returns the complete_at of the latest completed and not SKIPPED
 169         build request for this builder, or None if there are no such build
 170         requests. We need to filter out SKIPPED requests because we're
 171         using collapseRequests=True which is unfortunately marking all
 172         previous requests as complete when new buildset is created.
 173
 174         @returns: datetime instance or None, via Deferred
 175         """
 176
 177         bldrid = yield bldr.getBuilderId()
 178         completed = yield bldr.master.data.get(
 179                         ('builders', bldrid, 'buildrequests'),
 180                         [
 181                                 resultspec.Filter('complete', 'eq', [True]),
 182                                 resultspec.Filter('results', 'ne', [results.SKIPPED]),
 183                         ],
 184                         order=['-complete_at'], limit=1)
 185         if not completed:
 186                 return
 187
 188         complete_at = completed[0]['complete_at']
 189
 190         last_build = yield bldr.master.data.get(
 191                         ('builds', ),
 192                         [
 193                                 resultspec.Filter('builderid', 'eq', [bldrid]),
 194                         ],
 195                         order=['-started_at'], limit=1)
 196
 197         if last_build and last_build[0]:
 198                 last_complete_at = last_build[0]['complete_at']
 199                 if last_complete_at and (last_complete_at > complete_at):
 200                         return last_complete_at
 201
 202         return complete_at
 203
 204 @defer.inlineCallbacks
 205 def prioritizeBuilders(master, builders):
 206         """Returns sorted list of builders by their last timestamp of completed and
 207         not skipped build.
 208
 209         @returns: list of sorted builders
 210         """
 211
 212         def is_building(bldr):
 213                 return bool(bldr.building) or bool(bldr.old_building)
 214
 215         def bldr_info(bldr):
 216                 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
 217                 d.addCallback(lambda complete_at: (complete_at, bldr))
 218                 return d
 219
 220         def bldr_sort(item):
 221                 (complete_at, bldr) = item
 222
 223                 if not complete_at:
 224                         date = datetime.min
 225                         complete_at = date.replace(tzinfo=tzutc())
 226
 227                 if is_building(bldr):
 228                         date = datetime.max
 229                         complete_at = date.replace(tzinfo=tzutc())
 230
 231                 return (complete_at, bldr.name)
 232
 233         results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 234         results.sort(key=bldr_sort)
 235
 236         for r in results:
 237                 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 238
 239         return [r[1] for r in results]
 240
 241 c['prioritizeBuilders'] = prioritizeBuilders
 242
 243 ####### CHANGESOURCES
 244
 245 branchNames = [branches[b]["name"] for b in branches]
 246
 247 # find targets
 248 targets = set()
 249
 250 def populateTargets():
 251         log.msg("Populating targets, this will take time")
 252         sourcegit = work_dir + '/source.git'
 253         for branch in branchNames:
 254                 if os.path.isdir(sourcegit):
 255                         subprocess.call(["rm", "-rf", sourcegit])
 256
 257                 subprocess.call(["git", "clone", "--depth=1", "--branch="+branch, repo_url, sourcegit])
 258
 259                 os.makedirs(sourcegit + '/tmp', exist_ok=True)
 260                 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
 261                         stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
 262
 263                 while True:
 264                         line = findtargets.stdout.readline()
 265                         if not line:
 266                                 break
 267                         ta = line.decode().strip().split(' ')
 268                         targets.add(ta[0])
 269
 270                 subprocess.call(["rm", "-rf", sourcegit])
 271
 272 populateTargets()
 273
 274 # the 'change_source' setting tells the buildmaster how it should find out
 275 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 276
 277 c['change_source'] = []
 278 c['change_source'].append(GitPoller(
 279         repo_url,
 280         workdir=work_dir+'/work.git', branches=branchNames,
 281         pollAtLaunch=True, pollinterval=300))
 282
 283 ####### SCHEDULERS
 284
 285 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 286 # case, just kick off a 'basebuild' build
 287
 288 class TagChoiceParameter(BaseParameter):
 289         spec_attributes = ["strict", "choices"]
 290         type = "list"
 291         strict = True
 292
 293         def __init__(self, name, label=None, **kw):
 294                 super().__init__(name, label, **kw)
 295                 self._choice_list = []
 296
 297         @property
 298         def choices(self):
 299                 taglist = []
 300                 basever = re.search(r'-([0-9]+\.[0-9]+)$', "master")    # XXX FIXME
 301
 302                 if basever:
 303                         findtags = subprocess.Popen(
 304                                 ['git', 'ls-remote', '--tags', repo_url],
 305                                 stdout = subprocess.PIPE)
 306
 307                         while True:
 308                                 line = findtags.stdout.readline()
 309
 310                                 if not line:
 311                                         break
 312
 313                                 tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
 314
 315                                 if tagver and tagver[1].find(basever[1]) == 0:
 316                                         taglist.append(tagver[1])
 317
 318                 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
 319                 taglist.insert(0, '')
 320
 321                 self._choice_list = taglist
 322
 323                 return self._choice_list
 324
 325         def parse_from_arg(self, s):
 326                 if self.strict and s not in self._choice_list:
 327                         raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
 328                 return s
 329
 330 c['schedulers'] = []
 331 c['schedulers'].append(AnyBranchScheduler(
 332         name            = "all",
 333         change_filter   = util.ChangeFilter(branch=branchNames),
 334         treeStableTimer = 15*60,
 335         builderNames    = list(targets)))
 336
 337 c['schedulers'].append(ForceScheduler(
 338         name         = "force",
 339         buttonName   = "Force builds",
 340         label        = "Force build details",
 341         builderNames = [ "00_force_build" ],
 342
 343         codebases = [
 344                 util.CodebaseParameter(
 345                         "",
 346                         label      = "Repository",
 347                         branch     = util.FixedParameter(name = "branch",     default = ""),
 348                         revision   = util.FixedParameter(name = "revision",   default = ""),
 349                         repository = util.FixedParameter(name = "repository", default = ""),
 350                         project    = util.FixedParameter(name = "project",    default = "")
 351                 )
 352         ],
 353
 354         reason = util.StringParameter(
 355                 name     = "reason",
 356                 label    = "Reason",
 357                 default  = "Trigger build",
 358                 required = True,
 359                 size     = 80
 360         ),
 361
 362         properties = [
 363                 util.NestedParameter(
 364                         name="options",
 365                         label="Build Options",
 366                         layout="vertical",
 367                         fields=[
 368                                 util.ChoiceStringParameter(
 369                                         name    = "target",
 370                                         label   = "Build target",
 371                                         default = "all",
 372                                         choices = set( "all" ) | targets
 373                                 ),
 374                                 TagChoiceParameter(
 375                                         name    = "tag",
 376                                         label   = "Build tag",
 377                                         default = ""
 378                                 )
 379                         ]
 380                 )
 381         ]
 382 ))
 383
 384 ####### BUILDERS
 385
 386 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 387 # what steps, and which workers can execute them.  Note that any particular build will
 388 # only take place on one worker.
 389
 390 def IsTaggingRequested(step):
 391         tag = step.getProperty("tag")
 392         return tag and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", tag)
 393
 394 def IsNoMasterBuild(step):
 395         return step.getProperty("branch") != "master"
 396
 397 def IsUsignEnabled(step):
 398         branch = step.getProperty("branch")
 399         return branch and branches[branch].get("usign_key")
 400
 401 def IsSignEnabled(step):
 402         branch = step.getProperty("branch")
 403         return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 404
 405 def IsKmodArchiveEnabled(step):
 406         branch = step.getProperty("branch")
 407         return branch and branches[branch].get("kmod_archive")
 408
 409 def GetBaseVersion(branch):
 410         if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 411                 return branch.split('-')[1]
 412         else:
 413                 return "master"
 414
 415 @properties.renderer
 416 def GetVersionPrefix(props):
 417         branch = props.getProperty("branch")
 418         basever = GetBaseVersion(branch)
 419         if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 420                 return "%s/" % props["tag"]
 421         elif basever != "master":
 422                 return "%s-SNAPSHOT/" % basever
 423         else:
 424                 return ""
 425
 426 @util.renderer
 427 def GetConfigSeed(props):
 428         branch = props.getProperty("branch")
 429         return branch and branches[branch].get("config_seed") or ""
 430
 431 @util.renderer
 432 def GetRsyncParams(props, srcorbin, urlorkey):
 433         # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 434         branch = props.getProperty("branch")
 435         opt = srcorbin + "_" + urlorkey
 436         return branch and branches[branch].get(opt)
 437
 438 @util.renderer
 439 def GetUsignKey(props):
 440         branch = props.getProperty("branch")
 441         return branch and branches[branch].get("usign_key")
 442
 443 def GetNextBuild(builder, requests):
 444         for r in requests:
 445                 if r.properties:
 446                         # order tagged build first
 447                         if r.properties.hasProperty("tag"):
 448                                 return r
 449                         # then order by branch order
 450                         pbranch = r.properties.getProperty("branch")
 451                         for name in branchNames:
 452                                 if pbranch == name:
 453                                         return r
 454
 455         r = requests[0]
 456         log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 457         return r
 458
 459 def MakeEnv(overrides=None, tryccache=False):
 460         env = {
 461                 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
 462                 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
 463         }
 464         if tryccache:
 465                 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 466                 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 467                 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
 468         else:
 469                 env['CC'] = env['CCC']
 470                 env['CXX'] = env['CCXX']
 471                 env['CCACHE'] = ''
 472         if overrides is not None:
 473                 env.update(overrides)
 474         return env
 475
 476 @properties.renderer
 477 def NetLockDl(props):
 478         lock = None
 479         if props.hasProperty("dl_lock"):
 480                 lock = NetLocks[props["dl_lock"]]
 481         if lock is not None:
 482                 return [lock.access('exclusive')]
 483         else:
 484                 return []
 485
 486 @properties.renderer
 487 def NetLockUl(props):
 488         lock = None
 489         if props.hasProperty("ul_lock"):
 490                 lock = NetLocks[props["ul_lock"]]
 491         if lock is not None:
 492                 return [lock.access('exclusive')]
 493         else:
 494                 return []
 495
 496 @util.renderer
 497 def TagPropertyValue(props):
 498         if props.hasProperty("options"):
 499                 options = props.getProperty("options")
 500                 if type(options) is dict:
 501                         return options.get("tag")
 502         return None
 503
 504 def IsTargetSelected(target):
 505         def CheckTargetProperty(step):
 506                 try:
 507                         options = step.getProperty("options")
 508                         if type(options) is dict:
 509                                 selected_target = options.get("target", "all")
 510                                 if selected_target != "all" and selected_target != target:
 511                                         return False
 512                 except KeyError:
 513                         pass
 514
 515                 return True
 516
 517         return CheckTargetProperty
 518
 519 @util.renderer
 520 def UsignSec2Pub(props):
 521         branch = props.getProperty("branch")
 522         try:
 523                 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
 524                 seckey = branches[branch].get("usign_key")
 525                 seckey = base64.b64decode(seckey)
 526         except:
 527                 return None
 528
 529         return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
 530                 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
 531
 532
 533 c['builders'] = []
 534
 535 dlLock = locks.WorkerLock("worker_dl")
 536
 537 workerNames = [ ]
 538
 539 for worker in c['workers']:
 540         workerNames.append(worker.workername)
 541
 542 force_factory = BuildFactory()
 543
 544 c['builders'].append(BuilderConfig(
 545         name        = "00_force_build",
 546         workernames = workerNames,
 547         factory     = force_factory))
 548
 549 for target in targets:
 550         ts = target.split('/')
 551
 552         factory = BuildFactory()
 553
 554         # setup shared work directory if required
 555         factory.addStep(ShellCommand(
 556                 name = "sharedwd",
 557                 description = "Setting up shared work directory",
 558                 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 559                 workdir = ".",
 560                 haltOnFailure = True))
 561
 562         # find number of cores
 563         factory.addStep(SetPropertyFromCommand(
 564                 name = "nproc",
 565                 property = "nproc",
 566                 description = "Finding number of CPUs",
 567                 command = ["nproc"]))
 568
 569         # find gcc and g++ compilers
 570         factory.addStep(FileDownload(
 571                 name = "dlfindbinpl",
 572                 mastersrc = scripts_dir + '/findbin.pl',
 573                 workerdest = "../findbin.pl",
 574                 mode = 0o755))
 575
 576         factory.addStep(SetPropertyFromCommand(
 577                 name = "gcc",
 578                 property = "cc_command",
 579                 description = "Finding gcc command",
 580                 command = [
 581                         "../findbin.pl", "gcc", "", "",
 582                 ],
 583                 haltOnFailure = True))
 584
 585         factory.addStep(SetPropertyFromCommand(
 586                 name = "g++",
 587                 property = "cxx_command",
 588                 description = "Finding g++ command",
 589                 command = [
 590                         "../findbin.pl", "g++", "", "",
 591                 ],
 592                 haltOnFailure = True))
 593
 594         # see if ccache is available
 595         factory.addStep(SetPropertyFromCommand(
 596                 property = "ccache_command",
 597                 command = ["which", "ccache"],
 598                 description = "Testing for ccache command",
 599                 haltOnFailure = False,
 600                 flunkOnFailure = False,
 601                 warnOnFailure = False,
 602         ))
 603
 604         # Workaround bug when switching from a checked out tag back to a branch
 605         # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 606         factory.addStep(ShellCommand(
 607                 name = "gitcheckout",
 608                 description = "Ensure that Git HEAD is sane",
 609                 command = Interpolate("if [ -d .git ]; then git checkout -f %(prop:branch)s && git branch --set-upstream-to origin/%(prop:branch)s || rm -fr .git; else exit 0; fi"),
 610                 haltOnFailure = True))
 611
 612         # check out the source
 613         # Git() runs:
 614           # if repo doesn't exist: 'git clone repourl'
 615           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
 616           # 'git fetch -t repourl branch; git reset --hard revision'
 617         factory.addStep(Git(
 618                 name = "git",
 619                 repourl = repo_url,
 620                 mode = 'full',
 621                 method = 'fresh',
 622                 locks = NetLockDl,
 623                 haltOnFailure = True,
 624         ))
 625
 626         # update remote refs
 627         factory.addStep(ShellCommand(
 628                 name = "fetchrefs",
 629                 description = "Fetching Git remote refs",
 630                 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
 631                 haltOnFailure = True
 632         ))
 633
 634         # switch to tag
 635         factory.addStep(ShellCommand(
 636                 name = "switchtag",
 637                 description = "Checking out Git tag",
 638                 command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
 639                 haltOnFailure = True,
 640                 doStepIf = IsTaggingRequested
 641         ))
 642
 643         # Verify that Git HEAD points to a tag or branch
 644         # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 645         factory.addStep(ShellCommand(
 646                 name = "gitverify",
 647                 description = "Ensure that Git HEAD is pointing to a branch or tag",
 648                 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
 649                 haltOnFailure = True))
 650
 651         factory.addStep(ShellCommand(
 652                 name = "rmtmp",
 653                 description = "Remove tmp folder",
 654                 command=["rm", "-rf", "tmp/"]))
 655
 656         # feed
 657         factory.addStep(ShellCommand(
 658                 name = "rmfeedlinks",
 659                 description = "Remove feed symlinks",
 660                 command=["rm", "-rf", "package/feeds/"]))
 661
 662         factory.addStep(StringDownload(
 663                 name = "ccachecc",
 664                 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 665                 workerdest = "../ccache_cc.sh",
 666                 mode = 0o755,
 667         ))
 668
 669         factory.addStep(StringDownload(
 670                 name = "ccachecxx",
 671                 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 672                 workerdest = "../ccache_cxx.sh",
 673                 mode = 0o755,
 674         ))
 675
 676         # feed
 677         factory.addStep(ShellCommand(
 678                 name = "updatefeeds",
 679                 description = "Updating feeds",
 680                 command=["./scripts/feeds", "update"],
 681                 env = MakeEnv(tryccache=True),
 682                 haltOnFailure = True,
 683                 locks = NetLockDl,
 684         ))
 685
 686         # feed
 687         factory.addStep(ShellCommand(
 688                 name = "installfeeds",
 689                 description = "Installing feeds",
 690                 command=["./scripts/feeds", "install", "-a"],
 691                 env = MakeEnv(tryccache=True),
 692                 haltOnFailure = True
 693         ))
 694
 695         # seed config
 696         factory.addStep(StringDownload(
 697                 name = "dlconfigseed",
 698                 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 699                 workerdest = ".config",
 700                 mode = 0o644
 701         ))
 702
 703         # configure
 704         factory.addStep(ShellCommand(
 705                 name = "newconfig",
 706                 description = "Seeding .config",
 707                 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey)
 708         ))
 709
 710         factory.addStep(ShellCommand(
 711                 name = "delbin",
 712                 description = "Removing output directory",
 713                 command = ["rm", "-rf", "bin/"]
 714         ))
 715
 716         factory.addStep(ShellCommand(
 717                 name = "defconfig",
 718                 description = "Populating .config",
 719                 command = ["make", "defconfig"],
 720                 env = MakeEnv()
 721         ))
 722
 723         # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 724         factory.addStep(ShellCommand(
 725                 name = "checkarch",
 726                 description = "Checking architecture",
 727                 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
 728                 logEnviron = False,
 729                 want_stdout = False,
 730                 want_stderr = False,
 731                 haltOnFailure = True,
 732                 flunkOnFailure = False, # this is not a build FAILURE
 733         ))
 734
 735         # find libc suffix
 736         factory.addStep(SetPropertyFromCommand(
 737                 name = "libc",
 738                 property = "libc",
 739                 description = "Finding libc suffix",
 740                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 741
 742         # install build key
 743         factory.addStep(StringDownload(
 744                 name = "dlkeybuildpub",
 745                 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
 746                 workerdest = "key-build.pub",
 747                 mode = 0o600,
 748                 doStepIf = IsUsignEnabled,
 749         ))
 750
 751         factory.addStep(StringDownload(
 752                 name = "dlkeybuild",
 753                 s = "# fake private key",
 754                 workerdest = "key-build",
 755                 mode = 0o600,
 756                 doStepIf = IsUsignEnabled,
 757         ))
 758
 759         factory.addStep(StringDownload(
 760                 name = "dlkeybuilducert",
 761                 s = "# fake certificate",
 762                 workerdest = "key-build.ucert",
 763                 mode = 0o600,
 764                 doStepIf = IsUsignEnabled,
 765         ))
 766
 767         # prepare dl
 768         factory.addStep(ShellCommand(
 769                 name = "dldir",
 770                 description = "Preparing dl/",
 771                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 772                 logEnviron = False,
 773                 want_stdout = False
 774         ))
 775
 776         # prepare tar
 777         factory.addStep(ShellCommand(
 778                 name = "dltar",
 779                 description = "Building and installing GNU tar",
 780                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
 781                 env = MakeEnv(tryccache=True),
 782                 haltOnFailure = True
 783         ))
 784
 785         # populate dl
 786         factory.addStep(ShellCommand(
 787                 name = "dlrun",
 788                 description = "Populating dl/",
 789                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
 790                 env = MakeEnv(),
 791                 logEnviron = False,
 792                 locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
 793         ))
 794
 795         factory.addStep(ShellCommand(
 796                 name = "cleanbase",
 797                 description = "Cleaning base-files",
 798                 command=["make", "package/base-files/clean", "V=s"]
 799         ))
 800
 801         # build
 802         factory.addStep(ShellCommand(
 803                 name = "tools",
 804                 description = "Building and installing tools",
 805                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
 806                 env = MakeEnv(tryccache=True),
 807                 haltOnFailure = True
 808         ))
 809
 810         factory.addStep(ShellCommand(
 811                 name = "toolchain",
 812                 description = "Building and installing toolchain",
 813                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
 814                 env = MakeEnv(),
 815                 haltOnFailure = True
 816         ))
 817
 818         factory.addStep(ShellCommand(
 819                 name = "kmods",
 820                 description = "Building kmods",
 821                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 822                 env = MakeEnv(),
 823                 haltOnFailure = True
 824         ))
 825
 826         # find kernel version
 827         factory.addStep(SetPropertyFromCommand(
 828                 name = "kernelversion",
 829                 property = "kernelversion",
 830                 description = "Finding the effective Kernel version",
 831                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 832                 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") }
 833         ))
 834
 835         factory.addStep(ShellCommand(
 836                 name = "pkgclean",
 837                 description = "Cleaning up package build",
 838                 command=["make", "package/cleanup", "V=s"]
 839         ))
 840
 841         factory.addStep(ShellCommand(
 842                 name = "pkgbuild",
 843                 description = "Building packages",
 844                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 845                 env = MakeEnv(),
 846                 haltOnFailure = True
 847         ))
 848
 849         factory.addStep(ShellCommand(
 850                 name = "pkginstall",
 851                 description = "Installing packages",
 852                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
 853                 env = MakeEnv(),
 854                 haltOnFailure = True
 855         ))
 856
 857         factory.addStep(ShellCommand(
 858                 name = "pkgindex",
 859                 description = "Indexing packages",
 860                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
 861                 env = MakeEnv(),
 862                 haltOnFailure = True
 863         ))
 864
 865         factory.addStep(ShellCommand(
 866                 name = "images",
 867                 description = "Building and installing images",
 868                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
 869                 env = MakeEnv(),
 870                 haltOnFailure = True
 871         ))
 872
 873         factory.addStep(ShellCommand(
 874                 name = "buildinfo",
 875                 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
 876                 command = "make -j1 buildinfo V=s || true",
 877                 env = MakeEnv(),
 878                 haltOnFailure = True
 879         ))
 880
 881         factory.addStep(ShellCommand(
 882                 name = "json_overview_image_info",
 883                 description = "Generate profiles.json in target folder",
 884                 command = "make -j1 json_overview_image_info V=s || true",
 885                 env = MakeEnv(),
 886                 haltOnFailure = True
 887         ))
 888
 889         factory.addStep(ShellCommand(
 890                 name = "checksums",
 891                 description = "Calculating checksums",
 892                 command=["make", "-j1", "checksum", "V=s"],
 893                 env = MakeEnv(),
 894                 haltOnFailure = True
 895         ))
 896
 897         factory.addStep(ShellCommand(
 898                 name = "kmoddir",
 899                 description = "Creating kmod directory",
 900                 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
 901                 haltOnFailure = True,
 902                 doStepIf = IsKmodArchiveEnabled,
 903         ))
 904
 905         factory.addStep(ShellCommand(
 906                 name = "kmodprepare",
 907                 description = "Preparing kmod archive",
 908                 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 909                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
 910                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 911                 haltOnFailure = True,
 912                 doStepIf = IsKmodArchiveEnabled,
 913         ))
 914
 915         factory.addStep(ShellCommand(
 916                 name = "kmodindex",
 917                 description = "Indexing kmod archive",
 918                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
 919                         Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 920                 env = MakeEnv(),
 921                 haltOnFailure = True,
 922                 doStepIf = IsKmodArchiveEnabled,
 923         ))
 924
 925         # sign
 926         factory.addStep(MasterShellCommand(
 927                 name = "signprepare",
 928                 description = "Preparing temporary signing directory",
 929                 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
 930                 haltOnFailure = True,
 931                 doStepIf = IsSignEnabled,
 932
 933         ))
 934
 935         factory.addStep(ShellCommand(
 936                 name = "signpack",
 937                 description = "Packing files to sign",
 938                 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
 939                 haltOnFailure = True,
 940                 doStepIf = IsSignEnabled,
 941         ))
 942
 943         factory.addStep(FileUpload(
 944                 workersrc = "sign.tar.gz",
 945                 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
 946                 haltOnFailure = True,
 947                 doStepIf = IsSignEnabled,
 948         ))
 949
 950         factory.addStep(MasterShellCommand(
 951                 name = "signfiles",
 952                 description = "Signing files",
 953                 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
 954                 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
 955                 haltOnFailure = True,
 956                 doStepIf = IsSignEnabled,
 957         ))
 958
 959         factory.addStep(FileDownload(
 960                 name = "dlsigntargz",
 961                 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
 962                 workerdest = "sign.tar.gz",
 963                 haltOnFailure = True,
 964                 doStepIf = IsSignEnabled,
 965         ))
 966
 967         factory.addStep(ShellCommand(
 968                 name = "signunpack",
 969                 description = "Unpacking signed files",
 970                 command = ["tar", "-xzf", "sign.tar.gz"],
 971                 haltOnFailure = True,
 972                 doStepIf = IsSignEnabled,
 973         ))
 974
 975         # upload
 976         factory.addStep(ShellCommand(
 977                 name = "dirprepare",
 978                 description = "Preparing upload directory structure",
 979                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 980                 haltOnFailure = True
 981         ))
 982
 983         factory.addStep(ShellCommand(
 984                 name = "linkprepare",
 985                 description = "Preparing repository symlink",
 986                 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
 987                 doStepIf = IsNoMasterBuild,
 988                 haltOnFailure = True
 989         ))
 990
 991         factory.addStep(ShellCommand(
 992                 name = "kmoddirprepare",
 993                 description = "Preparing kmod archive upload directory",
 994                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 995                 haltOnFailure = True,
 996                 doStepIf = IsKmodArchiveEnabled,
 997         ))
 998
 999         factory.addStep(ShellCommand(
1000                 name = "dirupload",
1001                 description = "Uploading directory structure",
1002                 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1003                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1004                 haltOnFailure = True,
1005                 logEnviron = False,
1006                 locks = NetLockUl,
1007         ))
1008
1009         # download remote sha256sums to 'target-sha256sums'
1010         factory.addStep(ShellCommand(
1011                 name = "target-sha256sums",
1012                 description = "Fetching remote sha256sums for target",
1013                 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1014                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1015                 logEnviron = False,
1016                 haltOnFailure = False,
1017                 flunkOnFailure = False,
1018                 warnOnFailure = False,
1019         ))
1020
1021         # build list of files to upload
1022         factory.addStep(FileDownload(
1023                 name = "dlsha2rsyncpl",
1024                 mastersrc = scripts_dir + '/sha2rsync.pl',
1025                 workerdest = "../sha2rsync.pl",
1026                 mode = 0o755,
1027         ))
1028
1029         factory.addStep(ShellCommand(
1030                 name = "buildlist",
1031                 description = "Building list of files to upload",
1032                 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1033                 haltOnFailure = True,
1034         ))
1035
1036         factory.addStep(FileDownload(
1037                 name = "dlrsync.sh",
1038                 mastersrc = scripts_dir + '/rsync.sh',
1039                 workerdest = "../rsync.sh",
1040                 mode = 0o755
1041         ))
1042
1043         # upload new files and update existing ones
1044         factory.addStep(ShellCommand(
1045                 name = "targetupload",
1046                 description = "Uploading target files",
1047                 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1048                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1049                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1050                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1051                 haltOnFailure = True,
1052                 logEnviron = False,
1053         ))
1054
1055         # delete files which don't exist locally
1056         factory.addStep(ShellCommand(
1057                 name = "targetprune",
1058                 description = "Pruning target files",
1059                 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1060                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1061                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1062                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1063                 haltOnFailure = True,
1064                 logEnviron = False,
1065                 locks = NetLockUl,
1066         ))
1067
1068         factory.addStep(ShellCommand(
1069                 name = "kmodupload",
1070                 description = "Uploading kmod archive",
1071                 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1072                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1073                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1074                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1075                 haltOnFailure = True,
1076                 logEnviron = False,
1077                 locks = NetLockUl,
1078                 doStepIf = IsKmodArchiveEnabled,
1079         ))
1080
1081         factory.addStep(ShellCommand(
1082                 name = "sourcelist",
1083                 description = "Finding source archives to upload",
1084                 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1085                 haltOnFailure = True
1086         ))
1087
1088         factory.addStep(ShellCommand(
1089                 name = "sourceupload",
1090                 description = "Uploading source archives",
1091                 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1092                         [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1093                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1094                 haltOnFailure = True,
1095                 logEnviron = False,
1096                 locks = NetLockUl,
1097         ))
1098
1099         factory.addStep(ShellCommand(
1100                 name = "df",
1101                 description = "Reporting disk usage",
1102                 command=["df", "-h", "."],
1103                 env={'LC_ALL': 'C'},
1104                 haltOnFailure = False,
1105                 flunkOnFailure = False,
1106                 warnOnFailure = False,
1107                 alwaysRun = True
1108         ))
1109
1110         factory.addStep(ShellCommand(
1111                 name = "du",
1112                 description = "Reporting estimated file space usage",
1113                 command=["du", "-sh", "."],
1114                 env={'LC_ALL': 'C'},
1115                 haltOnFailure = False,
1116                 flunkOnFailure = False,
1117                 warnOnFailure = False,
1118                 alwaysRun = True
1119         ))
1120
1121         factory.addStep(ShellCommand(
1122                 name = "ccachestat",
1123                 description = "Reporting ccache stats",
1124                 command=["ccache", "-s"],
1125                 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1126                 want_stderr = False,
1127                 haltOnFailure = False,
1128                 flunkOnFailure = False,
1129                 warnOnFailure = False,
1130                 alwaysRun = True,
1131         ))
1132
1133         c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1134
1135         c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1136         force_factory.addStep(steps.Trigger(
1137                 name = "trigger_%s" % target,
1138                 description = "Triggering %s build" % target,
1139                 schedulerNames = [ "trigger_%s" % target ],
1140                 set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
1141                 doStepIf = IsTargetSelected(target)
1142         ))
1143
1144
1145 ####### STATUS TARGETS
1146
1147 # 'status' is a list of Status Targets. The results of each build will be
1148 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1149 # including web pages, email senders, and IRC bots.
1150
1151 if "status_bind" in inip1:
1152         c['www'] = {
1153                 'port': inip1.get("status_bind"),
1154                 'plugins': {
1155                         'waterfall_view': True,
1156                         'console_view': True,
1157                         'grid_view': True
1158                 }
1159         }
1160
1161         if "status_user" in inip1 and "status_password" in inip1:
1162                 c['www']['auth'] = util.UserPasswordAuth([
1163                         (inip1.get("status_user"), inip1.get("status_password"))
1164                 ])
1165                 c['www']['authz'] = util.Authz(
1166                         allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1167                         roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1168                 )
1169
1170 c['services'] = []
1171 if ini.has_section("irc"):
1172         iniirc = ini['irc']
1173         irc_host = iniirc.get("host", None)
1174         irc_port = iniirc.getint("port", 6667)
1175         irc_chan = iniirc.get("channel", None)
1176         irc_nick = iniirc.get("nickname", None)
1177         irc_pass = iniirc.get("password", None)
1178
1179         if irc_host and irc_nick and irc_chan:
1180                 irc = reporters.IRC(irc_host, irc_nick,
1181                         port = irc_port,
1182                         password = irc_pass,
1183                         channels = [ irc_chan ],
1184                         notify_events = [ 'exception', 'problem', 'recovery' ]
1185                 )
1186
1187                 c['services'].append(irc)
1188
1189 c['revlink'] = util.RevlinkMatch([
1190         r'https://git.openwrt.org/openwrt/(.*).git'
1191         ],
1192         r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1193
1194 ####### DB URL
1195
1196 c['db'] = {
1197         # This specifies what database buildbot uses to store its state.  You can leave
1198         # this at its default for all but the largest installations.
1199         'db_url' : "sqlite:///state.sqlite",
1200 }
1201
1202 c['buildbotNetUsageData'] = None