phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41 from buildbot.worker.local import LocalWorker
  42
  43
  44 if not os.path.exists("twistd.pid"):
  45     with open("twistd.pid", "w") as pidfile:
  46         pidfile.write("{}".format(os.getpid()))
  47
  48 # This is a sample buildmaster config file. It must be installed as
  49 # 'master.cfg' in your buildmaster's base directory.
  50
  51 ini = configparser.ConfigParser()
  52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  53
  54 if "general" not in ini or "phase1" not in ini:
  55     raise ValueError("Fix your configuration")
  56
  57 inip1 = ini["phase1"]
  58
  59 # Globals
  60 work_dir = os.path.abspath(ini["general"].get("workdir", "."))
  61 scripts_dir = os.path.abspath("../scripts")
  62
  63 repo_url = ini["repo"].get("url")
  64
  65 rsync_defopts = ["-v", "--timeout=120"]
  66
  67 # if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  68 #       rsync_bin_defopts += ["--contimeout=20"]
  69
  70 branches = {}
  71
  72
  73 def ini_parse_branch(section):
  74     b = {}
  75     name = section.get("name")
  76
  77     if not name:
  78         raise ValueError("missing 'name' in " + repr(section))
  79     if name in branches:
  80         raise ValueError("duplicate branch name in " + repr(section))
  81
  82     b["name"] = name
  83     b["bin_url"] = section.get("binary_url")
  84     b["bin_key"] = section.get("binary_password")
  85
  86     b["src_url"] = section.get("source_url")
  87     b["src_key"] = section.get("source_password")
  88
  89     b["gpg_key"] = section.get("gpg_key")
  90
  91     b["usign_key"] = section.get("usign_key")
  92     usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  93     b["usign_comment"] = section.get("usign_comment", usign_comment)
  94
  95     b["config_seed"] = section.get("config_seed")
  96
  97     b["kmod_archive"] = section.getboolean("kmod_archive", False)
  98
  99     branches[name] = b
 100     log.msg("Configured branch: {}".format(name))
 101
 102
 103 # PB port can be either a numeric port or a connection string
 104 pb_port = inip1.get("port") or 9989
 105
 106 # This is the dictionary that the buildmaster pays attention to. We also use
 107 # a shorter alias to save typing.
 108 c = BuildmasterConfig = {}
 109
 110 ####### PROJECT IDENTITY
 111
 112 # the 'title' string will appear at the top of this buildbot
 113 # installation's html.WebStatus home page (linked to the
 114 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 115
 116 c["title"] = ini["general"].get("title")
 117 c["titleURL"] = ini["general"].get("title_url")
 118
 119 # the 'buildbotURL' string should point to the location where the buildbot's
 120 # internal web server (usually the html.WebStatus page) is visible. This
 121 # typically uses the port number set in the Waterfall 'status' entry, but
 122 # with an externally-visible host name which the buildbot cannot figure out
 123 # without some help.
 124
 125 c["buildbotURL"] = inip1.get("buildbot_url")
 126
 127 ####### BUILDWORKERS
 128
 129 # The 'workers' list defines the set of recognized buildworkers. Each element is
 130 # a Worker object, specifying a unique worker name and password.  The same
 131 # worker name and password must be configured on the worker.
 132
 133 c["workers"] = []
 134 NetLocks = dict()
 135
 136
 137 def ini_parse_workers(section):
 138     name = section.get("name")
 139     password = section.get("password")
 140     phase = section.getint("phase")
 141     tagonly = section.getboolean("tag_only")
 142     rsyncipv4 = section.getboolean("rsync_ipv4")
 143
 144     if not name or not password or not phase == 1:
 145         log.msg("invalid worker configuration ignored: {}".format(repr(section)))
 146         return
 147
 148     sl_props = {"tag_only": tagonly}
 149     if "dl_lock" in section:
 150         lockname = section.get("dl_lock")
 151         sl_props["dl_lock"] = lockname
 152         if lockname not in NetLocks:
 153             NetLocks[lockname] = locks.MasterLock(lockname)
 154     if "ul_lock" in section:
 155         lockname = section.get("ul_lock")
 156         sl_props["ul_lock"] = lockname
 157         if lockname not in NetLocks:
 158             NetLocks[lockname] = locks.MasterLock(lockname)
 159     if rsyncipv4:
 160         sl_props[
 161             "rsync_ipv4"
 162         ] = True  # only set prop if required, we use '+' Interpolate substitution
 163
 164     log.msg("Configured worker: {}".format(name))
 165     # NB: phase1 build factory requires workers to be single-build only
 166     c["workers"].append(Worker(name, password, max_builds=1, properties=sl_props))
 167
 168
 169 for section in ini.sections():
 170     if section.startswith("branch "):
 171         ini_parse_branch(ini[section])
 172
 173     if section.startswith("worker "):
 174         ini_parse_workers(ini[section])
 175
 176 # list of branches in build-priority order
 177 branchNames = [branches[b]["name"] for b in branches]
 178
 179 c["protocols"] = {"pb": {"port": pb_port}}
 180
 181 # coalesce builds
 182 c["collapseRequests"] = True
 183
 184 # Reduce amount of backlog data
 185 c["configurators"] = [
 186     util.JanitorConfigurator(
 187         logHorizon=timedelta(days=3),
 188         hour=6,
 189     )
 190 ]
 191
 192
 193 @defer.inlineCallbacks
 194 def getNewestCompleteTimePrio(bldr):
 195     """Returns the priority and the complete_at of the latest completed and not SKIPPED
 196     build request for this builder, or None if there are no such build
 197     requests. We need to filter out SKIPPED requests because we're
 198     using collapseRequests=True which is unfortunately marking all
 199     previous requests as complete when new buildset is created.
 200
 201     @returns: (priority, datetime instance or None), via Deferred
 202     """
 203
 204     prio = yield bldr.get_highest_priority()
 205     if prio is None:
 206         prio = 0
 207
 208     bldrid = yield bldr.getBuilderId()
 209     completed = yield bldr.master.data.get(
 210         ("builders", bldrid, "buildrequests"),
 211         [
 212             resultspec.Filter("complete", "eq", [True]),
 213             resultspec.Filter("results", "ne", [results.SKIPPED]),
 214         ],
 215         order=["-complete_at"],
 216         limit=1,
 217     )
 218     if not completed:
 219         return (prio, None)
 220
 221     complete_at = completed[0]["complete_at"]
 222
 223     last_build = yield bldr.master.data.get(
 224         ("builds",),
 225         [
 226             resultspec.Filter("builderid", "eq", [bldrid]),
 227         ],
 228         order=["-started_at"],
 229         limit=1,
 230     )
 231
 232     if last_build and last_build[0]:
 233         last_complete_at = last_build[0]["complete_at"]
 234         if last_complete_at and (last_complete_at > complete_at):
 235             return (prio, last_complete_at)
 236
 237     return (prio, complete_at)
 238
 239
 240 @defer.inlineCallbacks
 241 def prioritizeBuilders(master, builders):
 242     """Returns sorted list of builders by their last timestamp of completed and
 243     not skipped build, ordered first by branch name.
 244
 245     @returns: list of sorted builders
 246     """
 247
 248     bldrNamePrio = {"__Janitor": 0, "00_force_build": 0}
 249     i = 1
 250     for bname in branchNames:
 251         bldrNamePrio[bname] = i
 252
 253     def is_building(bldr):
 254         return bool(bldr.building) or bool(bldr.old_building)
 255
 256     def bldr_info(bldr):
 257         d = defer.maybeDeferred(getNewestCompleteTimePrio, bldr)
 258         d.addCallback(lambda retval: (retval, bldr))
 259         return d
 260
 261     def bldr_sort(item):
 262         ((hiprio, complete_at), bldr) = item
 263
 264         # check if we have some high prio build requests pending (i.e. tag builds),
 265         # if so, front-run these builders, while preserving the per-branch static priority
 266         pos = 99
 267         for name, prio in bldrNamePrio.items():
 268             if bldr.name.startswith(name):
 269                 pos = prio + 50 - min(hiprio, 50)    # higher priority (larger positive number) raises position
 270                 break
 271
 272         # pos order: janitor/local (0), tag builds if any [1..50], !tag builds [51...]
 273
 274         if not complete_at:
 275             date = datetime.min
 276             complete_at = date.replace(tzinfo=tzutc())
 277
 278         if is_building(bldr):
 279             date = datetime.max
 280             complete_at = date.replace(tzinfo=tzutc())
 281
 282         return (pos, complete_at, bldr.name)
 283
 284     results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 285     results.sort(key=bldr_sort)
 286
 287     # for r in results:
 288     #   log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 289
 290     return [r[1] for r in results]
 291
 292
 293 c["prioritizeBuilders"] = prioritizeBuilders
 294
 295 ####### CHANGESOURCES
 296
 297 # find targets
 298 targets = dict()
 299
 300
 301 def populateTargets():
 302     """fetch a shallow clone of each configured branch in turn:
 303     execute dump-target-info.pl and collate the results to ensure
 304     targets that only exist in specific branches get built.
 305     This takes a while during master startup but is executed only once.
 306     """
 307     sourcegit = work_dir + "/source.git"
 308     for branch in branchNames:
 309         log.msg(f"Populating targets for {branch}, this will take time")
 310
 311         if os.path.isdir(sourcegit):
 312             subprocess.call(["rm", "-rf", sourcegit])
 313
 314         subprocess.call(
 315             [
 316                 "git",
 317                 "clone",
 318                 "-q",
 319                 "--depth=1",
 320                 "--branch=" + branch,
 321                 repo_url,
 322                 sourcegit,
 323             ]
 324         )
 325
 326         os.makedirs(sourcegit + "/tmp", exist_ok=True)
 327         findtargets = subprocess.Popen(
 328             ["./scripts/dump-target-info.pl", "targets"],
 329             stdout=subprocess.PIPE,
 330             stderr=subprocess.DEVNULL,
 331             cwd=sourcegit,
 332         )
 333
 334         targets[branch] = set()
 335         while True:
 336             line = findtargets.stdout.readline()
 337             if not line:
 338                 break
 339             ta = line.decode().strip().split(" ")
 340             targets[branch].add(ta[0])
 341
 342         subprocess.call(["rm", "-rf", sourcegit])
 343
 344
 345 populateTargets()
 346
 347 # the 'change_source' setting tells the buildmaster how it should find out
 348 # about source code changes.
 349
 350 c["change_source"] = []
 351 c["change_source"].append(
 352     GitPoller(
 353         repo_url,
 354         workdir=work_dir + "/work.git",
 355         branches=branchNames,
 356         pollAtLaunch=True,
 357         pollinterval=300,
 358     )
 359 )
 360
 361 ####### SCHEDULERS
 362
 363 # Configure the Schedulers, which decide how to react to incoming changes.
 364
 365
 366 # Selector for known valid tags
 367 class TagChoiceParameter(BaseParameter):
 368     spec_attributes = ["strict", "choices"]
 369     type = "list"
 370     strict = True
 371
 372     def __init__(self, name, label=None, **kw):
 373         super().__init__(name, label, **kw)
 374         self._choice_list = []
 375
 376     def getRevTags(self, findtag=None):
 377         taglist = []
 378         branchvers = []
 379
 380         # we will filter out tags that do no match the configured branches
 381         for b in branchNames:
 382             basever = re.search(r"-([0-9]+\.[0-9]+)$", b)
 383             if basever:
 384                 branchvers.append(basever[1])
 385
 386         # grab tags from remote repository
 387         alltags = subprocess.Popen(
 388             ["git", "ls-remote", "--tags", repo_url], stdout=subprocess.PIPE
 389         )
 390
 391         while True:
 392             line = alltags.stdout.readline()
 393
 394             if not line:
 395                 break
 396
 397             (rev, tag) = line.split()
 398
 399             # does it match known format? ('vNN.NN.NN(-rcN)')
 400             tagver = re.search(
 401                 r"\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$",
 402                 tag.decode().strip(),
 403             )
 404
 405             # only list valid tags matching configured branches
 406             if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
 407                 # if we want a specific tag, ignore all that don't match
 408                 if findtag and findtag != tagver[1]:
 409                     continue
 410                 taglist.append({"rev": rev.decode().strip(), "tag": tagver[1]})
 411
 412         return taglist
 413
 414     @property
 415     def choices(self):
 416         taglist = [rt["tag"] for rt in self.getRevTags()]
 417         taglist.sort(
 418             reverse=True,
 419             key=lambda tag: tag if re.search(r"-rc[0-9]+$", tag) else tag + "-z",
 420         )
 421         taglist.insert(0, "")
 422
 423         self._choice_list = taglist
 424
 425         return self._choice_list
 426
 427     def updateFromKwargs(self, properties, kwargs, **unused):
 428         tag = self.getFromKwargs(kwargs)
 429         properties[self.name] = tag
 430
 431         # find the commit matching the tag
 432         findtag = self.getRevTags(tag)
 433
 434         if not findtag:
 435             raise ValidationError("Couldn't find tag")
 436
 437         properties["force_revision"] = findtag[0]["rev"]
 438
 439         # find the branch matching the tag
 440         branch = None
 441         branchver = re.search(r"v([0-9]+\.[0-9]+)", tag)
 442         for b in branchNames:
 443             if b.endswith(branchver[1]):
 444                 branch = b
 445
 446         if not branch:
 447             raise ValidationError("Couldn't find branch")
 448
 449         properties["force_branch"] = branch
 450
 451     def parse_from_arg(self, s):
 452         if self.strict and s not in self._choice_list:
 453             raise ValidationError(
 454                 "'%s' does not belong to list of available choices '%s'"
 455                 % (s, self._choice_list)
 456             )
 457         return s
 458
 459
 460 @util.renderer
 461 @defer.inlineCallbacks
 462 def builderNames(props):
 463     """since we have per branch and per target builders,
 464     address the relevant builder for each new buildrequest
 465     based on the request's desired branch and target.
 466     """
 467     branch = props.getProperty("branch")
 468     target = props.getProperty("target", "")
 469
 470     if target == "all":
 471         target = ""
 472
 473     # if that didn't work, try sourcestamp to find a branch
 474     if not branch:
 475         # match builders with target branch
 476         ss = props.sourcestamps[0]
 477         if ss:
 478             branch = ss["branch"]
 479         else:
 480             log.msg("couldn't find builder")
 481             return []  # nothing works
 482
 483     bname = branch + "_" + target
 484     builders = []
 485
 486     for b in (yield props.master.data.get(("builders",))):
 487         if not b["name"].startswith(bname):
 488             continue
 489         builders.append(b["name"])
 490
 491     return builders
 492
 493
 494 c["schedulers"] = []
 495 c["schedulers"].append(
 496     AnyBranchScheduler(
 497         name="all",
 498         change_filter=util.ChangeFilter(branch=branchNames),
 499         treeStableTimer=15 * 60,
 500         builderNames=builderNames,
 501     )
 502 )
 503
 504 c["schedulers"].append(
 505     ForceScheduler(
 506         name="force",
 507         buttonName="Force builds",
 508         label="Force build details",
 509         builderNames=["00_force_build"],
 510         codebases=[
 511             util.CodebaseParameter(
 512                 "",
 513                 label="Repository",
 514                 branch=util.FixedParameter(name="branch", default=""),
 515                 revision=util.FixedParameter(name="revision", default=""),
 516                 repository=util.FixedParameter(name="repository", default=""),
 517                 project=util.FixedParameter(name="project", default=""),
 518             )
 519         ],
 520         reason=util.StringParameter(
 521             name="reason",
 522             label="Reason",
 523             default="Trigger build",
 524             required=True,
 525             size=80,
 526         ),
 527         properties=[
 528             # NB: avoid nesting to simplify processing of properties
 529             util.ChoiceStringParameter(
 530                 name="target",
 531                 label="Build target",
 532                 default="all",
 533                 choices=["all"] + [t for b in branchNames for t in targets[b]],
 534             ),
 535             TagChoiceParameter(name="tag", label="Build tag", default=""),
 536         ],
 537     )
 538 )
 539
 540 c["schedulers"].append(
 541     schedulers.Triggerable(name="trigger", builderNames=builderNames, priority=20)
 542 )
 543
 544 ####### BUILDERS
 545
 546 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 547 # what steps, and which workers can execute them.  Note that any particular build will
 548 # only take place on one worker.
 549
 550
 551 def IsNoMasterBuild(step):
 552     return step.getProperty("branch") != "master"
 553
 554
 555 def IsUsignEnabled(step):
 556     branch = step.getProperty("branch")
 557     return branch and branches[branch].get("usign_key")
 558
 559
 560 def IsSignEnabled(step):
 561     branch = step.getProperty("branch")
 562     return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 563
 564
 565 def IsKmodArchiveEnabled(step):
 566     branch = step.getProperty("branch")
 567     return branch and branches[branch].get("kmod_archive")
 568
 569
 570 def IsKmodArchiveAndRsyncEnabled(step):
 571     branch = step.getProperty("branch")
 572     return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url"))
 573
 574
 575 def GetBaseVersion(branch):
 576     if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 577         return branch.split("-")[1]
 578     else:
 579         return "master"
 580
 581
 582 @properties.renderer
 583 def GetVersionPrefix(props):
 584     branch = props.getProperty("branch")
 585     basever = GetBaseVersion(branch)
 586     if props.hasProperty("tag") and re.match(
 587         r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]
 588     ):
 589         return "%s/" % props["tag"][1:]
 590     elif basever != "master":
 591         return "%s-SNAPSHOT/" % basever
 592     else:
 593         return ""
 594
 595
 596 @util.renderer
 597 def GetConfigSeed(props):
 598     branch = props.getProperty("branch")
 599     return branch and branches[branch].get("config_seed") or ""
 600
 601
 602 @util.renderer
 603 def GetRsyncParams(props, srcorbin, urlorkey):
 604     # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 605     branch = props.getProperty("branch")
 606     opt = srcorbin + "_" + urlorkey
 607     return branch and branches[branch].get(opt)
 608
 609
 610 @util.renderer
 611 def GetUsignKey(props):
 612     branch = props.getProperty("branch")
 613     return branch and branches[branch].get("usign_key")
 614
 615
 616 def GetNextBuild(builder, requests):
 617     for r in requests:
 618         if r.properties:
 619             # order tagged build first
 620             if r.properties.hasProperty("tag"):
 621                 return r
 622
 623     r = requests[0]
 624     # log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 625     return r
 626
 627
 628 def MakeEnv(overrides=None, tryccache=False):
 629     env = {
 630         "CCC": Interpolate("%(prop:cc_command:-gcc)s"),
 631         "CCXX": Interpolate("%(prop:cxx_command:-g++)s"),
 632     }
 633     if tryccache:
 634         env["CC"] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 635         env["CXX"] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 636         env["CCACHE"] = Interpolate("%(prop:ccache_command:-)s")
 637     else:
 638         env["CC"] = env["CCC"]
 639         env["CXX"] = env["CCXX"]
 640         env["CCACHE"] = ""
 641     if overrides is not None:
 642         env.update(overrides)
 643     return env
 644
 645
 646 @properties.renderer
 647 def NetLockDl(props, extralock=None):
 648     lock = None
 649     if props.hasProperty("dl_lock"):
 650         lock = NetLocks[props["dl_lock"]]
 651     if lock is not None:
 652         return [lock.access("exclusive")]
 653     else:
 654         return []
 655
 656
 657 @properties.renderer
 658 def NetLockUl(props):
 659     lock = None
 660     if props.hasProperty("ul_lock"):
 661         lock = NetLocks[props["ul_lock"]]
 662     if lock is not None:
 663         return [lock.access("exclusive")]
 664     else:
 665         return []
 666
 667
 668 def IsTargetSelected(target):
 669     def CheckTargetProperty(step):
 670         selected_target = step.getProperty("target", "all")
 671         if selected_target != "all" and selected_target != target:
 672             return False
 673         return True
 674
 675     return CheckTargetProperty
 676
 677
 678 @util.renderer
 679 def UsignSec2Pub(props):
 680     branch = props.getProperty("branch")
 681     try:
 682         comment = (
 683             branches[branch].get("usign_comment") or "untrusted comment: secret key"
 684         )
 685         seckey = branches[branch].get("usign_key")
 686         seckey = base64.b64decode(seckey)
 687     except Exception:
 688         return None
 689
 690     return "{}\n{}".format(
 691         re.sub(r"\bsecret key$", "public key", comment),
 692         base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]),
 693     )
 694
 695
 696 def canStartBuild(builder, wfb, request):
 697     """filter out non tag requests for tag_only workers."""
 698     wtagonly = wfb.worker.properties.getProperty("tag_only")
 699     tag = request.properties.getProperty("tag")
 700
 701     if wtagonly and not tag:
 702         return False
 703
 704     return True
 705
 706
 707 c["builders"] = []
 708
 709 workerNames = []
 710
 711 for worker in c["workers"]:
 712     workerNames.append(worker.workername)
 713
 714 # add a single LocalWorker to handle the forcebuild builder
 715 c["workers"].append(LocalWorker("__local_force_build", max_builds=1))
 716
 717 force_factory = BuildFactory()
 718 force_factory.addStep(
 719     steps.Trigger(
 720         name="trigger_build",
 721         schedulerNames=["trigger"],
 722         sourceStamps=[
 723             {
 724                 "codebase": "",
 725                 "branch": Property("force_branch"),
 726                 "revision": Property("force_revision"),
 727                 "repository": repo_url,
 728                 "project": "",
 729             }
 730         ],
 731         set_properties={
 732             "reason": Property("reason"),
 733             "tag": Property("tag"),
 734             "target": Property("target"),
 735         },
 736     )
 737 )
 738
 739 c["builders"].append(
 740     BuilderConfig(
 741         name="00_force_build", workername="__local_force_build", factory=force_factory
 742     )
 743 )
 744
 745
 746 # NB the phase1 build factory assumes workers are single-build only
 747 def prepareFactory(target):
 748     (target, subtarget) = target.split("/")
 749
 750     factory = BuildFactory()
 751
 752     # setup shared work directory if required
 753     factory.addStep(
 754         ShellCommand(
 755             name="sharedwd",
 756             descriptionDone="Shared work directory set up",
 757             command='test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 758             workdir=".",
 759             haltOnFailure=True,
 760         )
 761     )
 762
 763     # find number of cores
 764     factory.addStep(
 765         SetPropertyFromCommand(
 766             name="nproc",
 767             property="nproc",
 768             description="Finding number of CPUs",
 769             command=["nproc"],
 770         )
 771     )
 772
 773     # find gcc and g++ compilers
 774     factory.addStep(
 775         FileDownload(
 776             name="dlfindbinpl",
 777             mastersrc=scripts_dir + "/findbin.pl",
 778             workerdest="../findbin.pl",
 779             mode=0o755,
 780         )
 781     )
 782
 783     factory.addStep(
 784         SetPropertyFromCommand(
 785             name="gcc",
 786             property="cc_command",
 787             description="Finding gcc command",
 788             command=["../findbin.pl", "gcc", "", ""],
 789             haltOnFailure=True,
 790         )
 791     )
 792
 793     factory.addStep(
 794         SetPropertyFromCommand(
 795             name="g++",
 796             property="cxx_command",
 797             description="Finding g++ command",
 798             command=["../findbin.pl", "g++", "", ""],
 799             haltOnFailure=True,
 800         )
 801     )
 802
 803     # see if ccache is available
 804     factory.addStep(
 805         SetPropertyFromCommand(
 806             name="ccache",
 807             property="ccache_command",
 808             description="Testing for ccache command",
 809             command=["which", "ccache"],
 810             haltOnFailure=False,
 811             flunkOnFailure=False,
 812             warnOnFailure=False,
 813             hideStepIf=lambda r, s: r == results.FAILURE,
 814         )
 815     )
 816
 817     # check out the source
 818     # Git() runs:
 819     # if repo doesn't exist: 'git clone repourl'
 820     # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 821     # git cat-file -e <commit>
 822     # git checkout -f <commit>
 823     # git checkout -B <branch>
 824     # git rev-parse HEAD
 825     factory.addStep(
 826         Git(
 827             name="git",
 828             repourl=repo_url,
 829             mode="full",
 830             method="fresh",
 831             locks=NetLockDl,
 832             haltOnFailure=True,
 833         )
 834     )
 835
 836     # workaround for https://github.com/openwrt/buildbot/issues/5
 837     factory.addStep(
 838         Git(
 839             name="git me once more please",
 840             repourl=repo_url,
 841             mode="full",
 842             method="fresh",
 843             locks=NetLockDl,
 844             haltOnFailure=True,
 845         )
 846     )
 847
 848     # update remote refs
 849     factory.addStep(
 850         ShellCommand(
 851             name="fetchrefs",
 852             description="Fetching Git remote refs",
 853             descriptionDone="Git remote refs fetched",
 854             command=[
 855                 "git",
 856                 "fetch",
 857                 "origin",
 858                 Interpolate(
 859                     "+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s"
 860                 ),
 861             ],
 862             haltOnFailure=True,
 863         )
 864     )
 865
 866     # getver.sh requires local branches to track upstream otherwise version computation fails.
 867     # Git() does not set tracking branches when cloning or switching, so work around this here
 868     factory.addStep(
 869         ShellCommand(
 870             name="trackupstream",
 871             description="Setting upstream branch",
 872             descriptionDone="getver.sh is happy now",
 873             command=["git", "branch", "-u", Interpolate("origin/%(prop:branch)s")],
 874             haltOnFailure=True,
 875         )
 876     )
 877
 878     # Verify that Git HEAD points to a tag or branch
 879     # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 880     factory.addStep(
 881         ShellCommand(
 882             name="gitverify",
 883             description="Ensuring that Git HEAD is pointing to a branch or tag",
 884             descriptionDone="Git HEAD is sane",
 885             command='git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
 886             haltOnFailure=True,
 887         )
 888     )
 889
 890     factory.addStep(
 891         StringDownload(
 892             name="ccachecc",
 893             s='#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 894             workerdest="../ccache_cc.sh",
 895             mode=0o755,
 896         )
 897     )
 898
 899     factory.addStep(
 900         StringDownload(
 901             name="ccachecxx",
 902             s='#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 903             workerdest="../ccache_cxx.sh",
 904             mode=0o755,
 905         )
 906     )
 907
 908     # feed
 909     factory.addStep(
 910         ShellCommand(
 911             name="updatefeeds",
 912             description="Updating feeds",
 913             command=["./scripts/feeds", "update"],
 914             env=MakeEnv(tryccache=True),
 915             haltOnFailure=True,
 916             locks=NetLockDl,
 917         )
 918     )
 919
 920     # feed
 921     factory.addStep(
 922         ShellCommand(
 923             name="installfeeds",
 924             description="Installing feeds",
 925             command=["./scripts/feeds", "install", "-a"],
 926             env=MakeEnv(tryccache=True),
 927             haltOnFailure=True,
 928         )
 929     )
 930
 931     # seed config
 932     factory.addStep(
 933         StringDownload(
 934             name="dlconfigseed",
 935             s=Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 936             workerdest=".config",
 937             mode=0o644,
 938         )
 939     )
 940
 941     # configure
 942     factory.addStep(
 943         ShellCommand(
 944             name="newconfig",
 945             descriptionDone=".config seeded",
 946             command=Interpolate(
 947                 "printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config",
 948                 target=target,
 949                 subtarget=subtarget,
 950                 usign=GetUsignKey,
 951             ),
 952         )
 953     )
 954
 955     factory.addStep(
 956         ShellCommand(
 957             name="defconfig",
 958             description="Populating .config",
 959             command=["make", "defconfig"],
 960             env=MakeEnv(),
 961         )
 962     )
 963
 964     # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 965     factory.addStep(
 966         ShellCommand(
 967             name="checkarch",
 968             description="Checking architecture",
 969             descriptionDone="Architecture validated",
 970             command='grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config'
 971             % (target, subtarget),
 972             logEnviron=False,
 973             want_stdout=False,
 974             want_stderr=False,
 975             haltOnFailure=True,
 976             flunkOnFailure=False,  # this is not a build FAILURE - TODO mark build as SKIPPED
 977         )
 978     )
 979
 980     # find libc suffix
 981     factory.addStep(
 982         SetPropertyFromCommand(
 983             name="libc",
 984             property="libc",
 985             description="Finding libc suffix",
 986             command=[
 987                 "sed",
 988                 "-ne",
 989                 '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }',
 990                 ".config",
 991             ],
 992         )
 993     )
 994
 995     # install build key
 996     factory.addStep(
 997         StringDownload(
 998             name="dlkeybuildpub",
 999             s=Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
1000             workerdest="key-build.pub",
1001             mode=0o600,
1002             doStepIf=IsUsignEnabled,
1003         )
1004     )
1005
1006     factory.addStep(
1007         StringDownload(
1008             name="dlkeybuild",
1009             s="# fake private key",
1010             workerdest="key-build",
1011             mode=0o600,
1012             doStepIf=IsUsignEnabled,
1013         )
1014     )
1015
1016     factory.addStep(
1017         StringDownload(
1018             name="dlkeybuilducert",
1019             s="# fake certificate",
1020             workerdest="key-build.ucert",
1021             mode=0o600,
1022             doStepIf=IsUsignEnabled,
1023         )
1024     )
1025
1026     # prepare dl
1027     factory.addStep(
1028         ShellCommand(
1029             name="dldir",
1030             description="Preparing dl/",
1031             descriptionDone="dl/ prepared",
1032             command='mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
1033             workdir=Property("builddir"),
1034             logEnviron=False,
1035             want_stdout=False,
1036         )
1037     )
1038
1039     # cleanup dl
1040     factory.addStep(
1041         ShellCommand(
1042             name="dlprune",
1043             description="Pruning dl/",
1044             descriptionDone="dl/ pruned",
1045             command="find dl/ -mindepth 1 -atime +15 -delete -print",
1046             logEnviron=False,
1047             haltOnFailure=False,
1048             flunkOnFailure=False,
1049             warnOnFailure=False,
1050         )
1051     )
1052
1053     # prepare tar
1054     factory.addStep(
1055         ShellCommand(
1056             name="dltar",
1057             description="Building and installing GNU tar",
1058             descriptionDone="GNU tar built and installed",
1059             command=[
1060                 "make",
1061                 Interpolate("-j%(prop:nproc:-1)s"),
1062                 "tools/tar/compile",
1063                 "V=s",
1064             ],
1065             env=MakeEnv(tryccache=True),
1066             haltOnFailure=True,
1067         )
1068     )
1069
1070     # populate dl
1071     factory.addStep(
1072         ShellCommand(
1073             name="dlrun",
1074             description="Populating dl/",
1075             descriptionDone="dl/ populated",
1076             command=["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
1077             env=MakeEnv(),
1078             logEnviron=False,
1079             locks=NetLockDl,
1080         )
1081     )
1082
1083     factory.addStep(
1084         ShellCommand(
1085             name="cleanbase",
1086             description="Cleaning base-files",
1087             command=["make", "package/base-files/clean", "V=s"],
1088         )
1089     )
1090
1091     # build
1092     factory.addStep(
1093         ShellCommand(
1094             name="tools",
1095             description="Building and installing tools",
1096             descriptionDone="Tools built and installed",
1097             command=[
1098                 "make",
1099                 Interpolate("-j%(prop:nproc:-1)s"),
1100                 "tools/install",
1101                 "V=s",
1102             ],
1103             env=MakeEnv(tryccache=True),
1104             haltOnFailure=True,
1105         )
1106     )
1107
1108     factory.addStep(
1109         ShellCommand(
1110             name="toolchain",
1111             description="Building and installing toolchain",
1112             descriptionDone="Toolchain built and installed",
1113             command=[
1114                 "make",
1115                 Interpolate("-j%(prop:nproc:-1)s"),
1116                 "toolchain/install",
1117                 "V=s",
1118             ],
1119             env=MakeEnv(),
1120             haltOnFailure=True,
1121         )
1122     )
1123
1124     factory.addStep(
1125         ShellCommand(
1126             name="kmods",
1127             description="Building kmods",
1128             descriptionDone="Kmods built",
1129             command=[
1130                 "make",
1131                 Interpolate("-j%(prop:nproc:-1)s"),
1132                 "target/compile",
1133                 "V=s",
1134                 "IGNORE_ERRORS=n m",
1135                 "BUILD_LOG=1",
1136             ],
1137             env=MakeEnv(),
1138             haltOnFailure=True,
1139         )
1140     )
1141
1142     # find kernel version
1143     factory.addStep(
1144         SetPropertyFromCommand(
1145             name="kernelversion",
1146             property="kernelversion",
1147             description="Finding the effective Kernel version",
1148             command="make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
1149             env={"TOPDIR": Interpolate("%(prop:builddir)s/build")},
1150         )
1151     )
1152
1153     factory.addStep(
1154         ShellCommand(
1155             name="pkgclean",
1156             description="Cleaning up package build",
1157             descriptionDone="Package build cleaned up",
1158             command=["make", "package/cleanup", "V=s"],
1159         )
1160     )
1161
1162     factory.addStep(
1163         ShellCommand(
1164             name="pkgbuild",
1165             description="Building packages",
1166             descriptionDone="Packages built",
1167             command=[
1168                 "make",
1169                 Interpolate("-j%(prop:nproc:-1)s"),
1170                 "package/compile",
1171                 "V=s",
1172                 "IGNORE_ERRORS=n m",
1173                 "BUILD_LOG=1",
1174             ],
1175             env=MakeEnv(),
1176             haltOnFailure=True,
1177         )
1178     )
1179
1180     factory.addStep(
1181         ShellCommand(
1182             name="pkginstall",
1183             description="Installing packages",
1184             descriptionDone="Packages installed",
1185             command=[
1186                 "make",
1187                 Interpolate("-j%(prop:nproc:-1)s"),
1188                 "package/install",
1189                 "V=s",
1190             ],
1191             env=MakeEnv(),
1192             haltOnFailure=True,
1193         )
1194     )
1195
1196     factory.addStep(
1197         ShellCommand(
1198             name="pkgindex",
1199             description="Indexing packages",
1200             descriptionDone="Packages indexed",
1201             command=[
1202                 "make",
1203                 Interpolate("-j%(prop:nproc:-1)s"),
1204                 "package/index",
1205                 "V=s",
1206                 "CONFIG_SIGNED_PACKAGES=",
1207             ],
1208             env=MakeEnv(),
1209             haltOnFailure=True,
1210         )
1211     )
1212
1213     factory.addStep(
1214         ShellCommand(
1215             name="images",
1216             description="Building and installing images",
1217             descriptionDone="Images built and installed",
1218             command=[
1219                 "make",
1220                 Interpolate("-j%(prop:nproc:-1)s"),
1221                 "target/install",
1222                 "V=s",
1223             ],
1224             env=MakeEnv(),
1225             haltOnFailure=True,
1226         )
1227     )
1228
1229     factory.addStep(
1230         ShellCommand(
1231             name="buildinfo",
1232             description="Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
1233             command="make -j1 buildinfo V=s || true",
1234             env=MakeEnv(),
1235             haltOnFailure=True,
1236         )
1237     )
1238
1239     factory.addStep(
1240         ShellCommand(
1241             name="json_overview_image_info",
1242             description="Generating profiles.json in target folder",
1243             command="make -j1 json_overview_image_info V=s || true",
1244             env=MakeEnv(),
1245             haltOnFailure=True,
1246         )
1247     )
1248
1249     factory.addStep(
1250         ShellCommand(
1251             name="checksums",
1252             description="Calculating checksums",
1253             descriptionDone="Checksums calculated",
1254             command=["make", "-j1", "checksum", "V=s"],
1255             env=MakeEnv(),
1256             haltOnFailure=True,
1257         )
1258     )
1259
1260     factory.addStep(
1261         ShellCommand(
1262             name="kmoddir",
1263             descriptionDone="Kmod directory created",
1264             command=[
1265                 "mkdir",
1266                 "-p",
1267                 Interpolate(
1268                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s",
1269                     target=target,
1270                     subtarget=subtarget,
1271                 ),
1272             ],
1273             haltOnFailure=True,
1274             doStepIf=IsKmodArchiveEnabled,
1275         )
1276     )
1277
1278     factory.addStep(
1279         ShellCommand(
1280             name="kmodprepare",
1281             description="Preparing kmod archive",
1282             descriptionDone="Kmod archive prepared",
1283             command=[
1284                 "rsync",
1285                 "--include=/kmod-*.ipk",
1286                 "--exclude=*",
1287                 "-va",
1288                 Interpolate(
1289                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/",
1290                     target=target,
1291                     subtarget=subtarget,
1292                 ),
1293                 Interpolate(
1294                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1295                     target=target,
1296                     subtarget=subtarget,
1297                 ),
1298             ],
1299             haltOnFailure=True,
1300             doStepIf=IsKmodArchiveEnabled,
1301         )
1302     )
1303
1304     factory.addStep(
1305         ShellCommand(
1306             name="kmodindex",
1307             description="Indexing kmod archive",
1308             descriptionDone="Kmod archive indexed",
1309             command=[
1310                 "make",
1311                 Interpolate("-j%(prop:nproc:-1)s"),
1312                 "package/index",
1313                 "V=s",
1314                 "CONFIG_SIGNED_PACKAGES=",
1315                 Interpolate(
1316                     "PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1317                     target=target,
1318                     subtarget=subtarget,
1319                 ),
1320             ],
1321             env=MakeEnv(),
1322             haltOnFailure=True,
1323             doStepIf=IsKmodArchiveEnabled,
1324         )
1325     )
1326
1327     # sign
1328     factory.addStep(
1329         MasterShellCommand(
1330             name="signprepare",
1331             descriptionDone="Temporary signing directory prepared",
1332             command=["mkdir", "-p", "%s/signing" % (work_dir)],
1333             haltOnFailure=True,
1334             doStepIf=IsSignEnabled,
1335         )
1336     )
1337
1338     factory.addStep(
1339         ShellCommand(
1340             name="signpack",
1341             description="Packing files to sign",
1342             descriptionDone="Files to sign packed",
1343             command=Interpolate(
1344                 "find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz",
1345                 target=target,
1346                 subtarget=subtarget,
1347             ),
1348             haltOnFailure=True,
1349             doStepIf=IsSignEnabled,
1350         )
1351     )
1352
1353     factory.addStep(
1354         FileUpload(
1355             workersrc="sign.tar.gz",
1356             masterdest="%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1357             haltOnFailure=True,
1358             doStepIf=IsSignEnabled,
1359         )
1360     )
1361
1362     factory.addStep(
1363         MasterShellCommand(
1364             name="signfiles",
1365             description="Signing files",
1366             descriptionDone="Files signed",
1367             command=[
1368                 "%s/signall.sh" % (scripts_dir),
1369                 "%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1370                 Interpolate("%(prop:branch)s"),
1371             ],
1372             env={"CONFIG_INI": os.getenv("BUILDMASTER_CONFIG", "./config.ini")},
1373             haltOnFailure=True,
1374             doStepIf=IsSignEnabled,
1375         )
1376     )
1377
1378     factory.addStep(
1379         FileDownload(
1380             name="dlsigntargz",
1381             mastersrc="%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1382             workerdest="sign.tar.gz",
1383             haltOnFailure=True,
1384             doStepIf=IsSignEnabled,
1385         )
1386     )
1387
1388     factory.addStep(
1389         ShellCommand(
1390             name="signunpack",
1391             description="Unpacking signed files",
1392             descriptionDone="Signed files unpacked",
1393             command=["tar", "-xzf", "sign.tar.gz"],
1394             haltOnFailure=True,
1395             doStepIf=IsSignEnabled,
1396         )
1397     )
1398
1399     # upload
1400     factory.addStep(
1401         ShellCommand(
1402             name="dirprepare",
1403             descriptionDone="Upload directory structure prepared",
1404             command=[
1405                 "mkdir",
1406                 "-p",
1407                 Interpolate(
1408                     "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s",
1409                     target=target,
1410                     subtarget=subtarget,
1411                     prefix=GetVersionPrefix,
1412                 ),
1413             ],
1414             haltOnFailure=True,
1415         )
1416     )
1417
1418     factory.addStep(
1419         ShellCommand(
1420             name="linkprepare",
1421             descriptionDone="Repository symlink prepared",
1422             command=[
1423                 "ln",
1424                 "-s",
1425                 "-f",
1426                 Interpolate(
1427                     "../packages-%(kw:basever)s",
1428                     basever=util.Transform(GetBaseVersion, Property("branch")),
1429                 ),
1430                 Interpolate(
1431                     "tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix
1432                 ),
1433             ],
1434             doStepIf=IsNoMasterBuild,
1435             haltOnFailure=True,
1436         )
1437     )
1438
1439     factory.addStep(
1440         ShellCommand(
1441             name="kmoddirprepare",
1442             descriptionDone="Kmod archive upload directory prepared",
1443             command=[
1444                 "mkdir",
1445                 "-p",
1446                 Interpolate(
1447                     "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s",
1448                     target=target,
1449                     subtarget=subtarget,
1450                     prefix=GetVersionPrefix,
1451                 ),
1452             ],
1453             haltOnFailure=True,
1454             doStepIf=IsKmodArchiveEnabled,
1455         )
1456     )
1457
1458     factory.addStep(
1459         ShellCommand(
1460             name="dirupload",
1461             description="Uploading directory structure",
1462             descriptionDone="Directory structure uploaded",
1463             command=["rsync", Interpolate("-az%(prop:rsync_ipv4:+4)s")]
1464             + rsync_defopts
1465             + [
1466                 "tmp/upload/",
1467                 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url")),
1468             ],
1469             env={
1470                 "RSYNC_PASSWORD": Interpolate(
1471                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1472                 )
1473             },
1474             haltOnFailure=True,
1475             logEnviron=False,
1476             locks=NetLockUl,
1477             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1478         )
1479     )
1480
1481     # download remote sha256sums to 'target-sha256sums'
1482     factory.addStep(
1483         ShellCommand(
1484             name="target-sha256sums",
1485             description="Fetching remote sha256sums for target",
1486             descriptionDone="Remote sha256sums for target fetched",
1487             command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
1488             + rsync_defopts
1489             + [
1490                 Interpolate(
1491                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
1492                     url=GetRsyncParams.withArgs("bin", "url"),
1493                     target=target,
1494                     subtarget=subtarget,
1495                     prefix=GetVersionPrefix,
1496                 ),
1497                 "target-sha256sums",
1498             ],
1499             env={
1500                 "RSYNC_PASSWORD": Interpolate(
1501                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1502                 )
1503             },
1504             logEnviron=False,
1505             haltOnFailure=False,
1506             flunkOnFailure=False,
1507             warnOnFailure=False,
1508             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1509         )
1510     )
1511
1512     # build list of files to upload
1513     factory.addStep(
1514         FileDownload(
1515             name="dlsha2rsyncpl",
1516             mastersrc=scripts_dir + "/sha2rsync.pl",
1517             workerdest="../sha2rsync.pl",
1518             mode=0o755,
1519         )
1520     )
1521
1522     factory.addStep(
1523         ShellCommand(
1524             name="buildlist",
1525             description="Building list of files to upload",
1526             descriptionDone="List of files to upload built",
1527             command=[
1528                 "../sha2rsync.pl",
1529                 "target-sha256sums",
1530                 Interpolate(
1531                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
1532                     target=target,
1533                     subtarget=subtarget,
1534                 ),
1535                 "rsynclist",
1536             ],
1537             haltOnFailure=True,
1538         )
1539     )
1540
1541     factory.addStep(
1542         FileDownload(
1543             name="dlrsync.sh",
1544             mastersrc=scripts_dir + "/rsync.sh",
1545             workerdest="../rsync.sh",
1546             mode=0o755,
1547         )
1548     )
1549
1550     # upload new files and update existing ones
1551     factory.addStep(
1552         ShellCommand(
1553             name="targetupload",
1554             description="Uploading target files",
1555             descriptionDone="Target files uploaded",
1556             command=[
1557                 "../rsync.sh",
1558                 "--exclude=/kmods/",
1559                 "--files-from=rsynclist",
1560                 "--delay-updates",
1561                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1562             ]
1563             + rsync_defopts
1564             + [
1565                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1566                 Interpolate(
1567                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1568                     target=target,
1569                     subtarget=subtarget,
1570                 ),
1571                 Interpolate(
1572                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1573                     url=GetRsyncParams.withArgs("bin", "url"),
1574                     target=target,
1575                     subtarget=subtarget,
1576                     prefix=GetVersionPrefix,
1577                 ),
1578             ],
1579             env={
1580                 "RSYNC_PASSWORD": Interpolate(
1581                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1582                 )
1583             },
1584             haltOnFailure=True,
1585             logEnviron=False,
1586             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1587         )
1588     )
1589
1590     # delete files which don't exist locally
1591     factory.addStep(
1592         ShellCommand(
1593             name="targetprune",
1594             description="Pruning target files",
1595             descriptionDone="Target files pruned",
1596             command=[
1597                 "../rsync.sh",
1598                 "--exclude=/kmods/",
1599                 "--delete",
1600                 "--existing",
1601                 "--ignore-existing",
1602                 "--delay-updates",
1603                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1604             ]
1605             + rsync_defopts
1606             + [
1607                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1608                 Interpolate(
1609                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1610                     target=target,
1611                     subtarget=subtarget,
1612                 ),
1613                 Interpolate(
1614                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1615                     url=GetRsyncParams.withArgs("bin", "url"),
1616                     target=target,
1617                     subtarget=subtarget,
1618                     prefix=GetVersionPrefix,
1619                 ),
1620             ],
1621             env={
1622                 "RSYNC_PASSWORD": Interpolate(
1623                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1624                 )
1625             },
1626             haltOnFailure=True,
1627             logEnviron=False,
1628             locks=NetLockUl,
1629             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1630         )
1631     )
1632
1633     factory.addStep(
1634         ShellCommand(
1635             name="kmodupload",
1636             description="Uploading kmod archive",
1637             descriptionDone="Kmod archive uploaded",
1638             command=[
1639                 "../rsync.sh",
1640                 "--delete",
1641                 "--delay-updates",
1642                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1643             ]
1644             + rsync_defopts
1645             + [
1646                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1647                 Interpolate(
1648                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1649                     target=target,
1650                     subtarget=subtarget,
1651                 ),
1652                 Interpolate(
1653                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/",
1654                     url=GetRsyncParams.withArgs("bin", "url"),
1655                     target=target,
1656                     subtarget=subtarget,
1657                     prefix=GetVersionPrefix,
1658                 ),
1659             ],
1660             env={
1661                 "RSYNC_PASSWORD": Interpolate(
1662                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1663                 )
1664             },
1665             haltOnFailure=True,
1666             logEnviron=False,
1667             locks=NetLockUl,
1668             doStepIf=IsKmodArchiveAndRsyncEnabled,
1669         )
1670     )
1671
1672     factory.addStep(
1673         ShellCommand(
1674             name="sourcelist",
1675             description="Finding source archives to upload",
1676             descriptionDone="Source archives to upload found",
1677             command="find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1678             haltOnFailure=True,
1679         )
1680     )
1681
1682     factory.addStep(
1683         ShellCommand(
1684             name="sourceupload",
1685             description="Uploading source archives",
1686             descriptionDone="Source archives uploaded",
1687             command=[
1688                 "../rsync.sh",
1689                 "--files-from=sourcelist",
1690                 "--size-only",
1691                 "--delay-updates",
1692             ]
1693             + rsync_defopts
1694             + [
1695                 Interpolate(
1696                     "--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s",
1697                     target=target,
1698                     subtarget=subtarget,
1699                 ),
1700                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1701                 "dl/",
1702                 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url")),
1703             ],
1704             env={
1705                 "RSYNC_PASSWORD": Interpolate(
1706                     "%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")
1707                 )
1708             },
1709             haltOnFailure=True,
1710             logEnviron=False,
1711             locks=NetLockUl,
1712             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1713         )
1714     )
1715
1716     factory.addStep(
1717         ShellCommand(
1718             name="df",
1719             description="Reporting disk usage",
1720             command=["df", "-h", "."],
1721             env={"LC_ALL": "C"},
1722             logEnviron=False,
1723             haltOnFailure=False,
1724             flunkOnFailure=False,
1725             warnOnFailure=False,
1726             alwaysRun=True,
1727         )
1728     )
1729
1730     factory.addStep(
1731         ShellCommand(
1732             name="du",
1733             description="Reporting estimated file space usage",
1734             command=["du", "-sh", "."],
1735             env={"LC_ALL": "C"},
1736             logEnviron=False,
1737             haltOnFailure=False,
1738             flunkOnFailure=False,
1739             warnOnFailure=False,
1740             alwaysRun=True,
1741         )
1742     )
1743
1744     factory.addStep(
1745         ShellCommand(
1746             name="ccachestat",
1747             description="Reporting ccache stats",
1748             command=["ccache", "-s"],
1749             logEnviron=False,
1750             want_stderr=False,
1751             haltOnFailure=False,
1752             flunkOnFailure=False,
1753             warnOnFailure=False,
1754             doStepIf=util.Transform(bool, Property("ccache_command")),
1755         )
1756     )
1757
1758     return factory
1759
1760
1761 for brname in branchNames:
1762     for target in targets[brname]:
1763         bldrname = brname + "_" + target
1764         c["builders"].append(
1765             BuilderConfig(
1766                 name=bldrname,
1767                 workernames=workerNames,
1768                 factory=prepareFactory(target),
1769                 tags=[
1770                     brname,
1771                 ],
1772                 nextBuild=GetNextBuild,
1773                 canStartBuild=canStartBuild,
1774             )
1775         )
1776
1777
1778 ####### STATUS TARGETS
1779
1780 # 'status' is a list of Status Targets. The results of each build will be
1781 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1782 # including web pages, email senders, and IRC bots.
1783
1784 if "status_bind" in inip1:
1785     c["www"] = {
1786         "port": inip1.get("status_bind"),
1787         "plugins": {"waterfall_view": True, "console_view": True, "grid_view": True},
1788     }
1789
1790     if "status_user" in inip1 and "status_password" in inip1:
1791         c["www"]["auth"] = util.UserPasswordAuth(
1792             [(inip1.get("status_user"), inip1.get("status_password"))]
1793         )
1794         c["www"]["authz"] = util.Authz(
1795             allowRules=[util.AnyControlEndpointMatcher(role="admins")],
1796             roleMatchers=[
1797                 util.RolesFromUsername(
1798                     roles=["admins"], usernames=[inip1.get("status_user")]
1799                 )
1800             ],
1801         )
1802
1803 c["services"] = []
1804 if ini.has_section("irc"):
1805     iniirc = ini["irc"]
1806     irc_host = iniirc.get("host", None)
1807     irc_port = iniirc.getint("port", 6667)
1808     irc_chan = iniirc.get("channel", None)
1809     irc_nick = iniirc.get("nickname", None)
1810     irc_pass = iniirc.get("password", None)
1811
1812     if irc_host and irc_nick and irc_chan:
1813         irc = reporters.IRC(
1814             irc_host,
1815             irc_nick,
1816             port=irc_port,
1817             password=irc_pass,
1818             channels=[irc_chan],
1819             notify_events=["exception", "problem", "recovery"],
1820         )
1821
1822         c["services"].append(irc)
1823
1824 c["revlink"] = util.RevlinkMatch(
1825     [r"https://git.openwrt.org/openwrt/(.*).git"],
1826     r"https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s",
1827 )
1828
1829 ####### DB URL
1830
1831 c["db"] = {
1832     # This specifies what database buildbot uses to store its state.  You can leave
1833     # this at its default for all but the largest installations.
1834     "db_url": "sqlite:///state.sqlite",
1835 }
1836
1837 c["buildbotNetUsageData"] = None