phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41 from buildbot.worker.local import LocalWorker
  42
  43
  44 if not os.path.exists("twistd.pid"):
  45     with open("twistd.pid", "w") as pidfile:
  46         pidfile.write("{}".format(os.getpid()))
  47
  48 # This is a sample buildmaster config file. It must be installed as
  49 # 'master.cfg' in your buildmaster's base directory.
  50
  51 ini = configparser.ConfigParser()
  52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  53
  54 if "general" not in ini or "phase1" not in ini:
  55     raise ValueError("Fix your configuration")
  56
  57 inip1 = ini["phase1"]
  58
  59 # Globals
  60 work_dir = os.path.abspath(ini["general"].get("workdir", "."))
  61 scripts_dir = os.path.abspath("../scripts")
  62
  63 repo_url = ini["repo"].get("url")
  64
  65 rsync_defopts = ["-v", "--timeout=120"]
  66
  67 # if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  68 #       rsync_bin_defopts += ["--contimeout=20"]
  69
  70 branches = {}
  71
  72
  73 def ini_parse_branch(section):
  74     b = {}
  75     name = section.get("name")
  76
  77     if not name:
  78         raise ValueError("missing 'name' in " + repr(section))
  79     if name in branches:
  80         raise ValueError("duplicate branch name in " + repr(section))
  81
  82     b["name"] = name
  83     b["bin_url"] = section.get("binary_url")
  84     b["bin_key"] = section.get("binary_password")
  85
  86     b["src_url"] = section.get("source_url")
  87     b["src_key"] = section.get("source_password")
  88
  89     b["gpg_key"] = section.get("gpg_key")
  90
  91     b["usign_key"] = section.get("usign_key")
  92     usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  93     b["usign_comment"] = section.get("usign_comment", usign_comment)
  94
  95     b["config_seed"] = section.get("config_seed")
  96
  97     b["kmod_archive"] = section.getboolean("kmod_archive", False)
  98
  99     branches[name] = b
 100     log.msg("Configured branch: {}".format(name))
 101
 102
 103 # PB port can be either a numeric port or a connection string
 104 pb_port = inip1.get("port") or 9989
 105
 106 # This is the dictionary that the buildmaster pays attention to. We also use
 107 # a shorter alias to save typing.
 108 c = BuildmasterConfig = {}
 109
 110 # PROJECT IDENTITY
 111
 112 # the 'title' string will appear at the top of this buildbot
 113 # installation's html.WebStatus home page (linked to the
 114 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 115
 116 c["title"] = ini["general"].get("title")
 117 c["titleURL"] = ini["general"].get("title_url")
 118
 119 # the 'buildbotURL' string should point to the location where the buildbot's
 120 # internal web server (usually the html.WebStatus page) is visible. This
 121 # typically uses the port number set in the Waterfall 'status' entry, but
 122 # with an externally-visible host name which the buildbot cannot figure out
 123 # without some help.
 124
 125 c["buildbotURL"] = inip1.get("buildbot_url")
 126
 127 # BUILDWORKERS
 128
 129 # The 'workers' list defines the set of recognized buildworkers. Each element is
 130 # a Worker object, specifying a unique worker name and password.  The same
 131 # worker name and password must be configured on the worker.
 132
 133 c["workers"] = []
 134 NetLocks = dict()
 135
 136
 137 def ini_parse_workers(section):
 138     name = section.get("name")
 139     password = section.get("password")
 140     phase = section.getint("phase")
 141     tagonly = section.getboolean("tag_only")
 142     rsyncipv4 = section.getboolean("rsync_ipv4")
 143
 144     if not name or not password or not phase == 1:
 145         log.msg("invalid worker configuration ignored: {}".format(repr(section)))
 146         return
 147
 148     sl_props = {"tag_only": tagonly}
 149     if "dl_lock" in section:
 150         lockname = section.get("dl_lock")
 151         sl_props["dl_lock"] = lockname
 152         if lockname not in NetLocks:
 153             NetLocks[lockname] = locks.MasterLock(lockname)
 154     if "ul_lock" in section:
 155         lockname = section.get("ul_lock")
 156         sl_props["ul_lock"] = lockname
 157         if lockname not in NetLocks:
 158             NetLocks[lockname] = locks.MasterLock(lockname)
 159     if rsyncipv4:
 160         sl_props[
 161             "rsync_ipv4"
 162         ] = True  # only set prop if required, we use '+' Interpolate substitution
 163
 164     log.msg("Configured worker: {}".format(name))
 165     # NB: phase1 build factory requires workers to be single-build only
 166     c["workers"].append(Worker(name, password, max_builds=1, properties=sl_props))
 167
 168
 169 for section in ini.sections():
 170     if section.startswith("branch "):
 171         ini_parse_branch(ini[section])
 172
 173     if section.startswith("worker "):
 174         ini_parse_workers(ini[section])
 175
 176 # list of branches in build-priority order
 177 branchNames = [branches[b]["name"] for b in branches]
 178
 179 c["protocols"] = {"pb": {"port": pb_port}}
 180
 181 # coalesce builds
 182 c["collapseRequests"] = True
 183
 184 # Reduce amount of backlog data
 185 c["configurators"] = [
 186     util.JanitorConfigurator(
 187         logHorizon=timedelta(days=3),
 188         hour=6,
 189     )
 190 ]
 191
 192
 193 @defer.inlineCallbacks
 194 def getNewestCompleteTimePrio(bldr):
 195     """Returns the priority and the complete_at of the latest completed and not SKIPPED
 196     build request for this builder, or None if there are no such build
 197     requests. We need to filter out SKIPPED requests because we're
 198     using collapseRequests=True which is unfortunately marking all
 199     previous requests as complete when new buildset is created.
 200
 201     @returns: (priority, datetime instance or None), via Deferred
 202     """
 203
 204     prio = yield bldr.get_highest_priority()
 205     if prio is None:
 206         prio = 0
 207
 208     bldrid = yield bldr.getBuilderId()
 209     completed = yield bldr.master.data.get(
 210         ("builders", bldrid, "buildrequests"),
 211         [
 212             resultspec.Filter("complete", "eq", [True]),
 213             resultspec.Filter("results", "ne", [results.SKIPPED]),
 214         ],
 215         order=["-complete_at"],
 216         limit=1,
 217     )
 218     if not completed:
 219         return (prio, None)
 220
 221     complete_at = completed[0]["complete_at"]
 222
 223     last_build = yield bldr.master.data.get(
 224         ("builds",),
 225         [
 226             resultspec.Filter("builderid", "eq", [bldrid]),
 227         ],
 228         order=["-started_at"],
 229         limit=1,
 230     )
 231
 232     if last_build and last_build[0]:
 233         last_complete_at = last_build[0]["complete_at"]
 234         if last_complete_at and (last_complete_at > complete_at):
 235             return (prio, last_complete_at)
 236
 237     return (prio, complete_at)
 238
 239
 240 @defer.inlineCallbacks
 241 def prioritizeBuilders(master, builders):
 242     """Returns sorted list of builders by their last timestamp of completed and
 243     not skipped build, ordered first by branch name.
 244
 245     @returns: list of sorted builders
 246     """
 247
 248     bldrNamePrio = {"__Janitor": 0, "00_force_build": 0}
 249     i = 1
 250     for bname in branchNames:
 251         bldrNamePrio[bname] = i
 252
 253     def is_building(bldr):
 254         return bool(bldr.building) or bool(bldr.old_building)
 255
 256     def bldr_info(bldr):
 257         d = defer.maybeDeferred(getNewestCompleteTimePrio, bldr)
 258         d.addCallback(lambda retval: (retval, bldr))
 259         return d
 260
 261     def bldr_sort(item):
 262         ((hiprio, complete_at), bldr) = item
 263
 264         # check if we have some high prio build requests pending (i.e. tag builds),
 265         # if so, front-run these builders, while preserving the per-branch static priority
 266         pos = 99
 267         for name, prio in bldrNamePrio.items():
 268             if bldr.name.startswith(name):
 269                 # higher priority (larger positive number) raises position
 270                 pos = prio + 50 - min(hiprio, 50)
 271                 break
 272
 273         # pos order: janitor/local (0), tag builds if any [1..50], !tag builds [51...]
 274
 275         if not complete_at:
 276             date = datetime.min
 277             complete_at = date.replace(tzinfo=tzutc())
 278
 279         if is_building(bldr):
 280             date = datetime.max
 281             complete_at = date.replace(tzinfo=tzutc())
 282
 283         return (pos, complete_at, bldr.name)
 284
 285     results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 286     results.sort(key=bldr_sort)
 287
 288     # for r in results:
 289     #   log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 290
 291     return [r[1] for r in results]
 292
 293
 294 c["prioritizeBuilders"] = prioritizeBuilders
 295
 296 # CHANGESOURCES
 297
 298 # find targets
 299 targets = dict()
 300
 301
 302 def populateTargets():
 303     """fetch a shallow clone of each configured branch in turn:
 304     execute dump-target-info.pl and collate the results to ensure
 305     targets that only exist in specific branches get built.
 306     This takes a while during master startup but is executed only once.
 307     """
 308     sourcegit = work_dir + "/source.git"
 309     for branch in branchNames:
 310         log.msg(f"Populating targets for {branch}, this will take time")
 311
 312         if os.path.isdir(sourcegit):
 313             subprocess.call(["rm", "-rf", sourcegit])
 314
 315         subprocess.call(
 316             [
 317                 "git",
 318                 "clone",
 319                 "-q",
 320                 "--depth=1",
 321                 "--branch=" + branch,
 322                 repo_url,
 323                 sourcegit,
 324             ]
 325         )
 326
 327         os.makedirs(sourcegit + "/tmp", exist_ok=True)
 328         findtargets = subprocess.Popen(
 329             ["./scripts/dump-target-info.pl", "targets"],
 330             stdout=subprocess.PIPE,
 331             stderr=subprocess.DEVNULL,
 332             cwd=sourcegit,
 333         )
 334
 335         targets[branch] = set()
 336         while True:
 337             line = findtargets.stdout.readline()
 338             if not line:
 339                 break
 340             ta = line.decode().strip().split(" ")
 341             targets[branch].add(ta[0])
 342
 343         subprocess.call(["rm", "-rf", sourcegit])
 344
 345
 346 populateTargets()
 347
 348 # the 'change_source' setting tells the buildmaster how it should find out
 349 # about source code changes.
 350
 351 c["change_source"] = []
 352 c["change_source"].append(
 353     GitPoller(
 354         repo_url,
 355         workdir=work_dir + "/work.git",
 356         branches=branchNames,
 357         pollAtLaunch=True,
 358         pollinterval=300,
 359     )
 360 )
 361
 362 # SCHEDULERS
 363
 364 # Configure the Schedulers, which decide how to react to incoming changes.
 365
 366
 367 # Selector for known valid tags
 368 class TagChoiceParameter(BaseParameter):
 369     spec_attributes = ["strict", "choices"]
 370     type = "list"
 371     strict = True
 372
 373     def __init__(self, name, label=None, **kw):
 374         super().__init__(name, label, **kw)
 375         self._choice_list = []
 376
 377     def getRevTags(self, findtag=None):
 378         taglist = []
 379         branchvers = []
 380
 381         # we will filter out tags that do no match the configured branches
 382         for b in branchNames:
 383             basever = re.search(r"-([0-9]+\.[0-9]+)$", b)
 384             if basever:
 385                 branchvers.append(basever[1])
 386
 387         # grab tags from remote repository
 388         alltags = subprocess.Popen(
 389             ["git", "ls-remote", "--tags", repo_url], stdout=subprocess.PIPE
 390         )
 391
 392         while True:
 393             line = alltags.stdout.readline()
 394
 395             if not line:
 396                 break
 397
 398             (rev, tag) = line.split()
 399
 400             # does it match known format? ('vNN.NN.NN(-rcN)')
 401             tagver = re.search(
 402                 r"\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$",
 403                 tag.decode().strip(),
 404             )
 405
 406             # only list valid tags matching configured branches
 407             if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
 408                 # if we want a specific tag, ignore all that don't match
 409                 if findtag and findtag != tagver[1]:
 410                     continue
 411                 taglist.append({"rev": rev.decode().strip(), "tag": tagver[1]})
 412
 413         return taglist
 414
 415     @property
 416     def choices(self):
 417         taglist = [rt["tag"] for rt in self.getRevTags()]
 418         taglist.sort(
 419             reverse=True,
 420             key=lambda tag: tag if re.search(r"-rc[0-9]+$", tag) else tag + "-z",
 421         )
 422         taglist.insert(0, "")
 423
 424         self._choice_list = taglist
 425
 426         return self._choice_list
 427
 428     def updateFromKwargs(self, properties, kwargs, **unused):
 429         tag = self.getFromKwargs(kwargs)
 430         properties[self.name] = tag
 431
 432         # find the commit matching the tag
 433         findtag = self.getRevTags(tag)
 434
 435         if not findtag:
 436             raise ValidationError("Couldn't find tag")
 437
 438         properties["force_revision"] = findtag[0]["rev"]
 439
 440         # find the branch matching the tag
 441         branch = None
 442         branchver = re.search(r"v([0-9]+\.[0-9]+)", tag)
 443         for b in branchNames:
 444             if b.endswith(branchver[1]):
 445                 branch = b
 446
 447         if not branch:
 448             raise ValidationError("Couldn't find branch")
 449
 450         properties["force_branch"] = branch
 451
 452     def parse_from_arg(self, s):
 453         if self.strict and s not in self._choice_list:
 454             raise ValidationError(
 455                 "'%s' does not belong to list of available choices '%s'"
 456                 % (s, self._choice_list)
 457             )
 458         return s
 459
 460
 461 @util.renderer
 462 @defer.inlineCallbacks
 463 def builderNames(props):
 464     """since we have per branch and per target builders,
 465     address the relevant builder for each new buildrequest
 466     based on the request's desired branch and target.
 467     """
 468     branch = props.getProperty("branch")
 469     target = props.getProperty("target", "")
 470
 471     if target == "all":
 472         target = ""
 473
 474     # if that didn't work, try sourcestamp to find a branch
 475     if not branch:
 476         # match builders with target branch
 477         ss = props.sourcestamps[0]
 478         if ss:
 479             branch = ss["branch"]
 480         else:
 481             log.msg("couldn't find builder")
 482             return []  # nothing works
 483
 484     bname = branch + "_" + target
 485     builders = []
 486
 487     for b in (yield props.master.data.get(("builders",))):
 488         if not b["name"].startswith(bname):
 489             continue
 490         builders.append(b["name"])
 491
 492     return builders
 493
 494
 495 c["schedulers"] = []
 496 c["schedulers"].append(
 497     AnyBranchScheduler(
 498         name="all",
 499         change_filter=util.ChangeFilter(branch=branchNames),
 500         treeStableTimer=15 * 60,
 501         builderNames=builderNames,
 502     )
 503 )
 504
 505 c["schedulers"].append(
 506     ForceScheduler(
 507         name="force",
 508         buttonName="Force builds",
 509         label="Force build details",
 510         builderNames=["00_force_build"],
 511         codebases=[
 512             util.CodebaseParameter(
 513                 "",
 514                 label="Repository",
 515                 branch=util.FixedParameter(name="branch", default=""),
 516                 revision=util.FixedParameter(name="revision", default=""),
 517                 repository=util.FixedParameter(name="repository", default=""),
 518                 project=util.FixedParameter(name="project", default=""),
 519             )
 520         ],
 521         reason=util.StringParameter(
 522             name="reason",
 523             label="Reason",
 524             default="Trigger build",
 525             required=True,
 526             size=80,
 527         ),
 528         properties=[
 529             # NB: avoid nesting to simplify processing of properties
 530             util.ChoiceStringParameter(
 531                 name="target",
 532                 label="Build target",
 533                 default="all",
 534                 choices=["all"] + [t for b in branchNames for t in targets[b]],
 535             ),
 536             TagChoiceParameter(name="tag", label="Build tag", default=""),
 537         ],
 538     )
 539 )
 540
 541 c["schedulers"].append(
 542     schedulers.Triggerable(name="trigger", builderNames=builderNames, priority=20)
 543 )
 544
 545 # BUILDERS
 546
 547 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 548 # what steps, and which workers can execute them.  Note that any particular build will
 549 # only take place on one worker.
 550
 551
 552 def IsNoMasterBuild(step):
 553     return step.getProperty("branch") != "main"
 554
 555
 556 def IsUsignEnabled(step):
 557     branch = step.getProperty("branch")
 558     return branch and branches[branch].get("usign_key")
 559
 560
 561 def IsSignEnabled(step):
 562     branch = step.getProperty("branch")
 563     return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 564
 565
 566 def IsKmodArchiveEnabled(step):
 567     branch = step.getProperty("branch")
 568     return branch and branches[branch].get("kmod_archive")
 569
 570
 571 def IsKmodArchiveAndRsyncEnabled(step):
 572     branch = step.getProperty("branch")
 573     return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url"))
 574
 575
 576 def GetBaseVersion(branch):
 577     if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 578         return branch.split("-")[1]
 579     else:
 580         return "main"
 581
 582
 583 @properties.renderer
 584 def GetVersionPrefix(props):
 585     branch = props.getProperty("branch")
 586     basever = GetBaseVersion(branch)
 587     if props.hasProperty("tag") and re.match(
 588         r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]
 589     ):
 590         return "%s/" % props["tag"][1:]
 591     elif basever != "main":
 592         return "%s-SNAPSHOT/" % basever
 593     else:
 594         return ""
 595
 596
 597 @util.renderer
 598 def GetConfigSeed(props):
 599     branch = props.getProperty("branch")
 600     return branch and branches[branch].get("config_seed") or ""
 601
 602
 603 @util.renderer
 604 def GetRsyncParams(props, srcorbin, urlorkey):
 605     # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 606     branch = props.getProperty("branch")
 607     opt = srcorbin + "_" + urlorkey
 608     return branch and branches[branch].get(opt)
 609
 610
 611 @util.renderer
 612 def GetUsignKey(props):
 613     branch = props.getProperty("branch")
 614     return branch and branches[branch].get("usign_key")
 615
 616
 617 def GetNextBuild(builder, requests):
 618     for r in requests:
 619         if r.properties:
 620             # order tagged build first
 621             if r.properties.hasProperty("tag"):
 622                 return r
 623
 624     r = requests[0]
 625     # log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 626     return r
 627
 628
 629 def MakeEnv(overrides=None, tryccache=False):
 630     env = {
 631         "CCC": Interpolate("%(prop:cc_command:-gcc)s"),
 632         "CCXX": Interpolate("%(prop:cxx_command:-g++)s"),
 633     }
 634     if tryccache:
 635         env["CC"] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 636         env["CXX"] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 637         env["CCACHE"] = Interpolate("%(prop:ccache_command:-)s")
 638     else:
 639         env["CC"] = env["CCC"]
 640         env["CXX"] = env["CCXX"]
 641         env["CCACHE"] = ""
 642     if overrides is not None:
 643         env.update(overrides)
 644     return env
 645
 646
 647 @properties.renderer
 648 def NetLockDl(props, extralock=None):
 649     lock = None
 650     if props.hasProperty("dl_lock"):
 651         lock = NetLocks[props["dl_lock"]]
 652     if lock is not None:
 653         return [lock.access("exclusive")]
 654     else:
 655         return []
 656
 657
 658 @properties.renderer
 659 def NetLockUl(props):
 660     lock = None
 661     if props.hasProperty("ul_lock"):
 662         lock = NetLocks[props["ul_lock"]]
 663     if lock is not None:
 664         return [lock.access("exclusive")]
 665     else:
 666         return []
 667
 668
 669 def IsTargetSelected(target):
 670     def CheckTargetProperty(step):
 671         selected_target = step.getProperty("target", "all")
 672         if selected_target != "all" and selected_target != target:
 673             return False
 674         return True
 675
 676     return CheckTargetProperty
 677
 678
 679 @util.renderer
 680 def UsignSec2Pub(props):
 681     branch = props.getProperty("branch")
 682     try:
 683         comment = (
 684             branches[branch].get("usign_comment") or "untrusted comment: secret key"
 685         )
 686         seckey = branches[branch].get("usign_key")
 687         seckey = base64.b64decode(seckey)
 688     except Exception:
 689         return None
 690
 691     return "{}\n{}".format(
 692         re.sub(r"\bsecret key$", "public key", comment),
 693         base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]),
 694     )
 695
 696
 697 def canStartBuild(builder, wfb, request):
 698     """filter out non tag requests for tag_only workers."""
 699     wtagonly = wfb.worker.properties.getProperty("tag_only")
 700     tag = request.properties.getProperty("tag")
 701
 702     if wtagonly and not tag:
 703         return False
 704
 705     return True
 706
 707
 708 c["builders"] = []
 709
 710 workerNames = []
 711
 712 for worker in c["workers"]:
 713     workerNames.append(worker.workername)
 714
 715 # add a single LocalWorker to handle the forcebuild builder
 716 c["workers"].append(LocalWorker("__local_force_build", max_builds=1))
 717
 718 force_factory = BuildFactory()
 719 force_factory.addStep(
 720     steps.Trigger(
 721         name="trigger_build",
 722         schedulerNames=["trigger"],
 723         sourceStamps=[
 724             {
 725                 "codebase": "",
 726                 "branch": Property("force_branch"),
 727                 "revision": Property("force_revision"),
 728                 "repository": repo_url,
 729                 "project": "",
 730             }
 731         ],
 732         set_properties={
 733             "reason": Property("reason"),
 734             "tag": Property("tag"),
 735             "target": Property("target"),
 736         },
 737     )
 738 )
 739
 740 c["builders"].append(
 741     BuilderConfig(
 742         name="00_force_build", workername="__local_force_build", factory=force_factory
 743     )
 744 )
 745
 746
 747 # NB the phase1 build factory assumes workers are single-build only
 748 def prepareFactory(target):
 749     (target, subtarget) = target.split("/")
 750
 751     factory = BuildFactory()
 752
 753     # setup shared work directory if required
 754     factory.addStep(
 755         ShellCommand(
 756             name="sharedwd",
 757             descriptionDone="Shared work directory set up",
 758             command='test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 759             workdir=".",
 760             haltOnFailure=True,
 761         )
 762     )
 763
 764     # find number of cores
 765     factory.addStep(
 766         SetPropertyFromCommand(
 767             name="nproc",
 768             property="nproc",
 769             description="Finding number of CPUs",
 770             command=["nproc"],
 771         )
 772     )
 773
 774     # find gcc and g++ compilers
 775     factory.addStep(
 776         FileDownload(
 777             name="dlfindbinpl",
 778             mastersrc=scripts_dir + "/findbin.pl",
 779             workerdest="../findbin.pl",
 780             mode=0o755,
 781         )
 782     )
 783
 784     factory.addStep(
 785         SetPropertyFromCommand(
 786             name="gcc",
 787             property="cc_command",
 788             description="Finding gcc command",
 789             command=["../findbin.pl", "gcc", "", ""],
 790             haltOnFailure=True,
 791         )
 792     )
 793
 794     factory.addStep(
 795         SetPropertyFromCommand(
 796             name="g++",
 797             property="cxx_command",
 798             description="Finding g++ command",
 799             command=["../findbin.pl", "g++", "", ""],
 800             haltOnFailure=True,
 801         )
 802     )
 803
 804     # see if ccache is available
 805     factory.addStep(
 806         SetPropertyFromCommand(
 807             name="ccache",
 808             property="ccache_command",
 809             description="Testing for ccache command",
 810             command=["which", "ccache"],
 811             haltOnFailure=False,
 812             flunkOnFailure=False,
 813             warnOnFailure=False,
 814             hideStepIf=lambda r, s: r == results.FAILURE,
 815         )
 816     )
 817
 818     # check out the source
 819     # Git() runs:
 820     # if repo doesn't exist: 'git clone repourl'
 821     # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 822     # git cat-file -e <commit>
 823     # git checkout -f <commit>
 824     # git checkout -B <branch>
 825     # git rev-parse HEAD
 826     factory.addStep(
 827         Git(
 828             name="git",
 829             repourl=repo_url,
 830             mode="full",
 831             method="fresh",
 832             locks=NetLockDl,
 833             haltOnFailure=True,
 834         )
 835     )
 836
 837     # workaround for https://github.com/openwrt/buildbot/issues/5
 838     factory.addStep(
 839         Git(
 840             name="git me once more please",
 841             repourl=repo_url,
 842             mode="full",
 843             method="fresh",
 844             locks=NetLockDl,
 845             haltOnFailure=True,
 846         )
 847     )
 848
 849     # update remote refs
 850     factory.addStep(
 851         ShellCommand(
 852             name="fetchrefs",
 853             description="Fetching Git remote refs",
 854             descriptionDone="Git remote refs fetched",
 855             command=[
 856                 "git",
 857                 "fetch",
 858                 "origin",
 859                 Interpolate(
 860                     "+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s"
 861                 ),
 862             ],
 863             haltOnFailure=True,
 864         )
 865     )
 866
 867     # getver.sh requires local branches to track upstream otherwise version computation fails.
 868     # Git() does not set tracking branches when cloning or switching, so work around this here
 869     factory.addStep(
 870         ShellCommand(
 871             name="trackupstream",
 872             description="Setting upstream branch",
 873             descriptionDone="getver.sh is happy now",
 874             command=["git", "branch", "-u", Interpolate("origin/%(prop:branch)s")],
 875             haltOnFailure=True,
 876         )
 877     )
 878
 879     # Verify that Git HEAD points to a tag or branch
 880     # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 881     factory.addStep(
 882         ShellCommand(
 883             name="gitverify",
 884             description="Ensuring that Git HEAD is pointing to a branch or tag",
 885             descriptionDone="Git HEAD is sane",
 886             command=(
 887                 "git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || "
 888                 "git show-ref --tags --dereference 2>/dev/null | sed -ne "
 889                 '"/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."'
 890             ),
 891             haltOnFailure=True,
 892         )
 893     )
 894
 895     factory.addStep(
 896         StringDownload(
 897             name="ccachecc",
 898             s='#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 899             workerdest="../ccache_cc.sh",
 900             mode=0o755,
 901         )
 902     )
 903
 904     factory.addStep(
 905         StringDownload(
 906             name="ccachecxx",
 907             s='#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 908             workerdest="../ccache_cxx.sh",
 909             mode=0o755,
 910         )
 911     )
 912
 913     # feed
 914     factory.addStep(
 915         ShellCommand(
 916             name="updatefeeds",
 917             description="Updating feeds",
 918             command=["./scripts/feeds", "update"],
 919             env=MakeEnv(tryccache=True),
 920             haltOnFailure=True,
 921             locks=NetLockDl,
 922         )
 923     )
 924
 925     # feed
 926     factory.addStep(
 927         ShellCommand(
 928             name="installfeeds",
 929             description="Installing feeds",
 930             command=["./scripts/feeds", "install", "-a"],
 931             env=MakeEnv(tryccache=True),
 932             haltOnFailure=True,
 933         )
 934     )
 935
 936     # seed config
 937     factory.addStep(
 938         StringDownload(
 939             name="dlconfigseed",
 940             s=Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 941             workerdest=".config",
 942             mode=0o644,
 943         )
 944     )
 945
 946     # configure
 947     factory.addStep(
 948         ShellCommand(
 949             name="newconfig",
 950             descriptionDone=".config seeded",
 951             command=Interpolate(
 952                 "printf 'CONFIG_TARGET_%(kw:target)s=y\\n"
 953                 "CONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\n"
 954                 "CONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config",
 955                 target=target,
 956                 subtarget=subtarget,
 957                 usign=GetUsignKey,
 958             ),
 959         )
 960     )
 961
 962     factory.addStep(
 963         ShellCommand(
 964             name="defconfig",
 965             description="Populating .config",
 966             command=["make", "defconfig"],
 967             env=MakeEnv(),
 968         )
 969     )
 970
 971     # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 972     factory.addStep(
 973         ShellCommand(
 974             name="checkarch",
 975             description="Checking architecture",
 976             descriptionDone="Architecture validated",
 977             command='grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config'
 978             % (target, subtarget),
 979             logEnviron=False,
 980             want_stdout=False,
 981             want_stderr=False,
 982             haltOnFailure=True,
 983             flunkOnFailure=False,  # this is not a build FAILURE - TODO mark build as SKIPPED
 984         )
 985     )
 986
 987     # find libc suffix
 988     factory.addStep(
 989         SetPropertyFromCommand(
 990             name="libc",
 991             property="libc",
 992             description="Finding libc suffix",
 993             command=[
 994                 "sed",
 995                 "-ne",
 996                 '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }',
 997                 ".config",
 998             ],
 999         )
1000     )
1001
1002     # install build key
1003     factory.addStep(
1004         StringDownload(
1005             name="dlkeybuildpub",
1006             s=Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
1007             workerdest="key-build.pub",
1008             mode=0o600,
1009             doStepIf=IsUsignEnabled,
1010         )
1011     )
1012
1013     factory.addStep(
1014         StringDownload(
1015             name="dlkeybuild",
1016             s="# fake private key",
1017             workerdest="key-build",
1018             mode=0o600,
1019             doStepIf=IsUsignEnabled,
1020         )
1021     )
1022
1023     factory.addStep(
1024         StringDownload(
1025             name="dlkeybuilducert",
1026             s="# fake certificate",
1027             workerdest="key-build.ucert",
1028             mode=0o600,
1029             doStepIf=IsUsignEnabled,
1030         )
1031     )
1032
1033     # prepare dl
1034     factory.addStep(
1035         ShellCommand(
1036             name="dldir",
1037             description="Preparing dl/",
1038             descriptionDone="dl/ prepared",
1039             command='mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
1040             workdir=Property("builddir"),
1041             logEnviron=False,
1042             want_stdout=False,
1043         )
1044     )
1045
1046     # cleanup dl
1047     factory.addStep(
1048         ShellCommand(
1049             name="dlprune",
1050             description="Pruning dl/",
1051             descriptionDone="dl/ pruned",
1052             command="find dl/ -mindepth 1 -atime +15 -delete -print",
1053             logEnviron=False,
1054             haltOnFailure=False,
1055             flunkOnFailure=False,
1056             warnOnFailure=False,
1057         )
1058     )
1059
1060     # prepare tar
1061     factory.addStep(
1062         ShellCommand(
1063             name="dltar",
1064             description="Building and installing GNU tar",
1065             descriptionDone="GNU tar built and installed",
1066             command=[
1067                 "make",
1068                 Interpolate("-j%(prop:nproc:-1)s"),
1069                 "tools/tar/compile",
1070                 "V=s",
1071             ],
1072             env=MakeEnv(tryccache=True),
1073             haltOnFailure=True,
1074         )
1075     )
1076
1077     # populate dl
1078     factory.addStep(
1079         ShellCommand(
1080             name="dlrun",
1081             description="Populating dl/",
1082             descriptionDone="dl/ populated",
1083             command=["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
1084             env=MakeEnv(),
1085             logEnviron=False,
1086             locks=NetLockDl,
1087         )
1088     )
1089
1090     factory.addStep(
1091         ShellCommand(
1092             name="cleanbase",
1093             description="Cleaning base-files",
1094             command=["make", "package/base-files/clean", "V=s"],
1095         )
1096     )
1097
1098     # build
1099     factory.addStep(
1100         ShellCommand(
1101             name="tools",
1102             description="Building and installing tools",
1103             descriptionDone="Tools built and installed",
1104             command=[
1105                 "make",
1106                 Interpolate("-j%(prop:nproc:-1)s"),
1107                 "tools/install",
1108                 "V=s",
1109             ],
1110             env=MakeEnv(tryccache=True),
1111             haltOnFailure=True,
1112         )
1113     )
1114
1115     factory.addStep(
1116         ShellCommand(
1117             name="toolchain",
1118             description="Building and installing toolchain",
1119             descriptionDone="Toolchain built and installed",
1120             command=[
1121                 "make",
1122                 Interpolate("-j%(prop:nproc:-1)s"),
1123                 "toolchain/install",
1124                 "V=s",
1125             ],
1126             env=MakeEnv(),
1127             haltOnFailure=True,
1128         )
1129     )
1130
1131     factory.addStep(
1132         ShellCommand(
1133             name="kmods",
1134             description="Building kmods",
1135             descriptionDone="Kmods built",
1136             command=[
1137                 "make",
1138                 Interpolate("-j%(prop:nproc:-1)s"),
1139                 "target/compile",
1140                 "V=s",
1141                 "IGNORE_ERRORS=n m",
1142                 "BUILD_LOG=1",
1143             ],
1144             env=MakeEnv(),
1145             haltOnFailure=True,
1146         )
1147     )
1148
1149     # find kernel version
1150     factory.addStep(
1151         SetPropertyFromCommand(
1152             name="kernelversion",
1153             property="kernelversion",
1154             description="Finding the effective Kernel version",
1155             command=(
1156                 "make --no-print-directory -C target/linux/ "
1157                 "val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | "
1158                 "xargs printf '%s-%s-%s\\n'"
1159             ),
1160             env={"TOPDIR": Interpolate("%(prop:builddir)s/build")},
1161         )
1162     )
1163
1164     factory.addStep(
1165         ShellCommand(
1166             name="pkgclean",
1167             description="Cleaning up package build",
1168             descriptionDone="Package build cleaned up",
1169             command=["make", "package/cleanup", "V=s"],
1170         )
1171     )
1172
1173     factory.addStep(
1174         ShellCommand(
1175             name="pkgbuild",
1176             description="Building packages",
1177             descriptionDone="Packages built",
1178             command=[
1179                 "make",
1180                 Interpolate("-j%(prop:nproc:-1)s"),
1181                 "package/compile",
1182                 "V=s",
1183                 "IGNORE_ERRORS=n m",
1184                 "BUILD_LOG=1",
1185             ],
1186             env=MakeEnv(),
1187             haltOnFailure=True,
1188         )
1189     )
1190
1191     factory.addStep(
1192         ShellCommand(
1193             name="pkginstall",
1194             description="Installing packages",
1195             descriptionDone="Packages installed",
1196             command=[
1197                 "make",
1198                 Interpolate("-j%(prop:nproc:-1)s"),
1199                 "package/install",
1200                 "V=s",
1201             ],
1202             env=MakeEnv(),
1203             haltOnFailure=True,
1204         )
1205     )
1206
1207     factory.addStep(
1208         ShellCommand(
1209             name="pkgindex",
1210             description="Indexing packages",
1211             descriptionDone="Packages indexed",
1212             command=[
1213                 "make",
1214                 Interpolate("-j%(prop:nproc:-1)s"),
1215                 "package/index",
1216                 "V=s",
1217                 "CONFIG_SIGNED_PACKAGES=",
1218             ],
1219             env=MakeEnv(),
1220             haltOnFailure=True,
1221         )
1222     )
1223
1224     factory.addStep(
1225         ShellCommand(
1226             name="images",
1227             description="Building and installing images",
1228             descriptionDone="Images built and installed",
1229             command=[
1230                 "make",
1231                 Interpolate("-j%(prop:nproc:-1)s"),
1232                 "target/install",
1233                 "V=s",
1234             ],
1235             env=MakeEnv(),
1236             haltOnFailure=True,
1237         )
1238     )
1239
1240     factory.addStep(
1241         ShellCommand(
1242             name="buildinfo",
1243             description="Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
1244             command="make -j1 buildinfo V=s || true",
1245             env=MakeEnv(),
1246             haltOnFailure=True,
1247         )
1248     )
1249
1250     factory.addStep(
1251         ShellCommand(
1252             name="json_overview_image_info",
1253             description="Generating profiles.json in target folder",
1254             command="make -j1 json_overview_image_info V=s || true",
1255             env=MakeEnv(),
1256             haltOnFailure=True,
1257         )
1258     )
1259
1260     factory.addStep(
1261         ShellCommand(
1262             name="checksums",
1263             description="Calculating checksums",
1264             descriptionDone="Checksums calculated",
1265             command=["make", "-j1", "checksum", "V=s"],
1266             env=MakeEnv(),
1267             haltOnFailure=True,
1268         )
1269     )
1270
1271     factory.addStep(
1272         ShellCommand(
1273             name="kmoddir",
1274             descriptionDone="Kmod directory created",
1275             command=[
1276                 "mkdir",
1277                 "-p",
1278                 Interpolate(
1279                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s",
1280                     target=target,
1281                     subtarget=subtarget,
1282                 ),
1283             ],
1284             haltOnFailure=True,
1285             doStepIf=IsKmodArchiveEnabled,
1286         )
1287     )
1288
1289     factory.addStep(
1290         ShellCommand(
1291             name="kmodprepare",
1292             description="Preparing kmod archive",
1293             descriptionDone="Kmod archive prepared",
1294             command=[
1295                 "rsync",
1296                 "--include=/kmod-*.ipk",
1297                 "--exclude=*",
1298                 "-va",
1299                 Interpolate(
1300                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/",
1301                     target=target,
1302                     subtarget=subtarget,
1303                 ),
1304                 Interpolate(
1305                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1306                     target=target,
1307                     subtarget=subtarget,
1308                 ),
1309             ],
1310             haltOnFailure=True,
1311             doStepIf=IsKmodArchiveEnabled,
1312         )
1313     )
1314
1315     factory.addStep(
1316         ShellCommand(
1317             name="kmodindex",
1318             description="Indexing kmod archive",
1319             descriptionDone="Kmod archive indexed",
1320             command=[
1321                 "make",
1322                 Interpolate("-j%(prop:nproc:-1)s"),
1323                 "package/index",
1324                 "V=s",
1325                 "CONFIG_SIGNED_PACKAGES=",
1326                 Interpolate(
1327                     "PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1328                     target=target,
1329                     subtarget=subtarget,
1330                 ),
1331             ],
1332             env=MakeEnv(),
1333             haltOnFailure=True,
1334             doStepIf=IsKmodArchiveEnabled,
1335         )
1336     )
1337
1338     # sign
1339     factory.addStep(
1340         MasterShellCommand(
1341             name="signprepare",
1342             descriptionDone="Temporary signing directory prepared",
1343             command=["mkdir", "-p", "%s/signing" % (work_dir)],
1344             haltOnFailure=True,
1345             doStepIf=IsSignEnabled,
1346         )
1347     )
1348
1349     factory.addStep(
1350         ShellCommand(
1351             name="signpack",
1352             description="Packing files to sign",
1353             descriptionDone="Files to sign packed",
1354             command=Interpolate(
1355                 "find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ "
1356                 "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ "
1357                 "-mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or "
1358                 "-name Packages -print0 | xargs -0 tar -czf sign.tar.gz",
1359                 target=target,
1360                 subtarget=subtarget,
1361             ),
1362             haltOnFailure=True,
1363             doStepIf=IsSignEnabled,
1364         )
1365     )
1366
1367     factory.addStep(
1368         FileUpload(
1369             workersrc="sign.tar.gz",
1370             masterdest="%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1371             haltOnFailure=True,
1372             doStepIf=IsSignEnabled,
1373         )
1374     )
1375
1376     factory.addStep(
1377         MasterShellCommand(
1378             name="signfiles",
1379             description="Signing files",
1380             descriptionDone="Files signed",
1381             command=[
1382                 "%s/signall.sh" % (scripts_dir),
1383                 "%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1384                 Interpolate("%(prop:branch)s"),
1385             ],
1386             env={"CONFIG_INI": os.getenv("BUILDMASTER_CONFIG", "./config.ini")},
1387             haltOnFailure=True,
1388             doStepIf=IsSignEnabled,
1389         )
1390     )
1391
1392     factory.addStep(
1393         FileDownload(
1394             name="dlsigntargz",
1395             mastersrc="%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1396             workerdest="sign.tar.gz",
1397             haltOnFailure=True,
1398             doStepIf=IsSignEnabled,
1399         )
1400     )
1401
1402     factory.addStep(
1403         ShellCommand(
1404             name="signunpack",
1405             description="Unpacking signed files",
1406             descriptionDone="Signed files unpacked",
1407             command=["tar", "-xzf", "sign.tar.gz"],
1408             haltOnFailure=True,
1409             doStepIf=IsSignEnabled,
1410         )
1411     )
1412
1413     # upload
1414     factory.addStep(
1415         ShellCommand(
1416             name="dirprepare",
1417             descriptionDone="Upload directory structure prepared",
1418             command=[
1419                 "mkdir",
1420                 "-p",
1421                 Interpolate(
1422                     "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s",
1423                     target=target,
1424                     subtarget=subtarget,
1425                     prefix=GetVersionPrefix,
1426                 ),
1427             ],
1428             haltOnFailure=True,
1429         )
1430     )
1431
1432     factory.addStep(
1433         ShellCommand(
1434             name="linkprepare",
1435             descriptionDone="Repository symlink prepared",
1436             command=[
1437                 "ln",
1438                 "-s",
1439                 "-f",
1440                 Interpolate(
1441                     "../packages-%(kw:basever)s",
1442                     basever=util.Transform(GetBaseVersion, Property("branch")),
1443                 ),
1444                 Interpolate(
1445                     "tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix
1446                 ),
1447             ],
1448             doStepIf=IsNoMasterBuild,
1449             haltOnFailure=True,
1450         )
1451     )
1452
1453     factory.addStep(
1454         ShellCommand(
1455             name="kmoddirprepare",
1456             descriptionDone="Kmod archive upload directory prepared",
1457             command=[
1458                 "mkdir",
1459                 "-p",
1460                 Interpolate(
1461                     "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s",
1462                     target=target,
1463                     subtarget=subtarget,
1464                     prefix=GetVersionPrefix,
1465                 ),
1466             ],
1467             haltOnFailure=True,
1468             doStepIf=IsKmodArchiveEnabled,
1469         )
1470     )
1471
1472     factory.addStep(
1473         ShellCommand(
1474             name="dirupload",
1475             description="Uploading directory structure",
1476             descriptionDone="Directory structure uploaded",
1477             command=["rsync", Interpolate("-az%(prop:rsync_ipv4:+4)s")]
1478             + rsync_defopts
1479             + [
1480                 "tmp/upload/",
1481                 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url")),
1482             ],
1483             env={
1484                 "RSYNC_PASSWORD": Interpolate(
1485                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1486                 )
1487             },
1488             haltOnFailure=True,
1489             logEnviron=False,
1490             locks=NetLockUl,
1491             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1492         )
1493     )
1494
1495     # download remote sha256sums to 'target-sha256sums'
1496     factory.addStep(
1497         ShellCommand(
1498             name="target-sha256sums",
1499             description="Fetching remote sha256sums for target",
1500             descriptionDone="Remote sha256sums for target fetched",
1501             command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
1502             + rsync_defopts
1503             + [
1504                 Interpolate(
1505                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
1506                     url=GetRsyncParams.withArgs("bin", "url"),
1507                     target=target,
1508                     subtarget=subtarget,
1509                     prefix=GetVersionPrefix,
1510                 ),
1511                 "target-sha256sums",
1512             ],
1513             env={
1514                 "RSYNC_PASSWORD": Interpolate(
1515                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1516                 )
1517             },
1518             logEnviron=False,
1519             haltOnFailure=False,
1520             flunkOnFailure=False,
1521             warnOnFailure=False,
1522             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1523         )
1524     )
1525
1526     # build list of files to upload
1527     factory.addStep(
1528         FileDownload(
1529             name="dlsha2rsyncpl",
1530             mastersrc=scripts_dir + "/sha2rsync.pl",
1531             workerdest="../sha2rsync.pl",
1532             mode=0o755,
1533         )
1534     )
1535
1536     factory.addStep(
1537         ShellCommand(
1538             name="buildlist",
1539             description="Building list of files to upload",
1540             descriptionDone="List of files to upload built",
1541             command=[
1542                 "../sha2rsync.pl",
1543                 "target-sha256sums",
1544                 Interpolate(
1545                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
1546                     target=target,
1547                     subtarget=subtarget,
1548                 ),
1549                 "rsynclist",
1550             ],
1551             haltOnFailure=True,
1552         )
1553     )
1554
1555     factory.addStep(
1556         FileDownload(
1557             name="dlrsync.sh",
1558             mastersrc=scripts_dir + "/rsync.sh",
1559             workerdest="../rsync.sh",
1560             mode=0o755,
1561         )
1562     )
1563
1564     # upload new files and update existing ones
1565     factory.addStep(
1566         ShellCommand(
1567             name="targetupload",
1568             description="Uploading target files",
1569             descriptionDone="Target files uploaded",
1570             command=[
1571                 "../rsync.sh",
1572                 "--exclude=/kmods/",
1573                 "--files-from=rsynclist",
1574                 "--delay-updates",
1575                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1576             ]
1577             + rsync_defopts
1578             + [
1579                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1580                 Interpolate(
1581                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1582                     target=target,
1583                     subtarget=subtarget,
1584                 ),
1585                 Interpolate(
1586                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1587                     url=GetRsyncParams.withArgs("bin", "url"),
1588                     target=target,
1589                     subtarget=subtarget,
1590                     prefix=GetVersionPrefix,
1591                 ),
1592             ],
1593             env={
1594                 "RSYNC_PASSWORD": Interpolate(
1595                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1596                 )
1597             },
1598             haltOnFailure=True,
1599             logEnviron=False,
1600             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1601         )
1602     )
1603
1604     # delete files which don't exist locally
1605     factory.addStep(
1606         ShellCommand(
1607             name="targetprune",
1608             description="Pruning target files",
1609             descriptionDone="Target files pruned",
1610             command=[
1611                 "../rsync.sh",
1612                 "--exclude=/kmods/",
1613                 "--delete",
1614                 "--existing",
1615                 "--ignore-existing",
1616                 "--delay-updates",
1617                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1618             ]
1619             + rsync_defopts
1620             + [
1621                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1622                 Interpolate(
1623                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1624                     target=target,
1625                     subtarget=subtarget,
1626                 ),
1627                 Interpolate(
1628                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1629                     url=GetRsyncParams.withArgs("bin", "url"),
1630                     target=target,
1631                     subtarget=subtarget,
1632                     prefix=GetVersionPrefix,
1633                 ),
1634             ],
1635             env={
1636                 "RSYNC_PASSWORD": Interpolate(
1637                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1638                 )
1639             },
1640             haltOnFailure=True,
1641             logEnviron=False,
1642             locks=NetLockUl,
1643             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1644         )
1645     )
1646
1647     factory.addStep(
1648         ShellCommand(
1649             name="kmodupload",
1650             description="Uploading kmod archive",
1651             descriptionDone="Kmod archive uploaded",
1652             command=[
1653                 "../rsync.sh",
1654                 "--delete",
1655                 "--delay-updates",
1656                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1657             ]
1658             + rsync_defopts
1659             + [
1660                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1661                 Interpolate(
1662                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1663                     target=target,
1664                     subtarget=subtarget,
1665                 ),
1666                 Interpolate(
1667                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/",
1668                     url=GetRsyncParams.withArgs("bin", "url"),
1669                     target=target,
1670                     subtarget=subtarget,
1671                     prefix=GetVersionPrefix,
1672                 ),
1673             ],
1674             env={
1675                 "RSYNC_PASSWORD": Interpolate(
1676                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1677                 )
1678             },
1679             haltOnFailure=True,
1680             logEnviron=False,
1681             locks=NetLockUl,
1682             doStepIf=IsKmodArchiveAndRsyncEnabled,
1683         )
1684     )
1685
1686     factory.addStep(
1687         ShellCommand(
1688             name="sourcelist",
1689             description="Finding source archives to upload",
1690             descriptionDone="Source archives to upload found",
1691             command=(
1692                 "find dl/ -maxdepth 1 -type f -not -size 0 "
1693                 "-not -name '.*' -not -name '*.hash' -not -name "
1694                 "'*.dl' -newer .config -printf '%f\\n' > sourcelist"
1695             ),
1696             haltOnFailure=True,
1697         )
1698     )
1699
1700     factory.addStep(
1701         ShellCommand(
1702             name="sourceupload",
1703             description="Uploading source archives",
1704             descriptionDone="Source archives uploaded",
1705             command=[
1706                 "../rsync.sh",
1707                 "--files-from=sourcelist",
1708                 "--size-only",
1709                 "--delay-updates",
1710             ]
1711             + rsync_defopts
1712             + [
1713                 Interpolate(
1714                     "--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s",
1715                     target=target,
1716                     subtarget=subtarget,
1717                 ),
1718                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1719                 "dl/",
1720                 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url")),
1721             ],
1722             env={
1723                 "RSYNC_PASSWORD": Interpolate(
1724                     "%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")
1725                 )
1726             },
1727             haltOnFailure=True,
1728             logEnviron=False,
1729             locks=NetLockUl,
1730             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1731         )
1732     )
1733
1734     factory.addStep(
1735         ShellCommand(
1736             name="df",
1737             description="Reporting disk usage",
1738             command=["df", "-h", "."],
1739             env={"LC_ALL": "C"},
1740             logEnviron=False,
1741             haltOnFailure=False,
1742             flunkOnFailure=False,
1743             warnOnFailure=False,
1744             alwaysRun=True,
1745         )
1746     )
1747
1748     factory.addStep(
1749         ShellCommand(
1750             name="du",
1751             description="Reporting estimated file space usage",
1752             command=["du", "-sh", "."],
1753             env={"LC_ALL": "C"},
1754             logEnviron=False,
1755             haltOnFailure=False,
1756             flunkOnFailure=False,
1757             warnOnFailure=False,
1758             alwaysRun=True,
1759         )
1760     )
1761
1762     factory.addStep(
1763         ShellCommand(
1764             name="ccachestat",
1765             description="Reporting ccache stats",
1766             command=["ccache", "-s"],
1767             logEnviron=False,
1768             want_stderr=False,
1769             haltOnFailure=False,
1770             flunkOnFailure=False,
1771             warnOnFailure=False,
1772             doStepIf=util.Transform(bool, Property("ccache_command")),
1773         )
1774     )
1775
1776     return factory
1777
1778
1779 for brname in branchNames:
1780     for target in targets[brname]:
1781         bldrname = brname + "_" + target
1782         c["builders"].append(
1783             BuilderConfig(
1784                 name=bldrname,
1785                 workernames=workerNames,
1786                 factory=prepareFactory(target),
1787                 tags=[
1788                     brname,
1789                 ],
1790                 nextBuild=GetNextBuild,
1791                 canStartBuild=canStartBuild,
1792             )
1793         )
1794
1795
1796 # STATUS TARGETS
1797
1798 # 'status' is a list of Status Targets. The results of each build will be
1799 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1800 # including web pages, email senders, and IRC bots.
1801
1802 if "status_bind" in inip1:
1803     c["www"] = {
1804         "port": inip1.get("status_bind"),
1805         "plugins": {"waterfall_view": True, "console_view": True, "grid_view": True},
1806     }
1807
1808     if "status_user" in inip1 and "status_password" in inip1:
1809         c["www"]["auth"] = util.UserPasswordAuth(
1810             [(inip1.get("status_user"), inip1.get("status_password"))]
1811         )
1812         c["www"]["authz"] = util.Authz(
1813             allowRules=[util.AnyControlEndpointMatcher(role="admins")],
1814             roleMatchers=[
1815                 util.RolesFromUsername(
1816                     roles=["admins"], usernames=[inip1.get("status_user")]
1817                 )
1818             ],
1819         )
1820
1821 c["services"] = []
1822 if ini.has_section("irc"):
1823     iniirc = ini["irc"]
1824     irc_host = iniirc.get("host", None)
1825     irc_port = iniirc.getint("port", 6667)
1826     irc_chan = iniirc.get("channel", None)
1827     irc_nick = iniirc.get("nickname", None)
1828     irc_pass = iniirc.get("password", None)
1829
1830     if irc_host and irc_nick and irc_chan:
1831         irc = reporters.IRC(
1832             irc_host,
1833             irc_nick,
1834             port=irc_port,
1835             password=irc_pass,
1836             channels=[irc_chan],
1837             notify_events=["exception", "problem", "recovery"],
1838         )
1839
1840         c["services"].append(irc)
1841
1842 c["revlink"] = util.RevlinkMatch(
1843     [r"https://git.openwrt.org/openwrt/(.*).git"],
1844     r"https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s",
1845 )
1846
1847 # DB URL
1848
1849 c["db"] = {
1850     # This specifies what database buildbot uses to store its state.  You can leave
1851     # this at its default for all but the largest installations.
1852     "db_url": "sqlite:///state.sqlite",
1853 }
1854
1855 c["buildbotNetUsageData"] = None