phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.process import buildstep
  21 from buildbot.plugins import reporters
  22 from buildbot.plugins import schedulers
  23 from buildbot.plugins import steps
  24 from buildbot.plugins import util
  25 from buildbot.process import properties
  26 from buildbot.process import results
  27 from buildbot.process.factory import BuildFactory
  28 from buildbot.process.properties import Interpolate
  29 from buildbot.process.properties import Property
  30 from buildbot.schedulers.basic import AnyBranchScheduler
  31 from buildbot.schedulers.forcesched import BaseParameter
  32 from buildbot.schedulers.forcesched import ForceScheduler
  33 from buildbot.schedulers.forcesched import ValidationError
  34 from buildbot.steps.master import MasterShellCommand
  35 from buildbot.steps.shell import SetPropertyFromCommand
  36 from buildbot.steps.shell import ShellCommand
  37 from buildbot.steps.source.git import Git
  38 from buildbot.steps.transfer import FileDownload
  39 from buildbot.steps.transfer import FileUpload
  40 from buildbot.steps.transfer import StringDownload
  41 from buildbot.worker import Worker
  42 from buildbot.worker.local import LocalWorker
  43
  44
  45 if not os.path.exists("twistd.pid"):
  46     with open("twistd.pid", "w") as pidfile:
  47         pidfile.write("{}".format(os.getpid()))
  48
  49 # This is a sample buildmaster config file. It must be installed as
  50 # 'master.cfg' in your buildmaster's base directory.
  51
  52 ini = configparser.ConfigParser()
  53 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  54
  55 if "general" not in ini or "phase1" not in ini:
  56     raise ValueError("Fix your configuration")
  57
  58 inip1 = ini["phase1"]
  59
  60 # Globals
  61 work_dir = os.path.abspath(ini["general"].get("workdir", "."))
  62 scripts_dir = os.path.abspath("../scripts")
  63
  64 repo_url = ini["repo"].get("url")
  65 tree_stable_timer = ini["repo"].getint("tree_stable_timer", 15 * 60)
  66
  67 rsync_defopts = ["-v", "--timeout=120"]
  68
  69 # if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  70 #       rsync_bin_defopts += ["--contimeout=20"]
  71
  72 branches = {}
  73
  74
  75 def ini_parse_branch(section):
  76     b = {}
  77     name = section.get("name")
  78
  79     if not name:
  80         raise ValueError("missing 'name' in " + repr(section))
  81     if name in branches:
  82         raise ValueError("duplicate branch name in " + repr(section))
  83
  84     b["name"] = name
  85     b["bin_url"] = section.get("binary_url")
  86     b["bin_key"] = section.get("binary_password")
  87
  88     b["src_url"] = section.get("source_url")
  89     b["src_key"] = section.get("source_password")
  90
  91     b["gpg_key"] = section.get("gpg_key")
  92
  93     b["usign_key"] = section.get("usign_key")
  94     usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  95     b["usign_comment"] = section.get("usign_comment", usign_comment)
  96
  97     b["config_seed"] = section.get("config_seed")
  98     b["build_targets"] = section.get("build_targets")
  99
 100     b["kmod_archive"] = section.getboolean("kmod_archive", False)
 101
 102     branches[name] = b
 103     log.msg("Configured branch: {}".format(name))
 104
 105
 106 # PB port can be either a numeric port or a connection string
 107 pb_port = inip1.get("port") or 9989
 108
 109 # This is the dictionary that the buildmaster pays attention to. We also use
 110 # a shorter alias to save typing.
 111 c = BuildmasterConfig = {}
 112
 113 # PROJECT IDENTITY
 114
 115 # the 'title' string will appear at the top of this buildbot
 116 # installation's html.WebStatus home page (linked to the
 117 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 118
 119 c["title"] = ini["general"].get("title")
 120 c["titleURL"] = ini["general"].get("title_url")
 121
 122 # the 'buildbotURL' string should point to the location where the buildbot's
 123 # internal web server (usually the html.WebStatus page) is visible. This
 124 # typically uses the port number set in the Waterfall 'status' entry, but
 125 # with an externally-visible host name which the buildbot cannot figure out
 126 # without some help.
 127
 128 c["buildbotURL"] = inip1.get("buildbot_url")
 129
 130 # BUILDWORKERS
 131
 132 # The 'workers' list defines the set of recognized buildworkers. Each element is
 133 # a Worker object, specifying a unique worker name and password.  The same
 134 # worker name and password must be configured on the worker.
 135
 136 c["workers"] = []
 137 NetLocks = dict()
 138
 139
 140 def ini_parse_workers(section):
 141     name = section.get("name")
 142     password = section.get("password")
 143     phase = section.getint("phase")
 144     tagonly = section.getboolean("tag_only")
 145     rsyncipv4 = section.getboolean("rsync_ipv4")
 146
 147     if not name or not password or not phase == 1:
 148         log.msg("invalid worker configuration ignored: {}".format(repr(section)))
 149         return
 150
 151     sl_props = {"tag_only": tagonly}
 152     if "dl_lock" in section:
 153         lockname = section.get("dl_lock")
 154         sl_props["dl_lock"] = lockname
 155         if lockname not in NetLocks:
 156             NetLocks[lockname] = locks.MasterLock(lockname)
 157     if "ul_lock" in section:
 158         lockname = section.get("ul_lock")
 159         sl_props["ul_lock"] = lockname
 160         if lockname not in NetLocks:
 161             NetLocks[lockname] = locks.MasterLock(lockname)
 162     if rsyncipv4:
 163         sl_props[
 164             "rsync_ipv4"
 165         ] = True  # only set prop if required, we use '+' Interpolate substitution
 166
 167     log.msg("Configured worker: {}".format(name))
 168     # NB: phase1 build factory requires workers to be single-build only
 169     c["workers"].append(Worker(name, password, max_builds=1, properties=sl_props))
 170
 171
 172 for section in ini.sections():
 173     if section.startswith("branch "):
 174         ini_parse_branch(ini[section])
 175
 176     if section.startswith("worker "):
 177         ini_parse_workers(ini[section])
 178
 179 # list of branches in build-priority order
 180 branchNames = [branches[b]["name"] for b in branches]
 181
 182 c["protocols"] = {"pb": {"port": pb_port}}
 183
 184 # coalesce builds
 185 c["collapseRequests"] = True
 186
 187 # Reduce amount of backlog data
 188 c["configurators"] = [
 189     util.JanitorConfigurator(
 190         logHorizon=timedelta(days=3),
 191         hour=6,
 192     )
 193 ]
 194
 195
 196 @defer.inlineCallbacks
 197 def getNewestCompleteTimePrio(bldr):
 198     """Returns the priority and the complete_at of the latest completed and not SKIPPED
 199     build request for this builder, or None if there are no such build
 200     requests. We need to filter out SKIPPED requests because we're
 201     using collapseRequests=True which is unfortunately marking all
 202     previous requests as complete when new buildset is created.
 203
 204     @returns: (priority, datetime instance or None), via Deferred
 205     """
 206
 207     prio = yield bldr.get_highest_priority()
 208     if prio is None:
 209         prio = 0
 210
 211     bldrid = yield bldr.getBuilderId()
 212     completed = yield bldr.master.data.get(
 213         ("builders", bldrid, "buildrequests"),
 214         [
 215             resultspec.Filter("complete", "eq", [True]),
 216             resultspec.Filter("results", "ne", [results.SKIPPED]),
 217         ],
 218         order=["-complete_at"],
 219         limit=1,
 220     )
 221     if not completed:
 222         return (prio, None)
 223
 224     complete_at = completed[0]["complete_at"]
 225
 226     last_build = yield bldr.master.data.get(
 227         ("builds",),
 228         [
 229             resultspec.Filter("builderid", "eq", [bldrid]),
 230         ],
 231         order=["-started_at"],
 232         limit=1,
 233     )
 234
 235     if last_build and last_build[0]:
 236         last_complete_at = last_build[0]["complete_at"]
 237         if last_complete_at and (last_complete_at > complete_at):
 238             return (prio, last_complete_at)
 239
 240     return (prio, complete_at)
 241
 242
 243 @defer.inlineCallbacks
 244 def prioritizeBuilders(master, builders):
 245     """Returns sorted list of builders by their last timestamp of completed and
 246     not skipped build, ordered first by branch name.
 247
 248     @returns: list of sorted builders
 249     """
 250
 251     bldrNamePrio = {"__Janitor": 0, "00_force_build": 0}
 252     i = 1
 253     for bname in branchNames:
 254         bldrNamePrio[bname] = i
 255
 256     def is_building(bldr):
 257         return bool(bldr.building) or bool(bldr.old_building)
 258
 259     def bldr_info(bldr):
 260         d = defer.maybeDeferred(getNewestCompleteTimePrio, bldr)
 261         d.addCallback(lambda retval: (retval, bldr))
 262         return d
 263
 264     def bldr_sort(item):
 265         ((hiprio, complete_at), bldr) = item
 266
 267         # check if we have some high prio build requests pending (i.e. tag builds),
 268         # if so, front-run these builders, while preserving the per-branch static priority
 269         pos = 99
 270         for name, prio in bldrNamePrio.items():
 271             if bldr.name.startswith(name):
 272                 # higher priority (larger positive number) raises position
 273                 pos = prio + 50 - min(hiprio, 50)
 274                 break
 275
 276         # pos order: janitor/local (0), tag builds if any [1..50], !tag builds [51...]
 277
 278         if not complete_at:
 279             date = datetime.min
 280             complete_at = date.replace(tzinfo=tzutc())
 281
 282         if is_building(bldr):
 283             date = datetime.max
 284             complete_at = date.replace(tzinfo=tzutc())
 285
 286         return (pos, complete_at, bldr.name)
 287
 288     results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 289     results.sort(key=bldr_sort)
 290
 291     # for r in results:
 292     #   log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 293
 294     return [r[1] for r in results]
 295
 296
 297 c["prioritizeBuilders"] = prioritizeBuilders
 298
 299 # CHANGESOURCES
 300
 301 # find targets
 302 targets = dict()
 303
 304
 305 def populateTargets():
 306     def buildTargetsConfigured(branch):
 307         builders = branches[branch].get("build_targets")
 308         return builders and set(filter(None, [t.strip() for t in builders.split("\n")]))
 309
 310     for branch in branchNames:
 311         targets[branch] = buildTargetsConfigured(branch)
 312         if not targets[branch]:
 313             populateTargetsForBranch(branch)
 314
 315
 316 def populateTargetsForBranch(branch):
 317     """fetches a shallow clone for passed `branch` and then
 318     executes dump-target-info.pl and collates the results to ensure
 319     targets that only exist in specific branches get built.
 320     This takes a while during master startup but is executed only once.
 321     """
 322     targets[branch] = set()
 323     sourcegit = work_dir + "/source.git"
 324
 325     log.msg(f"Populating targets for {branch}, this will take time")
 326
 327     if os.path.isdir(sourcegit):
 328         subprocess.call(["rm", "-rf", sourcegit])
 329
 330     subprocess.call(
 331         [
 332             "git",
 333             "clone",
 334             "-q",
 335             "--depth=1",
 336             "--branch=" + branch,
 337             repo_url,
 338             sourcegit,
 339         ]
 340     )
 341
 342     os.makedirs(sourcegit + "/tmp", exist_ok=True)
 343     findtargets = subprocess.Popen(
 344         ["./scripts/dump-target-info.pl", "targets"],
 345         stdout=subprocess.PIPE,
 346         stderr=subprocess.DEVNULL,
 347         cwd=sourcegit,
 348     )
 349
 350     while True:
 351         line = findtargets.stdout.readline()
 352         if not line:
 353             break
 354         ta = line.decode().strip().split(" ")
 355         targets[branch].add(ta[0])
 356
 357     subprocess.call(["rm", "-rf", sourcegit])
 358
 359
 360 populateTargets()
 361
 362 # the 'change_source' setting tells the buildmaster how it should find out
 363 # about source code changes.
 364
 365 c["change_source"] = []
 366 c["change_source"].append(
 367     GitPoller(
 368         repo_url,
 369         workdir=work_dir + "/work.git",
 370         branches=branchNames,
 371         pollAtLaunch=True,
 372         pollInterval=300,
 373     )
 374 )
 375
 376 # SCHEDULERS
 377
 378 # Configure the Schedulers, which decide how to react to incoming changes.
 379
 380
 381 # Selector for known valid tags
 382 class TagChoiceParameter(BaseParameter):
 383     spec_attributes = ["strict", "choices"]
 384     type = "list"
 385     strict = True
 386
 387     def __init__(self, name, label=None, **kw):
 388         super().__init__(name, label, **kw)
 389         self._choice_list = []
 390
 391     def getRevTags(self, findtag=None):
 392         taglist = []
 393         branchvers = []
 394
 395         # we will filter out tags that do no match the configured branches
 396         for b in branchNames:
 397             basever = re.search(r"-([0-9]+\.[0-9]+)$", b)
 398             if basever:
 399                 branchvers.append(basever[1])
 400
 401         # grab tags from remote repository
 402         alltags = subprocess.Popen(
 403             ["git", "ls-remote", "--tags", repo_url], stdout=subprocess.PIPE
 404         )
 405
 406         while True:
 407             line = alltags.stdout.readline()
 408
 409             if not line:
 410                 break
 411
 412             (rev, tag) = line.split()
 413
 414             # does it match known format? ('vNN.NN.NN(-rcN)')
 415             tagver = re.search(
 416                 r"\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$",
 417                 tag.decode().strip(),
 418             )
 419
 420             # only list valid tags matching configured branches
 421             if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
 422                 # if we want a specific tag, ignore all that don't match
 423                 if findtag and findtag != tagver[1]:
 424                     continue
 425                 taglist.append({"rev": rev.decode().strip(), "tag": tagver[1]})
 426
 427         return taglist
 428
 429     @property
 430     def choices(self):
 431         taglist = [rt["tag"] for rt in self.getRevTags()]
 432         taglist.sort(
 433             reverse=True,
 434             key=lambda tag: tag if re.search(r"-rc[0-9]+$", tag) else tag + "-z",
 435         )
 436         taglist.insert(0, "")
 437
 438         self._choice_list = taglist
 439
 440         return self._choice_list
 441
 442     def updateFromKwargs(self, properties, kwargs, **unused):
 443         tag = self.getFromKwargs(kwargs)
 444         properties[self.name] = tag
 445
 446         # find the commit matching the tag
 447         findtag = self.getRevTags(tag)
 448
 449         if not findtag:
 450             raise ValidationError("Couldn't find tag")
 451
 452         properties["force_revision"] = findtag[0]["rev"]
 453
 454         # find the branch matching the tag
 455         branch = None
 456         branchver = re.search(r"v([0-9]+\.[0-9]+)", tag)
 457         for b in branchNames:
 458             if b.endswith(branchver[1]):
 459                 branch = b
 460
 461         if not branch:
 462             raise ValidationError("Couldn't find branch")
 463
 464         properties["force_branch"] = branch
 465
 466     def parse_from_arg(self, s):
 467         if self.strict and s not in self._choice_list:
 468             raise ValidationError(
 469                 "'%s' does not belong to list of available choices '%s'"
 470                 % (s, self._choice_list)
 471             )
 472         return s
 473
 474
 475 @util.renderer
 476 @defer.inlineCallbacks
 477 def builderNames(props):
 478     """since we have per branch and per target builders,
 479     address the relevant builder for each new buildrequest
 480     based on the request's desired branch and target.
 481     """
 482     branch = props.getProperty("branch")
 483     target = props.getProperty("target", "")
 484
 485     if target == "all":
 486         target = ""
 487
 488     # if that didn't work, try sourcestamp to find a branch
 489     if not branch:
 490         # match builders with target branch
 491         ss = props.sourcestamps[0]
 492         if ss:
 493             branch = ss["branch"]
 494         else:
 495             log.msg("couldn't find builder")
 496             return []  # nothing works
 497
 498     bname = branch + "_" + target
 499     builders = []
 500
 501     for b in (yield props.master.data.get(("builders",))):
 502         if not b["name"].startswith(bname):
 503             continue
 504         builders.append(b["name"])
 505
 506     return builders
 507
 508
 509 c["schedulers"] = []
 510 c["schedulers"].append(
 511     AnyBranchScheduler(
 512         name="all",
 513         change_filter=util.ChangeFilter(branch=branchNames),
 514         treeStableTimer=tree_stable_timer,
 515         builderNames=builderNames,
 516     )
 517 )
 518
 519 c["schedulers"].append(
 520     ForceScheduler(
 521         name="force",
 522         buttonName="Force builds",
 523         label="Force build details",
 524         builderNames=["00_force_build"],
 525         codebases=[
 526             util.CodebaseParameter(
 527                 "",
 528                 label="Repository",
 529                 branch=util.FixedParameter(name="branch", default=""),
 530                 revision=util.FixedParameter(name="revision", default=""),
 531                 repository=util.FixedParameter(name="repository", default=""),
 532                 project=util.FixedParameter(name="project", default=""),
 533             )
 534         ],
 535         reason=util.StringParameter(
 536             name="reason",
 537             label="Reason",
 538             default="Trigger build",
 539             required=True,
 540             size=80,
 541         ),
 542         properties=[
 543             # NB: avoid nesting to simplify processing of properties
 544             util.ChoiceStringParameter(
 545                 name="target",
 546                 label="Build target",
 547                 default="all",
 548                 choices=["all"] + [t for b in branchNames for t in targets[b]],
 549             ),
 550             TagChoiceParameter(name="tag", label="Build tag", default=""),
 551         ],
 552     )
 553 )
 554
 555 c["schedulers"].append(
 556     schedulers.Triggerable(name="trigger", builderNames=builderNames, priority=20)
 557 )
 558
 559 # BUILDERS
 560
 561 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 562 # what steps, and which workers can execute them.  Note that any particular build will
 563 # only take place on one worker.
 564
 565
 566 def IsNoMasterBuild(step):
 567     branch = step.getProperty("branch")
 568     return branch not in ["main", "master"]
 569
 570
 571 def IsUsignEnabled(step):
 572     branch = step.getProperty("branch")
 573     return branch and branches[branch].get("usign_key")
 574
 575
 576 def IsSignEnabled(step):
 577     branch = step.getProperty("branch")
 578     return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 579
 580
 581 def IsKmodArchiveEnabled(step):
 582     branch = step.getProperty("branch")
 583     return branch and branches[branch].get("kmod_archive")
 584
 585
 586 def IsKmodArchiveAndRsyncEnabled(step):
 587     branch = step.getProperty("branch")
 588     return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url"))
 589
 590
 591 def IsRemoteShaSumsAvailable(step):
 592     return step.getProperty("have_remote_shasums")
 593
 594
 595 def GetBaseVersion(branch):
 596     if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 597         return branch.split("-")[1]
 598     else:
 599         return "main"
 600
 601
 602 @properties.renderer
 603 def GetVersionPrefix(props):
 604     branch = props.getProperty("branch")
 605     basever = GetBaseVersion(branch)
 606     if props.hasProperty("tag") and re.match(
 607         r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]
 608     ):
 609         return "%s/" % props["tag"][1:]
 610     elif basever != "main":
 611         return "%s-SNAPSHOT/" % basever
 612     else:
 613         return ""
 614
 615
 616 @util.renderer
 617 def GetConfigSeed(props):
 618     branch = props.getProperty("branch")
 619     return branch and branches[branch].get("config_seed") or ""
 620
 621
 622 @util.renderer
 623 def GetRsyncParams(props, srcorbin, urlorkey):
 624     # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 625     branch = props.getProperty("branch")
 626     opt = srcorbin + "_" + urlorkey
 627     return branch and branches[branch].get(opt)
 628
 629
 630 @util.renderer
 631 def GetUsignKey(props):
 632     branch = props.getProperty("branch")
 633     return branch and branches[branch].get("usign_key")
 634
 635
 636 def GetNextBuild(builder, requests):
 637     for r in requests:
 638         if r.properties:
 639             # order tagged build first
 640             if r.properties.hasProperty("tag"):
 641                 return r
 642
 643     r = requests[0]
 644     # log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 645     return r
 646
 647
 648 def MakeEnv(overrides=None, tryccache=False):
 649     env = {
 650         "CCC": Interpolate("%(prop:cc_command:-gcc)s"),
 651         "CCXX": Interpolate("%(prop:cxx_command:-g++)s"),
 652     }
 653     if tryccache:
 654         env["CC"] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 655         env["CXX"] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 656         env["CCACHE"] = Interpolate("%(prop:ccache_command:-)s")
 657     else:
 658         env["CC"] = env["CCC"]
 659         env["CXX"] = env["CCXX"]
 660         env["CCACHE"] = ""
 661     if overrides is not None:
 662         env.update(overrides)
 663     return env
 664
 665
 666 @properties.renderer
 667 def NetLockDl(props, extralock=None):
 668     lock = None
 669     if props.hasProperty("dl_lock"):
 670         lock = NetLocks[props["dl_lock"]]
 671     if lock is not None:
 672         return [lock.access("exclusive")]
 673     else:
 674         return []
 675
 676
 677 @properties.renderer
 678 def NetLockUl(props):
 679     lock = None
 680     if props.hasProperty("ul_lock"):
 681         lock = NetLocks[props["ul_lock"]]
 682     if lock is not None:
 683         return [lock.access("exclusive")]
 684     else:
 685         return []
 686
 687
 688 def IsTargetSelected(target):
 689     def CheckTargetProperty(step):
 690         selected_target = step.getProperty("target", "all")
 691         if selected_target != "all" and selected_target != target:
 692             return False
 693         return True
 694
 695     return CheckTargetProperty
 696
 697
 698 @util.renderer
 699 def UsignSec2Pub(props):
 700     branch = props.getProperty("branch")
 701     try:
 702         comment = (
 703             branches[branch].get("usign_comment") or "untrusted comment: secret key"
 704         )
 705         seckey = branches[branch].get("usign_key")
 706         seckey = base64.b64decode(seckey)
 707     except Exception:
 708         return None
 709
 710     return "{}\n{}".format(
 711         re.sub(r"\bsecret key$", "public key", comment),
 712         base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]),
 713     )
 714
 715
 716 def canStartBuild(builder, wfb, request):
 717     """filter out non tag requests for tag_only workers."""
 718     wtagonly = wfb.worker.properties.getProperty("tag_only")
 719     tag = request.properties.getProperty("tag")
 720
 721     if wtagonly and not tag:
 722         return False
 723
 724     return True
 725
 726
 727 c["builders"] = []
 728
 729 workerNames = []
 730
 731 for worker in c["workers"]:
 732     workerNames.append(worker.workername)
 733
 734 # add a single LocalWorker to handle the forcebuild builder
 735 c["workers"].append(LocalWorker("__local_force_build", max_builds=1))
 736
 737 force_factory = BuildFactory()
 738 force_factory.addStep(
 739     steps.Trigger(
 740         name="trigger_build",
 741         schedulerNames=["trigger"],
 742         sourceStamps=[
 743             {
 744                 "codebase": "",
 745                 "branch": Property("force_branch"),
 746                 "revision": Property("force_revision"),
 747                 "repository": repo_url,
 748                 "project": "",
 749             }
 750         ],
 751         set_properties={
 752             "reason": Property("reason"),
 753             "tag": Property("tag"),
 754             "target": Property("target"),
 755         },
 756     )
 757 )
 758
 759 c["builders"].append(
 760     BuilderConfig(
 761         name="00_force_build", workername="__local_force_build", factory=force_factory
 762     )
 763 )
 764
 765
 766 # CUSTOM CLASS
 767
 768 # Extension of ShellCommand and sets in property:
 769 # - True: the command succeded
 770 # - False: the command failed
 771 class ShellCommandAndSetProperty(buildstep.ShellMixin, buildstep.BuildStep):
 772     name = "shellandsetproperty"
 773     renderables = ['property']
 774
 775     def __init__(
 776         self,
 777         property=None,
 778         **kwargs,
 779     ):
 780         kwargs = self.setupShellMixin(kwargs)
 781
 782         self.property = property
 783
 784         super().__init__(**kwargs)
 785
 786     @defer.inlineCallbacks
 787     def run(self):
 788         cmd = yield self.makeRemoteShellCommand()
 789
 790         yield self.runCommand(cmd)
 791
 792         self.setProperty(self.property, not cmd.didFail(), "ShellCommandAndSetProperty Step")
 793
 794         return cmd.results()
 795
 796
 797 # NB the phase1 build factory assumes workers are single-build only
 798 def prepareFactory(target):
 799     (target, subtarget) = target.split("/")
 800
 801     factory = BuildFactory()
 802
 803     # setup shared work directory if required
 804     factory.addStep(
 805         ShellCommand(
 806             name="sharedwd",
 807             descriptionDone="Shared work directory set up",
 808             command='test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 809             workdir=".",
 810             haltOnFailure=True,
 811         )
 812     )
 813
 814     # find number of cores
 815     factory.addStep(
 816         SetPropertyFromCommand(
 817             name="nproc",
 818             property="nproc",
 819             description="Finding number of CPUs",
 820             command=["nproc"],
 821         )
 822     )
 823
 824     # find gcc and g++ compilers
 825     factory.addStep(
 826         FileDownload(
 827             name="dlfindbinpl",
 828             mastersrc=scripts_dir + "/findbin.pl",
 829             workerdest="../findbin.pl",
 830             mode=0o755,
 831         )
 832     )
 833
 834     factory.addStep(
 835         SetPropertyFromCommand(
 836             name="gcc",
 837             property="cc_command",
 838             description="Finding gcc command",
 839             command=["../findbin.pl", "gcc", "", ""],
 840             haltOnFailure=True,
 841         )
 842     )
 843
 844     factory.addStep(
 845         SetPropertyFromCommand(
 846             name="g++",
 847             property="cxx_command",
 848             description="Finding g++ command",
 849             command=["../findbin.pl", "g++", "", ""],
 850             haltOnFailure=True,
 851         )
 852     )
 853
 854     # see if ccache is available
 855     factory.addStep(
 856         SetPropertyFromCommand(
 857             name="ccache",
 858             property="ccache_command",
 859             description="Testing for ccache command",
 860             command=["which", "ccache"],
 861             haltOnFailure=False,
 862             flunkOnFailure=False,
 863             warnOnFailure=False,
 864             hideStepIf=lambda r, s: r == results.FAILURE,
 865         )
 866     )
 867
 868     # check out the source
 869     # Git() runs:
 870     # if repo doesn't exist: 'git clone repourl'
 871     # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 872     # git cat-file -e <commit>
 873     # git checkout -f <commit>
 874     # git checkout -B <branch>
 875     # git rev-parse HEAD
 876     factory.addStep(
 877         Git(
 878             name="git",
 879             repourl=repo_url,
 880             mode="full",
 881             method="fresh",
 882             locks=NetLockDl,
 883             haltOnFailure=True,
 884         )
 885     )
 886
 887     # workaround for https://github.com/openwrt/buildbot/issues/5
 888     factory.addStep(
 889         Git(
 890             name="git me once more please",
 891             repourl=repo_url,
 892             mode="full",
 893             method="fresh",
 894             locks=NetLockDl,
 895             haltOnFailure=True,
 896         )
 897     )
 898
 899     # update remote refs
 900     factory.addStep(
 901         ShellCommand(
 902             name="fetchrefs",
 903             description="Fetching Git remote refs",
 904             descriptionDone="Git remote refs fetched",
 905             command=[
 906                 "git",
 907                 "fetch",
 908                 "origin",
 909                 Interpolate(
 910                     "+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s"
 911                 ),
 912             ],
 913             haltOnFailure=True,
 914         )
 915     )
 916
 917     # getver.sh requires local branches to track upstream otherwise version computation fails.
 918     # Git() does not set tracking branches when cloning or switching, so work around this here
 919     factory.addStep(
 920         ShellCommand(
 921             name="trackupstream",
 922             description="Setting upstream branch",
 923             descriptionDone="getver.sh is happy now",
 924             command=["git", "branch", "-u", Interpolate("origin/%(prop:branch)s")],
 925             haltOnFailure=True,
 926         )
 927     )
 928
 929     # Verify that Git HEAD points to a tag or branch
 930     # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 931     factory.addStep(
 932         ShellCommand(
 933             name="gitverify",
 934             description="Ensuring that Git HEAD is pointing to a branch or tag",
 935             descriptionDone="Git HEAD is sane",
 936             command=(
 937                 "git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || "
 938                 "git show-ref --tags --dereference 2>/dev/null | sed -ne "
 939                 '"/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."'
 940             ),
 941             haltOnFailure=True,
 942         )
 943     )
 944
 945     factory.addStep(
 946         StringDownload(
 947             name="ccachecc",
 948             s='#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 949             workerdest="../ccache_cc.sh",
 950             mode=0o755,
 951         )
 952     )
 953
 954     factory.addStep(
 955         StringDownload(
 956             name="ccachecxx",
 957             s='#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 958             workerdest="../ccache_cxx.sh",
 959             mode=0o755,
 960         )
 961     )
 962
 963     # feed
 964     factory.addStep(
 965         ShellCommand(
 966             name="updatefeeds",
 967             description="Updating feeds",
 968             command=["./scripts/feeds", "update"],
 969             env=MakeEnv(tryccache=True),
 970             haltOnFailure=True,
 971             locks=NetLockDl,
 972         )
 973     )
 974
 975     # feed
 976     factory.addStep(
 977         ShellCommand(
 978             name="installfeeds",
 979             description="Installing feeds",
 980             command=["./scripts/feeds", "install", "-a"],
 981             env=MakeEnv(tryccache=True),
 982             haltOnFailure=True,
 983         )
 984     )
 985
 986     # seed config
 987     factory.addStep(
 988         StringDownload(
 989             name="dlconfigseed",
 990             s=Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 991             workerdest=".config",
 992             mode=0o644,
 993         )
 994     )
 995
 996     # configure
 997     factory.addStep(
 998         ShellCommand(
 999             name="newconfig",
1000             descriptionDone=".config seeded",
1001             command=Interpolate(
1002                 "printf 'CONFIG_TARGET_%(kw:target)s=y\\n"
1003                 "CONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\n"
1004                 "CONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config",
1005                 target=target,
1006                 subtarget=subtarget,
1007                 usign=GetUsignKey,
1008             ),
1009         )
1010     )
1011
1012     factory.addStep(
1013         ShellCommand(
1014             name="defconfig",
1015             description="Populating .config",
1016             command=["make", "defconfig"],
1017             env=MakeEnv(),
1018         )
1019     )
1020
1021     # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
1022     factory.addStep(
1023         ShellCommand(
1024             name="checkarch",
1025             description="Checking architecture",
1026             descriptionDone="Architecture validated",
1027             command='grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config'
1028             % (target, subtarget),
1029             logEnviron=False,
1030             want_stdout=False,
1031             want_stderr=False,
1032             haltOnFailure=True,
1033             flunkOnFailure=False,  # this is not a build FAILURE - TODO mark build as SKIPPED
1034         )
1035     )
1036
1037     # find libc suffix
1038     factory.addStep(
1039         SetPropertyFromCommand(
1040             name="libc",
1041             property="libc",
1042             description="Finding libc suffix",
1043             command=[
1044                 "sed",
1045                 "-ne",
1046                 '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }',
1047                 ".config",
1048             ],
1049         )
1050     )
1051
1052     # install build key
1053     factory.addStep(
1054         StringDownload(
1055             name="dlkeybuildpub",
1056             s=Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
1057             workerdest="key-build.pub",
1058             mode=0o600,
1059             doStepIf=IsUsignEnabled,
1060         )
1061     )
1062
1063     factory.addStep(
1064         StringDownload(
1065             name="dlkeybuild",
1066             s="# fake private key",
1067             workerdest="key-build",
1068             mode=0o600,
1069             doStepIf=IsUsignEnabled,
1070         )
1071     )
1072
1073     factory.addStep(
1074         StringDownload(
1075             name="dlkeybuilducert",
1076             s="# fake certificate",
1077             workerdest="key-build.ucert",
1078             mode=0o600,
1079             doStepIf=IsUsignEnabled,
1080         )
1081     )
1082
1083     # prepare dl
1084     factory.addStep(
1085         ShellCommand(
1086             name="dldir",
1087             description="Preparing dl/",
1088             descriptionDone="dl/ prepared",
1089             command='mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
1090             workdir=Property("builddir"),
1091             logEnviron=False,
1092             want_stdout=False,
1093         )
1094     )
1095
1096     # cleanup dl
1097     factory.addStep(
1098         ShellCommand(
1099             name="dlprune",
1100             description="Pruning dl/",
1101             descriptionDone="dl/ pruned",
1102             command="find dl/ -mindepth 1 -atime +15 -delete -print",
1103             logEnviron=False,
1104             haltOnFailure=False,
1105             flunkOnFailure=False,
1106             warnOnFailure=False,
1107         )
1108     )
1109
1110     # prepare tar
1111     factory.addStep(
1112         ShellCommand(
1113             name="dltar",
1114             description="Building and installing GNU tar",
1115             descriptionDone="GNU tar built and installed",
1116             command=[
1117                 "make",
1118                 Interpolate("-j%(prop:nproc:-1)s"),
1119                 "tools/tar/compile",
1120                 "V=s",
1121             ],
1122             env=MakeEnv(tryccache=True),
1123             haltOnFailure=True,
1124         )
1125     )
1126
1127     # populate dl
1128     factory.addStep(
1129         ShellCommand(
1130             name="dlrun",
1131             description="Populating dl/",
1132             descriptionDone="dl/ populated",
1133             command=["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
1134             env=MakeEnv(),
1135             logEnviron=False,
1136             locks=NetLockDl,
1137         )
1138     )
1139
1140     factory.addStep(
1141         ShellCommand(
1142             name="cleanbase",
1143             description="Cleaning base-files",
1144             command=["make", "package/base-files/clean", "V=s"],
1145         )
1146     )
1147
1148     # build
1149     factory.addStep(
1150         ShellCommand(
1151             name="tools",
1152             description="Building and installing tools",
1153             descriptionDone="Tools built and installed",
1154             command=[
1155                 "make",
1156                 Interpolate("-j%(prop:nproc:-1)s"),
1157                 "tools/install",
1158                 "V=s",
1159             ],
1160             env=MakeEnv(tryccache=True),
1161             haltOnFailure=True,
1162         )
1163     )
1164
1165     factory.addStep(
1166         ShellCommand(
1167             name="toolchain",
1168             description="Building and installing toolchain",
1169             descriptionDone="Toolchain built and installed",
1170             command=[
1171                 "make",
1172                 Interpolate("-j%(prop:nproc:-1)s"),
1173                 "toolchain/install",
1174                 "V=s",
1175             ],
1176             env=MakeEnv(),
1177             haltOnFailure=True,
1178         )
1179     )
1180
1181     factory.addStep(
1182         ShellCommand(
1183             name="kmods",
1184             description="Building kmods",
1185             descriptionDone="Kmods built",
1186             command=[
1187                 "make",
1188                 Interpolate("-j%(prop:nproc:-1)s"),
1189                 "target/compile",
1190                 "V=s",
1191                 "IGNORE_ERRORS=n m",
1192                 "BUILD_LOG=1",
1193             ],
1194             env=MakeEnv(),
1195             haltOnFailure=True,
1196         )
1197     )
1198
1199     # find kernel version
1200     factory.addStep(
1201         SetPropertyFromCommand(
1202             name="kernelversion",
1203             property="kernelversion",
1204             description="Finding the effective Kernel version",
1205             command=(
1206                 "make --no-print-directory -C target/linux/ "
1207                 "val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | "
1208                 "xargs printf '%s-%s-%s\\n'"
1209             ),
1210             env={"TOPDIR": Interpolate("%(prop:builddir)s/build")},
1211         )
1212     )
1213
1214     factory.addStep(
1215         ShellCommand(
1216             name="pkgclean",
1217             description="Cleaning up package build",
1218             descriptionDone="Package build cleaned up",
1219             command=["make", "package/cleanup", "V=s"],
1220         )
1221     )
1222
1223     factory.addStep(
1224         ShellCommand(
1225             name="pkgbuild",
1226             description="Building packages",
1227             descriptionDone="Packages built",
1228             command=[
1229                 "make",
1230                 Interpolate("-j%(prop:nproc:-1)s"),
1231                 "package/compile",
1232                 "V=s",
1233                 "IGNORE_ERRORS=n m",
1234                 "BUILD_LOG=1",
1235             ],
1236             env=MakeEnv(),
1237             haltOnFailure=True,
1238         )
1239     )
1240
1241     factory.addStep(
1242         ShellCommand(
1243             name="pkginstall",
1244             description="Installing packages",
1245             descriptionDone="Packages installed",
1246             command=[
1247                 "make",
1248                 Interpolate("-j%(prop:nproc:-1)s"),
1249                 "package/install",
1250                 "V=s",
1251             ],
1252             env=MakeEnv(),
1253             haltOnFailure=True,
1254         )
1255     )
1256
1257     factory.addStep(
1258         ShellCommand(
1259             name="images",
1260             description="Building and installing images",
1261             descriptionDone="Images built and installed",
1262             command=[
1263                 "make",
1264                 Interpolate("-j%(prop:nproc:-1)s"),
1265                 "target/install",
1266                 "V=s",
1267             ],
1268             env=MakeEnv(),
1269             haltOnFailure=True,
1270         )
1271     )
1272
1273     factory.addStep(
1274         ShellCommand(
1275             name="buildinfo",
1276             description="Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
1277             command="make -j1 buildinfo V=s || true",
1278             env=MakeEnv(),
1279             haltOnFailure=True,
1280         )
1281     )
1282
1283     factory.addStep(
1284         ShellCommand(
1285             name="json_overview_image_info",
1286             description="Generating profiles.json in target folder",
1287             command="make -j1 json_overview_image_info V=s || true",
1288             env=MakeEnv(),
1289             haltOnFailure=True,
1290         )
1291     )
1292
1293     factory.addStep(
1294         ShellCommand(
1295             name="kmoddir",
1296             descriptionDone="Kmod directory created",
1297             command=[
1298                 "mkdir",
1299                 "-p",
1300                 Interpolate(
1301                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s",
1302                     target=target,
1303                     subtarget=subtarget,
1304                 ),
1305             ],
1306             haltOnFailure=True,
1307             doStepIf=IsKmodArchiveEnabled,
1308         )
1309     )
1310
1311     factory.addStep(
1312         ShellCommand(
1313             name="kmodprepare",
1314             description="Preparing kmod archive",
1315             descriptionDone="Kmod archive prepared",
1316             command=[
1317                 "rsync",
1318                 "--remove-source-files",
1319                 "--include=/kmod-*.ipk",
1320                 "--include=/kmod-*.apk",
1321                 "--exclude=*",
1322                 "-va",
1323                 Interpolate(
1324                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/",
1325                     target=target,
1326                     subtarget=subtarget,
1327                 ),
1328                 Interpolate(
1329                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1330                     target=target,
1331                     subtarget=subtarget,
1332                 ),
1333             ],
1334             haltOnFailure=True,
1335             doStepIf=IsKmodArchiveEnabled,
1336         )
1337     )
1338
1339     factory.addStep(
1340         ShellCommand(
1341             name="pkgindex",
1342             description="Indexing packages",
1343             descriptionDone="Packages indexed",
1344             command=[
1345                 "make",
1346                 Interpolate("-j%(prop:nproc:-1)s"),
1347                 "package/index",
1348                 "V=s",
1349                 "CONFIG_SIGNED_PACKAGES=",
1350             ],
1351             env=MakeEnv(),
1352             haltOnFailure=True,
1353         )
1354     )
1355
1356     factory.addStep(
1357         ShellCommand(
1358             name="kmodindex",
1359             description="Indexing kmod archive",
1360             descriptionDone="Kmod archive indexed",
1361             command=[
1362                 "make",
1363                 Interpolate("-j%(prop:nproc:-1)s"),
1364                 "package/index",
1365                 "V=s",
1366                 "CONFIG_SIGNED_PACKAGES=",
1367                 Interpolate(
1368                     "PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1369                     target=target,
1370                     subtarget=subtarget,
1371                 ),
1372             ],
1373             env=MakeEnv(),
1374             haltOnFailure=True,
1375             doStepIf=IsKmodArchiveEnabled,
1376         )
1377     )
1378
1379     factory.addStep(
1380         ShellCommand(
1381             name="checksums",
1382             description="Calculating checksums",
1383             descriptionDone="Checksums calculated",
1384             command=["make", "-j1", "checksum", "V=s"],
1385             env=MakeEnv(),
1386             haltOnFailure=True,
1387         )
1388     )
1389
1390     # download remote sha256sums to 'target-sha256sums'
1391     factory.addStep(
1392         ShellCommandAndSetProperty(
1393             name="target-sha256sums",
1394             description="Fetching remote sha256sums for target",
1395             descriptionDone="Remote sha256sums for target fetched",
1396             command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
1397             + rsync_defopts
1398             + [
1399                 Interpolate(
1400                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
1401                     url=GetRsyncParams.withArgs("bin", "url"),
1402                     target=target,
1403                     subtarget=subtarget,
1404                     prefix=GetVersionPrefix,
1405                 ),
1406                 "target-sha256sums",
1407             ],
1408             env={
1409                 "RSYNC_PASSWORD": Interpolate(
1410                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1411                 )
1412             },
1413             property="have_remote_shasums",
1414             logEnviron=False,
1415             haltOnFailure=False,
1416             flunkOnFailure=False,
1417             warnOnFailure=False,
1418             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1419         )
1420     )
1421
1422     factory.addStep(
1423         ShellCommand(
1424             name="target-sha256sums_kmodsparse",
1425             description="Extract kmods from remote sha256sums",
1426             descriptionDone="Kmods extracted",
1427             command="sed \"/ \\*kmods\\//! d\" target-sha256sums | tee target-sha256sums-kmods",
1428             haltOnFailure=False,
1429             doStepIf=IsRemoteShaSumsAvailable,
1430         )
1431     )
1432
1433     factory.addStep(
1434         ShellCommand(
1435             name="mergesha256sum",
1436             description="Merge sha256sums kmods with sha256sums",
1437             descriptionDone="Sha256sums merged",
1438             command=[
1439                 "sort",
1440                 "-t", " ",
1441                 "-k", 2,
1442                 "-u",
1443                 Interpolate(
1444                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
1445                     target=target,
1446                     subtarget=subtarget,
1447                 ),
1448                 "target-sha256sums-kmods",
1449                 "-o",
1450                 Interpolate(
1451                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
1452                     target=target,
1453                     subtarget=subtarget,
1454                 ),
1455             ],
1456             env={"LC_ALL": "C"},
1457             haltOnFailure=False,
1458             doStepIf=IsRemoteShaSumsAvailable,
1459         )
1460     )
1461
1462     # sign
1463     factory.addStep(
1464         MasterShellCommand(
1465             name="signprepare",
1466             descriptionDone="Temporary signing directory prepared",
1467             command=["mkdir", "-p", "%s/signing" % (work_dir)],
1468             haltOnFailure=True,
1469             doStepIf=IsSignEnabled,
1470         )
1471     )
1472
1473     factory.addStep(
1474         ShellCommand(
1475             name="signpack",
1476             description="Packing files to sign",
1477             descriptionDone="Files to sign packed",
1478             command=Interpolate(
1479                 "find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ "
1480                 "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ "
1481                 "-mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or "
1482                 "-name Packages -print0 -or -name packages.adb -print0 "
1483                 "| xargs -0 tar -czf sign.tar.gz",
1484                 target=target,
1485                 subtarget=subtarget,
1486             ),
1487             haltOnFailure=True,
1488             doStepIf=IsSignEnabled,
1489         )
1490     )
1491
1492     factory.addStep(
1493         FileUpload(
1494             workersrc="sign.tar.gz",
1495             masterdest="%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1496             haltOnFailure=True,
1497             doStepIf=IsSignEnabled,
1498         )
1499     )
1500
1501     factory.addStep(
1502         MasterShellCommand(
1503             name="signfiles",
1504             description="Signing files",
1505             descriptionDone="Files signed",
1506             command=[
1507                 "%s/signall.sh" % (scripts_dir),
1508                 "%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1509                 Interpolate("%(prop:branch)s"),
1510             ],
1511             env={"CONFIG_INI": os.getenv("BUILDMASTER_CONFIG", "./config.ini")},
1512             haltOnFailure=True,
1513             doStepIf=IsSignEnabled,
1514         )
1515     )
1516
1517     factory.addStep(
1518         FileDownload(
1519             name="dlsigntargz",
1520             mastersrc="%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1521             workerdest="sign.tar.gz",
1522             haltOnFailure=True,
1523             doStepIf=IsSignEnabled,
1524         )
1525     )
1526
1527     factory.addStep(
1528         ShellCommand(
1529             name="signunpack",
1530             description="Unpacking signed files",
1531             descriptionDone="Signed files unpacked",
1532             command=["tar", "-xzf", "sign.tar.gz"],
1533             haltOnFailure=True,
1534             doStepIf=IsSignEnabled,
1535         )
1536     )
1537
1538     # upload
1539     factory.addStep(
1540         ShellCommand(
1541             name="dirprepare",
1542             descriptionDone="Upload directory structure prepared",
1543             command=[
1544                 "mkdir",
1545                 "-p",
1546                 Interpolate(
1547                     "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s",
1548                     target=target,
1549                     subtarget=subtarget,
1550                     prefix=GetVersionPrefix,
1551                 ),
1552             ],
1553             haltOnFailure=True,
1554         )
1555     )
1556
1557     factory.addStep(
1558         ShellCommand(
1559             name="linkprepare",
1560             descriptionDone="Repository symlink prepared",
1561             command=[
1562                 "ln",
1563                 "-s",
1564                 "-f",
1565                 Interpolate(
1566                     "../packages-%(kw:basever)s",
1567                     basever=util.Transform(GetBaseVersion, Property("branch")),
1568                 ),
1569                 Interpolate(
1570                     "tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix
1571                 ),
1572             ],
1573             doStepIf=IsNoMasterBuild,
1574             haltOnFailure=True,
1575         )
1576     )
1577
1578     factory.addStep(
1579         ShellCommand(
1580             name="kmoddirprepare",
1581             descriptionDone="Kmod archive upload directory prepared",
1582             command=[
1583                 "mkdir",
1584                 "-p",
1585                 Interpolate(
1586                     "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s",
1587                     target=target,
1588                     subtarget=subtarget,
1589                     prefix=GetVersionPrefix,
1590                 ),
1591             ],
1592             haltOnFailure=True,
1593             doStepIf=IsKmodArchiveEnabled,
1594         )
1595     )
1596
1597     factory.addStep(
1598         ShellCommand(
1599             name="dirupload",
1600             description="Uploading directory structure",
1601             descriptionDone="Directory structure uploaded",
1602             command=["rsync", Interpolate("-az%(prop:rsync_ipv4:+4)s")]
1603             + rsync_defopts
1604             + [
1605                 "tmp/upload/",
1606                 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url")),
1607             ],
1608             env={
1609                 "RSYNC_PASSWORD": Interpolate(
1610                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1611                 )
1612             },
1613             haltOnFailure=True,
1614             logEnviron=False,
1615             locks=NetLockUl,
1616             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1617         )
1618     )
1619
1620     # build list of files to upload
1621     factory.addStep(
1622         FileDownload(
1623             name="dlsha2rsyncpl",
1624             mastersrc=scripts_dir + "/sha2rsync.pl",
1625             workerdest="../sha2rsync.pl",
1626             mode=0o755,
1627         )
1628     )
1629
1630     factory.addStep(
1631         ShellCommand(
1632             name="buildlist",
1633             description="Building list of files to upload",
1634             descriptionDone="List of files to upload built",
1635             command=[
1636                 "../sha2rsync.pl",
1637                 "target-sha256sums",
1638                 Interpolate(
1639                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
1640                     target=target,
1641                     subtarget=subtarget,
1642                 ),
1643                 "rsynclist",
1644             ],
1645             haltOnFailure=True,
1646         )
1647     )
1648
1649     factory.addStep(
1650         FileDownload(
1651             name="dlrsync.sh",
1652             mastersrc=scripts_dir + "/rsync.sh",
1653             workerdest="../rsync.sh",
1654             mode=0o755,
1655         )
1656     )
1657
1658     # upload new files and update existing ones
1659     factory.addStep(
1660         ShellCommand(
1661             name="targetupload",
1662             description="Uploading target files",
1663             descriptionDone="Target files uploaded",
1664             command=[
1665                 "../rsync.sh",
1666                 "--exclude=/kmods/",
1667                 "--exclude=/kmods/**",
1668                 "--files-from=rsynclist",
1669                 "--delay-updates",
1670                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1671             ]
1672             + rsync_defopts
1673             + [
1674                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1675                 Interpolate(
1676                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1677                     target=target,
1678                     subtarget=subtarget,
1679                 ),
1680                 Interpolate(
1681                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1682                     url=GetRsyncParams.withArgs("bin", "url"),
1683                     target=target,
1684                     subtarget=subtarget,
1685                     prefix=GetVersionPrefix,
1686                 ),
1687             ],
1688             env={
1689                 "RSYNC_PASSWORD": Interpolate(
1690                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1691                 )
1692             },
1693             haltOnFailure=True,
1694             logEnviron=False,
1695             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1696         )
1697     )
1698
1699     # delete files which don't exist locally
1700     factory.addStep(
1701         ShellCommand(
1702             name="targetprune",
1703             description="Pruning target files",
1704             descriptionDone="Target files pruned",
1705             command=[
1706                 "../rsync.sh",
1707                 "--exclude=/kmods/",
1708                 "--delete",
1709                 "--existing",
1710                 "--ignore-existing",
1711                 "--delay-updates",
1712                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1713             ]
1714             + rsync_defopts
1715             + [
1716                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1717                 Interpolate(
1718                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1719                     target=target,
1720                     subtarget=subtarget,
1721                 ),
1722                 Interpolate(
1723                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1724                     url=GetRsyncParams.withArgs("bin", "url"),
1725                     target=target,
1726                     subtarget=subtarget,
1727                     prefix=GetVersionPrefix,
1728                 ),
1729             ],
1730             env={
1731                 "RSYNC_PASSWORD": Interpolate(
1732                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1733                 )
1734             },
1735             haltOnFailure=True,
1736             logEnviron=False,
1737             locks=NetLockUl,
1738             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1739         )
1740     )
1741
1742     factory.addStep(
1743         ShellCommand(
1744             name="kmodupload",
1745             description="Uploading kmod archive",
1746             descriptionDone="Kmod archive uploaded",
1747             command=[
1748                 "../rsync.sh",
1749                 "--delete",
1750                 "--delay-updates",
1751                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1752             ]
1753             + rsync_defopts
1754             + [
1755                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1756                 Interpolate(
1757                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1758                     target=target,
1759                     subtarget=subtarget,
1760                 ),
1761                 Interpolate(
1762                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/",
1763                     url=GetRsyncParams.withArgs("bin", "url"),
1764                     target=target,
1765                     subtarget=subtarget,
1766                     prefix=GetVersionPrefix,
1767                 ),
1768             ],
1769             env={
1770                 "RSYNC_PASSWORD": Interpolate(
1771                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1772                 )
1773             },
1774             haltOnFailure=True,
1775             logEnviron=False,
1776             locks=NetLockUl,
1777             doStepIf=IsKmodArchiveAndRsyncEnabled,
1778         )
1779     )
1780
1781     factory.addStep(
1782         ShellCommand(
1783             name="sourcelist",
1784             description="Finding source archives to upload",
1785             descriptionDone="Source archives to upload found",
1786             command=(
1787                 "find dl/ -maxdepth 1 -type f -not -size 0 "
1788                 "-not -name '.*' -not -name '*.hash' -not -name "
1789                 "'*.dl' -newer .config -printf '%f\\n' > sourcelist"
1790             ),
1791             haltOnFailure=True,
1792         )
1793     )
1794
1795     factory.addStep(
1796         ShellCommand(
1797             name="sourceupload",
1798             description="Uploading source archives",
1799             descriptionDone="Source archives uploaded",
1800             command=[
1801                 "../rsync.sh",
1802                 "--files-from=sourcelist",
1803                 "--size-only",
1804                 "--delay-updates",
1805             ]
1806             + rsync_defopts
1807             + [
1808                 Interpolate(
1809                     "--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s",
1810                     target=target,
1811                     subtarget=subtarget,
1812                 ),
1813                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1814                 "dl/",
1815                 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url")),
1816             ],
1817             env={
1818                 "RSYNC_PASSWORD": Interpolate(
1819                     "%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")
1820                 )
1821             },
1822             haltOnFailure=True,
1823             logEnviron=False,
1824             locks=NetLockUl,
1825             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1826         )
1827     )
1828
1829     factory.addStep(
1830         ShellCommand(
1831             name="df",
1832             description="Reporting disk usage",
1833             command=["df", "-h", "."],
1834             env={"LC_ALL": "C"},
1835             logEnviron=False,
1836             haltOnFailure=False,
1837             flunkOnFailure=False,
1838             warnOnFailure=False,
1839             alwaysRun=True,
1840         )
1841     )
1842
1843     factory.addStep(
1844         ShellCommand(
1845             name="du",
1846             description="Reporting estimated file space usage",
1847             command=["du", "-sh", "."],
1848             env={"LC_ALL": "C"},
1849             logEnviron=False,
1850             haltOnFailure=False,
1851             flunkOnFailure=False,
1852             warnOnFailure=False,
1853             alwaysRun=True,
1854         )
1855     )
1856
1857     factory.addStep(
1858         ShellCommand(
1859             name="ccachestat",
1860             description="Reporting ccache stats",
1861             command=["ccache", "-s"],
1862             logEnviron=False,
1863             want_stderr=False,
1864             haltOnFailure=False,
1865             flunkOnFailure=False,
1866             warnOnFailure=False,
1867             doStepIf=util.Transform(bool, Property("ccache_command")),
1868         )
1869     )
1870
1871     return factory
1872
1873
1874 for brname in branchNames:
1875     for target in targets[brname]:
1876         bldrname = brname + "_" + target
1877         c["builders"].append(
1878             BuilderConfig(
1879                 name=bldrname,
1880                 workernames=workerNames,
1881                 factory=prepareFactory(target),
1882                 tags=[
1883                     brname,
1884                 ],
1885                 nextBuild=GetNextBuild,
1886                 canStartBuild=canStartBuild,
1887             )
1888         )
1889
1890
1891 # STATUS TARGETS
1892
1893 # 'status' is a list of Status Targets. The results of each build will be
1894 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1895 # including web pages, email senders, and IRC bots.
1896
1897 if "status_bind" in inip1:
1898     c["www"] = {
1899         "port": inip1.get("status_bind"),
1900         "plugins": {"waterfall_view": True, "console_view": True, "grid_view": True},
1901     }
1902
1903     if "status_user" in inip1 and "status_password" in inip1:
1904         c["www"]["auth"] = util.UserPasswordAuth(
1905             [(inip1.get("status_user"), inip1.get("status_password"))]
1906         )
1907         c["www"]["authz"] = util.Authz(
1908             allowRules=[util.AnyControlEndpointMatcher(role="admins")],
1909             roleMatchers=[
1910                 util.RolesFromUsername(
1911                     roles=["admins"], usernames=[inip1.get("status_user")]
1912                 )
1913             ],
1914         )
1915
1916 c["services"] = []
1917 if ini.has_section("irc"):
1918     iniirc = ini["irc"]
1919     irc_host = iniirc.get("host", None)
1920     irc_port = iniirc.getint("port", 6667)
1921     irc_chan = iniirc.get("channel", None)
1922     irc_nick = iniirc.get("nickname", None)
1923     irc_pass = iniirc.get("password", None)
1924
1925     if irc_host and irc_nick and irc_chan:
1926         irc = reporters.IRC(
1927             irc_host,
1928             irc_nick,
1929             port=irc_port,
1930             password=irc_pass,
1931             channels=[irc_chan],
1932             notify_events=["exception", "problem", "recovery"],
1933         )
1934
1935         c["services"].append(irc)
1936
1937 c["revlink"] = util.RevlinkMatch(
1938     [r"https://git.openwrt.org/openwrt/(.*).git"],
1939     r"https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s",
1940 )
1941
1942 # DB URL
1943
1944 c["db"] = {
1945     # This specifies what database buildbot uses to store its state.  You can leave
1946     # this at its default for all but the largest installations.
1947     "db_url": "sqlite:///state.sqlite",
1948 }
1949
1950 c["buildbotNetUsageData"] = None